chore: merge main into litellm_batch_jsonl_line_item_callbacks

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

.github/workflows/_test-unit-base.yml

@@ -3,6 +3,11 @@ name: _Unit Test Base (Reusable)
 on:
   workflow_call:
     inputs:
+      python-version:
+        description: "Python version used to install dependencies and run tests"
+        required: false
+        type: string
+        default: "3.12"
       test-path:
         description: >-
           Space-separated pytest paths to run. A path that no longer exists is
@@ -78,7 +83,7 @@ permissions:
   contents: read
 
 env:
-  UV_PYTHON: "3.12"
+  UV_PYTHON: ${{ inputs.python-version }}
   LITELLM_LOCAL_MODEL_COST_MAP: "True"
 
 jobs:

.github/workflows/test-unit-proxy-db.yml

@@ -39,6 +39,21 @@ concurrency:
 #     pinning the whole file to one worker (the default --dist=loadscope
 #     behavior for single-file targets).
 jobs:
+  lens-python-310:
+    name: Lens Python 3.10
+    permissions:
+      contents: read
+      id-token: write
+      pull-requests: write
+    uses: ./.github/workflows/_test-unit-base.yml
+    with:
+      python-version: "3.10"
+      test-path: tests/unit/proxy/lens/test_inference.py
+      workers: 0
+      reruns: 0
+      timeout-minutes: 5
+      artifact-name: lens-python-310
+
   # Fast guard — fails the workflow when a test directory or file inside a sharded
   # tree is claimed by no shard. The semantic-shard design has no catch-all bucket,
   # so an unassigned child runs nowhere; assert_ci_coverage.py holds the tree list

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -22,7 +22,9 @@ from litellm._uuid import uuid
 from litellm.proxy._types import *
 from litellm.proxy.auth.auth_checks import delete_cached_project_object
 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
-from litellm.proxy.management.teams.authz import is_team_admin
+from litellm.proxy.management.teams.authz import TEAM_OR_ORG_ADMIN, TeamAccess, is_team_admin, is_team_member
+from litellm.proxy.management.teams.dependencies import get_team_access
+from litellm.proxy.management.users.service import org_admin_org_ids
 from litellm.proxy.management_endpoints.common_utils import _set_object_metadata_field
 from litellm.proxy.management_endpoints.team_admin_field_permissions import team_admin_may_manage_projects
 from litellm.proxy.management_helpers.utils import (
@@ -118,6 +120,37 @@ async def _check_user_permission_for_project(
     return is_team_admin(user_api_key_dict, team) or user_api_key_dict.user_id in (team.admins or [])
 
 
+async def _can_view_team_projects(
+    user_api_key_dict: UserAPIKeyAuth,
+    team_id: str | None,
+    prisma_client: PrismaClient,
+    team_access: TeamAccess,
+) -> bool:
+    if user_api_key_has_admin_view(user_api_key_dict):
+        return True
+    if not team_id or not user_api_key_dict.user_id:
+        return False
+    team_row: Final = await _team_table(prisma_client).find_unique(where={"team_id": team_id})
+    if team_row is None:
+        return False
+    team: Final = LiteLLM_TeamTable.model_validate(team_row.model_dump())
+    return is_team_member(user_api_key_dict, team) or await team_access.allows(
+        user_api_key_dict, team, TEAM_OR_ORG_ADMIN
+    )
+
+
+def _visible_projects_where(team_ids: list[str], admin_org_ids: frozenset[str]) -> dict[str, object]:
+    # bounded-ok: one caller's team memberships
+    member_scope: Final[dict[str, object]] = {"team_id": {"in": team_ids}}
+    if not admin_org_ids:
+        return member_scope
+    org_scope: Final[dict[str, object]] = {
+        # bounded-ok: orgs one caller administers
+        "litellm_team_table": {"is": {"organization_id": {"in": sorted(admin_org_ids)}}}
+    }
+    return {"OR": [member_scope, org_scope]}
+
+
 async def _validate_team_exists(
     team_id: str,
     prisma_client: PrismaClient,
@@ -973,6 +1006,7 @@ async def delete_project(
 async def project_info(
     project_id: str,
     user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+    team_access: TeamAccess = Depends(get_team_access),
 ):
     """
     Get information about a specific project
@@ -1009,21 +1043,7 @@ async def project_info(
                 param="project_id",
             )
 
-        # Check if user has access to this project (admin or team member)
-        is_admin = user_api_key_has_admin_view(user_api_key_dict)
-        is_team_member = False
-
-        if project.team_id and user_api_key_dict.user_id:
-            team = await _team_table(prisma_client).find_unique(where={"team_id": project.team_id})
-            if team:
-                caller_user_id = user_api_key_dict.user_id
-                for m in team.members_with_roles or []:
-                    m_user_id = m.get("user_id") if isinstance(m, dict) else getattr(m, "user_id", None)
-                    if m_user_id == caller_user_id:
-                        is_team_member = True
-                        break
-
-        if not (is_admin or is_team_member):
+        if not await _can_view_team_projects(user_api_key_dict, project.team_id, prisma_client, team_access):
             raise HTTPException(
                 status_code=403,
                 detail={"error": "You don't have access to this project"},
@@ -1072,16 +1092,13 @@ async def list_projects(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
-            # Look up the user's team memberships via the reverse-index on
-            # LiteLLM_UserTable.teams (maintained by team_member_add alongside
-            # members_with_roles). This avoids a full scan of all team rows.
             user_record: Final = await _user_table(prisma_client).find_unique(
                 where={"user_id": user_api_key_dict.user_id},
+                include={"organization_memberships": True},
             )
-            user_team_ids: list[str] = user_record.teams if user_record is not None and user_record.teams else []
-
+            user: Final = None if user_record is None else LiteLLM_UserTable.model_validate(user_record.model_dump())
             projects = await _project_table(prisma_client).find_many(
-                where={"team_id": {"in": user_team_ids}},
+                where=_visible_projects_where(user.teams if user is not None else [], org_admin_org_ids(user)),
                 include={"litellm_budget_table": True, "object_permission": True},
             )
 

enterprise/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-enterprise"
-version = "0.1.73"
+version = "0.1.74"
 description = "Package for LiteLLM Enterprise features"
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.1.73"
+version = "0.1.74"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-enterprise==",

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.105"
+version = "0.4.106"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -30,7 +30,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.105"
+version = "0.4.106"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm-rust/crates/traces-cache/src/cache.rs

@@ -58,9 +58,8 @@ impl SnapshotKey {
     }
 }
 
-/// How long a read result stays reusable: traces still receiving spans, or read with spend
-/// unavailable, are re-read after `LIVE_TTL`; traces quiet for `SETTLED_AFTER_MS` are kept for
-/// `SETTLED_TTL`.
+/// Native sessions can resume without a terminal record, so their reads retain `LIVE_TTL`.
+/// Other traces with known spend settle after `SETTLED_AFTER_MS` of inactivity.
 #[derive(Clone, Copy, Debug, Eq, PartialEq)]
 pub enum Freshness {
     Live,
@@ -69,6 +68,12 @@ pub enum Freshness {
 
 impl Freshness {
     pub fn of(rows: &[TraceSpansRow], spend_known: bool, snapshot_ms: u64) -> Self {
+        if rows
+            .iter()
+            .any(|row| matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk"))
+        {
+            return Self::Live;
+        }
         let last_end_ms = rows
             .iter()
             .map(|row| row.start_ns.saturating_add_unsigned(row.duration_ns) / 1_000_000)
@@ -393,4 +398,18 @@ mod tests {
             expected
         );
     }
+
+    #[rstest]
+    #[case::claude_code("claude-code")]
+    #[case::claude_agent_sdk("claude-agent-sdk")]
+    fn idle_native_sessions_remain_live(#[case] framework: &str) {
+        let native = TraceSpansRow {
+            framework: framework.into(),
+            ..row("native")
+        };
+        assert_eq!(
+            Freshness::of(&[row("root"), native], true, LAST_END_MS + SETTLED_AFTER_MS),
+            Freshness::Live
+        );
+    }
 }

litellm-rust/crates/traces-cache/tests/read.rs

@@ -706,6 +706,54 @@ async fn failed_reads_are_not_cached() {
     assert_eq!(store.calls(Operation::TraceSpans), 2);
 }
 
+#[rstest]
+#[case::claude_code("claude-code")]
+#[case::claude_agent_sdk("claude-agent-sdk")]
+#[tokio::test]
+async fn resumed_native_sessions_refresh_after_live_ttl(#[case] framework: &str) {
+    let original = TraceSpansRow {
+        framework: framework.into(),
+        ..span(0)
+    };
+    let store = FakeStore::with_spans("ref", vec![original.clone()]);
+    let reader = TraceReader::new(usize::MAX);
+    let access = access();
+    let first = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(first.spans.len(), 1);
+
+    store.state.lock().unwrap().trace_spans.insert(
+        "ref".into(),
+        vec![
+            original,
+            TraceSpansRow {
+                start_ns: now_ns(),
+                ..span(1)
+            },
+        ],
+    );
+    let cached = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(cached.spans.len(), 1);
+    assert_eq!(store.calls(Operation::TraceSpans), 1);
+
+    tokio::time::sleep(LIVE_TTL + Duration::from_millis(200)).await;
+    let resumed = reader
+        .get_trace(&store, &access, "trace", "ref")
+        .await
+        .unwrap()
+        .unwrap();
+    assert_eq!(resumed.spans.len(), 2);
+    assert_eq!(store.calls(Operation::TraceSpans), 2);
+    assert_eq!(first.spans.len(), 1);
+}
+
 #[rstest]
 #[tokio::test]
 async fn listed_runs_are_read_once_until_a_live_run_expires() {

litellm-rust/crates/traces-clickhouse/tests/reads.rs

@@ -600,9 +600,17 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals(
 }
 
 #[rstest]
+#[case::same_key("same-key", Some(0.25))]
+#[case::other_key("other-key", None)]
+#[case::foreign_team("foreign-team", None)]
+#[case::call_id_other_key("call-id-other-key", None)]
+#[case::call_id_foreign_team("call-id-foreign-team", None)]
+#[case::transport_only("transport-only", None)]
 #[tokio::test]
 async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads(
     #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+    #[case] id: &str,
+    #[case] expected: Option<f64>,
 ) -> TestResult {
     let fixture = migrated_database?;
     let client = &fixture.database.client;
@@ -705,20 +713,18 @@ async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_rea
         .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
         .await?;
     assert_eq!(page.data.len(), cases.len());
-    for (id, _, _, _, expected) in cases {
-        let summary = page
-            .data
-            .iter()
-            .find(|summary| summary.trace_id == id)
-            .ok_or("missing run")?;
-        let detail = reader
-            .get_trace(&store, &access, id, &summary.trace_ref)
-            .await?
-            .ok_or("missing trace")?;
-        assert_eq!(detail.summary.spend, expected, "{id}");
-        assert_eq!(summary.spend, expected, "{id}");
-        assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
-    }
+    let summary = page
+        .data
+        .iter()
+        .find(|summary| summary.trace_id == id)
+        .ok_or("missing run")?;
+    let detail = reader
+        .get_trace(&store, &access, id, &summary.trace_ref)
+        .await?
+        .ok_or("missing trace")?;
+    assert_eq!(detail.summary.spend, expected, "{id}");
+    assert_eq!(summary.spend, expected, "{id}");
+    assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
     Ok(())
 }
 

litellm-rust/crates/traces/src/normalize/format/claude_code.rs

@@ -159,10 +159,23 @@ fn exported_tool_results(attributes: &BTreeMap<String, String>) -> String {
     let Ok(body) = serde_json::from_str::<Value>(attr(attributes, "body")) else {
         return json!({"warning": "Claude's API body export is missing or truncated. Some tool results may be unavailable."}).to_string();
     };
-    let results: Vec<Value> = body.get("messages").and_then(Value::as_array)
-        .and_then(|messages| messages.last())
-        .filter(|message| message.get("role").and_then(Value::as_str) == Some("user"))
-        .and_then(|message| message.get("content").and_then(Value::as_array))
+    let message = body
+        .get("messages")
+        .and_then(Value::as_array)
+        .and_then(|messages| {
+            messages
+                .iter()
+                .rev()
+                .find(|message| message.get("role").and_then(Value::as_str) != Some("system"))
+        })
+        .filter(|message| message.get("role").and_then(Value::as_str) == Some("user"));
+    let Some(content) = message.and_then(|message| message.get("content")) else {
+        return json!({"warning": "Claude's API body export has an unexpected message shape. Some tool results may be unavailable."}).to_string();
+    };
+    if !content.is_array() && !content.is_string() {
+        return json!({"warning": "Claude's API body export has an unexpected content shape. Some tool results may be unavailable."}).to_string();
+    }
+    let results: Vec<Value> = content.as_array()
         .into_iter()
         .flatten()
         .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_result"))

litellm-rust/crates/traces/src/otlp/logs.rs

@@ -28,16 +28,31 @@ fn text<'a>(attributes: &'a [KeyValue], key: &str) -> &'a str {
     }
 }
 
-fn message(record: LogRecord) -> Span {
+fn absent_id(id: &[u8], length: usize) -> bool {
+    id.is_empty() || (id.len() == length && id.iter().all(|byte| *byte == 0))
+}
+
+struct LogContext {
+    missing_trace: bool,
+    missing_parent: bool,
+}
+
+fn message(record: LogRecord) -> (Span, LogContext) {
     let timestamp = if record.time_unix_nano == 0 {
         record.observed_time_unix_nano
     } else {
         record.time_unix_nano
     };
+    let missing_trace = absent_id(&record.trace_id, 16);
+    let missing_parent = absent_id(&record.span_id, 8);
     let mut hash = Sha256::new();
     hash.update(b"litellm.claude.message.v1\0");
-    hash.update(&record.trace_id);
-    hash.update(&record.span_id);
+    if !missing_trace {
+        hash.update(&record.trace_id);
+    }
+    if !missing_parent {
+        hash.update(&record.span_id);
+    }
     hash.update(text(&record.attributes, "event.name"));
     let uuid = text(&record.attributes, "message.uuid");
     if uuid.is_empty() {
@@ -50,28 +65,53 @@ fn message(record: LogRecord) -> Span {
     } else {
         hash.update(uuid);
     }
+    if missing_trace {
+        hash.update(b"\0unassigned\0");
+        hash.update(text(&record.attributes, "session.id"));
+    }
+    let identity = hash.finalize();
+    let trace_id = if missing_trace {
+        let session = text(&record.attributes, "session.id");
+        if session.is_empty() {
+            identity[..16].to_vec()
+        } else {
+            span::session_trace_id(session)
+        }
+    } else {
+        record.trace_id
+    };
     let failed = match value(&record.attributes, "success") {
         Some(Value::BoolValue(success)) => !success,
         Some(Value::StringValue(success)) => success == "false",
         _ => false,
     };
-    Span {
-        trace_id: record.trace_id,
-        span_id: hash.finalize()[..8].to_vec(),
-        parent_span_id: record.span_id,
-        name: format!("claude_code.{}", text(&record.attributes, "event.name")),
-        kind: 1,
-        start_time_unix_nano: timestamp,
-        end_time_unix_nano: timestamp,
-        status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(|| {
-            Status {
-                code: 2,
-                message: text(&record.attributes, "error").to_owned(),
-            }
-        }),
-        attributes: record.attributes,
-        ..Span::default()
-    }
+    (
+        Span {
+            trace_id,
+            span_id: identity[..8].to_vec(),
+            parent_span_id: if missing_trace || missing_parent {
+                Vec::new()
+            } else {
+                record.span_id
+            },
+            name: format!("claude_code.{}", text(&record.attributes, "event.name")),
+            kind: 1,
+            start_time_unix_nano: timestamp,
+            end_time_unix_nano: timestamp,
+            status: (text(&record.attributes, "event.name") == "tool_result" && failed).then(
+                || Status {
+                    code: 2,
+                    message: text(&record.attributes, "error").to_owned(),
+                },
+            ),
+            attributes: record.attributes,
+            ..Span::default()
+        },
+        LogContext {
+            missing_trace,
+            missing_parent,
+        },
+    )
 }
 
 pub(super) fn flatten(
@@ -79,6 +119,7 @@ pub(super) fn flatten(
     limits: DecodeLimits,
 ) -> Result<Vec<DecodedSpan>, Error> {
     let mut count = 0usize;
+    let mut contexts = Vec::new();
     let mut resources = Vec::new();
     for resource in request.resource_logs {
         let mut scopes = Vec::new();
@@ -111,7 +152,11 @@ pub(super) fn flatten(
                                     text(&record.attributes, "query_source"),
                                 )))
                 })
-                .map(message)
+                .map(|record| {
+                    let (span, context) = message(record);
+                    contexts.push(context);
+                    span
+                })
                 .collect();
             scopes.push(ScopeSpans {
                 scope: scope.scope,
@@ -125,10 +170,23 @@ pub(super) fn flatten(
             schema_url: resource.schema_url,
         });
     }
-    span::flatten(
+    let (mut spans, mut budget) = span::flatten_with_budget(
         ExportTraceServiceRequest {
             resource_spans: resources,
         },
         limits,
-    )
+    )?;
+    budget.consume(contexts.len() * size_of::<LogContext>())?;
+    for (span, context) in spans.iter_mut().zip(contexts) {
+        if context.missing_trace {
+            span.attributes.remove("lens.original_trace_id");
+        }
+        if context.missing_trace || context.missing_parent {
+            let key = "lens.capture.warning";
+            let warning = "This native log has incomplete trace context. Its content is retained, but its execution parent is unconfirmed.";
+            budget.consume(key.len() + warning.len() + 96)?;
+            span.attributes.insert(key.to_owned(), warning.to_owned());
+        }
+    }
+    Ok(spans)
 }

litellm-rust/crates/traces/src/otlp/span.rs

@@ -20,12 +20,19 @@ pub(super) fn flatten(
     request: ExportTraceServiceRequest,
     limits: DecodeLimits,
 ) -> Result<Vec<DecodedSpan>, Error> {
+    flatten_with_budget(request, limits).map(|(spans, _)| spans)
+}
+
+pub(super) fn flatten_with_budget(
+    request: ExportTraceServiceRequest,
+    limits: DecodeLimits,
+) -> Result<(Vec<DecodedSpan>, Budget), Error> {
     let mut budget = Budget::new(limits);
     let mut spans = Vec::new();
     for resource in request.resource_spans {
         append_resource(resource, &mut budget, &mut spans)?;
     }
-    Ok(spans)
+    Ok((spans, budget))
 }
 
 fn append_resource(
@@ -124,6 +131,10 @@ fn hex_bytes(bytes: &[u8]) -> String {
     bytes.iter().map(|byte| format!("{byte:02x}")).collect()
 }
 
+pub(super) fn session_trace_id(session: &str) -> Vec<u8> {
+    Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16].to_vec()
+}
+
 fn decoded_span(
     span: Span,
     resource_attributes: &Shared<BTreeMap<String, String>>,
@@ -145,8 +156,7 @@ fn decoded_span(
             .get("session.id")
             .filter(|value| !value.is_empty())
     {
-        let trace_id =
-            hex_bytes(&Sha256::digest(format!("litellm.claude.session.v1\0{session}"))[..16]);
+        let trace_id = hex_bytes(&session_trace_id(session));
         let actor = span_attributes.get("agent_id").unwrap_or(session).clone();
         budget.consume(original_trace_id.len() + actor.len() + 256)?;
         span_attributes.insert("lens.original_trace_id".to_owned(), original_trace_id);

litellm-rust/crates/traces/tests/otlp.rs

@@ -1673,7 +1673,7 @@ fn session_capture_joins_native_logs_and_traces_across_turns_without_changing_sp
 
 #[rstest]
 #[case::short_trace(vec![1;15], vec![2;8], 1)]
-#[case::zero_parent(vec![1;16], vec![0;8], 1)]
+#[case::short_parent(vec![1;16], vec![2;7], 1)]
 #[case::timestamp(vec![1;16], vec![2;8], i64::MAX as u64 + 1)]
 fn native_logs_reject_invalid_context(
     #[case] trace: Vec<u8>,
@@ -1692,6 +1692,106 @@ fn native_logs_reject_invalid_context(
     ));
 }
 
+#[rstest]
+#[case::json_empty(true, false, true, true)]
+#[case::protobuf_empty(false, false, true, true)]
+#[case::json_zero(true, true, true, true)]
+#[case::protobuf_zero(false, true, true, true)]
+#[case::trace_only(true, false, true, false)]
+#[case::parent_only(false, false, false, true)]
+fn contextless_native_logs_preserve_the_entire_batch(
+    #[case] json: bool,
+    #[case] zero: bool,
+    #[case] missing_trace: bool,
+    #[case] missing_parent: bool,
+) {
+    use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
+    use prost::Message;
+    let mut request = log_request("repl_main_thread");
+    let valid = request.resource_logs[0].scope_logs[0].log_records[0].clone();
+    let mut uncorrelated = valid.clone();
+    if missing_trace {
+        uncorrelated.trace_id = if zero { vec![0; 16] } else { Vec::new() };
+    }
+    if missing_parent {
+        uncorrelated.span_id = if zero { vec![0; 8] } else { Vec::new() };
+    }
+    let mut standalone = uncorrelated.clone();
+    standalone
+        .attributes
+        .retain(|attribute| attribute.key != "session.id");
+    request.resource_logs[0].scope_logs[0].log_records = vec![valid, uncorrelated, standalone];
+    request.resource_logs[0].resource = Some(opentelemetry_proto::tonic::resource::v1::Resource {
+        attributes: vec![KeyValue {
+            key: "lens.session.capture".into(),
+            value: Some(AnyValue {
+                value: Some(Value::StringValue("true".into())),
+            }),
+            ..Default::default()
+        }],
+        ..Default::default()
+    });
+    let bytes = if json {
+        serde_json::to_vec(&request).unwrap()
+    } else {
+        request.encode_to_vec()
+    };
+    let media = json.then_some("application/json");
+    let limits = litellm_traces::DecodeLimits {
+        attributes: 6,
+        ..Default::default()
+    };
+    let spans = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
+    let replayed = litellm_traces::decode_otlp_logs_with_limits(&bytes, media, limits).unwrap();
+    assert_eq!(spans.len(), 3);
+    assert_eq!(
+        serde_json::to_value(&spans).unwrap(),
+        serde_json::to_value(&replayed).unwrap()
+    );
+    assert_eq!(spans[0].trace_id, spans[1].trace_id);
+    assert_ne!(spans[1].trace_id, spans[2].trace_id);
+    assert_ne!(spans[0].span_id, spans[1].span_id);
+    if missing_trace {
+        assert_ne!(spans[1].span_id, spans[2].span_id);
+        assert!(!spans[1].attributes.contains_key("lens.original_trace_id"));
+    }
+    assert_eq!(spans[0].parent_span_id, "02".repeat(8));
+    assert!(spans[1].parent_span_id.is_empty());
+    assert!(spans[2].parent_span_id.is_empty());
+    assert!(!spans[0].attributes.contains_key("lens.capture.warning"));
+    assert!(spans[1].attributes["lens.capture.warning"].contains("unconfirmed"));
+    assert!(spans[2].attributes["lens.capture.warning"].contains("unconfirmed"));
+    assert_eq!(spans[0].normalized.output, spans[1].normalized.output);
+    assert_eq!(spans[0].normalized.output, spans[2].normalized.output);
+    assert_eq!(spans[1].normalized.observation_type, ObservationType::Chain);
+}
+
+#[rstest]
+#[case::session(true)]
+#[case::standalone(false)]
+fn absent_native_log_context_has_encoding_independent_identity(#[case] session: bool) {
+    use prost::Message;
+    let mut request = log_request("repl_main_thread");
+    let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
+    record.trace_id.clear();
+    record.span_id.clear();
+    if !session {
+        record.attributes.retain(|entry| entry.key != "session.id");
+    }
+    let omitted = litellm_traces::decode_otlp_logs(
+        &serde_json::to_vec(&request).unwrap(),
+        Some("application/json"),
+    )
+    .unwrap();
+    let record = &mut request.resource_logs[0].scope_logs[0].log_records[0];
+    record.trace_id = vec![0; 16];
+    record.span_id = vec![0; 8];
+    let zeroed = litellm_traces::decode_otlp_logs(&request.encode_to_vec(), None).unwrap();
+    assert_eq!(omitted[0].trace_id, zeroed[0].trace_id);
+    assert_eq!(omitted[0].span_id, zeroed[0].span_id);
+    assert_eq!(omitted[0].normalized.output, zeroed[0].normalized.output);
+}
+
 #[rstest]
 #[case::nodes(litellm_traces::DecodeLimits { nodes: 4, ..Default::default() })]
 #[case::depth(litellm_traces::DecodeLimits { depth: 2, ..Default::default() })]
@@ -1778,6 +1878,18 @@ fn interactive_claude_exports_join_replies_with_native_child_execution_context()
 #[rstest]
 #[case::tool_result("tool_result", "", false)]
 #[case::complete_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":[{"type":"text","text":"exit 3 output"},{"type":"image","source":{"data":"PRIVATE_IMAGE"}}]}]}],"system":"PRIVATE_SYSTEM"}"#, false)]
+#[case::headless_body("api_request_body", r#"{"messages":[{"role":"user","content":[{"type":"tool_result","tool_use_id":"call-1","is_error":true,"content":"exit 3 output"}]},{"role":"system","content":"PRIVATE_SYSTEM"}]}"#, false)]
+#[case::missing_messages("api_request_body", r#"{}"#, true)]
+#[case::wrong_content(
+    "api_request_body",
+    r#"{"messages":[{"role":"user","content":{}}]}"#,
+    true
+)]
+#[case::unexpected_last_message(
+    "api_request_body",
+    r#"{"messages":[{"role":"assistant","content":"unexpected"}]}"#,
+    true
+)]
 #[case::truncated_body("api_request_body", "{truncated", true)]
 fn native_tool_logs_supply_arguments_and_results_without_fake_calls(
     #[case] event: &str,
@@ -1849,3 +1961,39 @@ fn interactive_claude_body_export_retains_failed_command_stdout() {
         .unwrap();
     assert_eq!(failed["content"], "Exit code 3\nRAW-EXPECTED");
 }
+
+#[rstest]
+#[case::new_prompt(serde_json::json!({"role":"user", "content":"Continue"}))]
+#[case::new_blocks(serde_json::json!({"role":"user", "content":[{"type":"text", "text":"Continue"}]}))]
+fn native_body_exports_do_not_replay_old_tool_results(#[case] final_message: serde_json::Value) {
+    use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value::Value};
+    let mut request = log_request("repl_main_thread");
+    let body = serde_json::json!({"messages": [
+        {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "old-call", "content": "OLD_RESULT"}]},
+        {"role": "assistant", "content": "Done"}, final_message,
+        {"role": "system", "content": "PRIVATE_SYSTEM"}
+    ]}).to_string();
+    request.resource_logs[0].scope_logs[0].log_records[0].attributes = [
+        ("event.name", "api_request_body"),
+        ("query_source", "repl_main_thread"),
+        ("body", body.as_str()),
+    ]
+    .into_iter()
+    .map(|(key, value)| KeyValue {
+        key: key.into(),
+        value: Some(AnyValue {
+            value: Some(Value::StringValue(value.into())),
+        }),
+        ..Default::default()
+    })
+    .collect();
+    let spans = litellm_traces::decode_otlp_logs(
+        &serde_json::to_vec(&request).unwrap(),
+        Some("application/json"),
+    )
+    .unwrap();
+    assert_eq!(
+        serde_json::from_str::<serde_json::Value>(&spans[0].normalized.output).unwrap(),
+        serde_json::json!({"tool_results":[]})
+    );
+}

litellm/caching/caching_handler.py

@@ -35,7 +35,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
 from litellm.litellm_core_utils.logging_utils import (
     _assemble_complete_response_from_streaming_chunks,
 )
-from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
 from litellm.types.integrations.custom_logger import converted_stream_requested
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import ResponsesAPIResponse
@@ -43,9 +43,11 @@ from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
     CachingDetails,
     CallTypes,
+    Choices,
     Embedding,
     EmbeddingResponse,
     ModelResponse,
+    TextChoices,
     TextCompletionResponse,
     TranscriptionResponse,
     Usage,
@@ -111,6 +113,43 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
     return "choices" in cached_result
 
 
+def _starts_content_block(event: object) -> bool:
+    text: Final = event.decode("utf-8", errors="replace") if isinstance(event, bytes) else event
+    return isinstance(text, str) and "event: content_block_start" in text.splitlines()
+
+
+def is_response_without_output(result: object) -> bool:
+    if isinstance(result, (ModelResponse, TextCompletionResponse)):
+        return not result.choices
+    if isinstance(result, ResponsesAPIResponse):
+        return not getattr(result, "output", None)
+    if not isinstance(result, dict):
+        return False
+    cached_stream_events: Final = result.get(CACHED_STREAM_EVENTS_KEY)
+    if isinstance(cached_stream_events, (list, tuple)):
+        return not any(_starts_content_block(event) for event in cached_stream_events)
+    if "choices" in result:
+        return not result["choices"]
+    if result.get("object") == "response":
+        return not result.get("output")
+    if result.get("type") == "message":
+        return not result.get("content")
+    return False
+
+
+def _choice_carries_output(choice: Choices | TextChoices) -> bool:
+    payload: Final[Mapping[str, object]] = (
+        choice.message.model_dump(exclude_none=True) if isinstance(choice, Choices) else {"text": choice.text}
+    )
+    return any(key != "role" and value not in ("", [], {}) for key, value in payload.items())
+
+
+def _assembled_stream_without_output(response: ModelResponse | TextCompletionResponse) -> bool:
+    """The stream wrapper closes a stream whose chunks carried no choice with an empty choice of its own, so the
+    assembled response has choices and the no-output check on it alone would store that empty answer."""
+    return not any(_choice_carries_output(choice) for choice in response.choices)
+
+
 def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool:
     if kwargs.get("stream", False) is True:
         return True
@@ -398,6 +437,9 @@ class LLMCachingHandler:
             print_verbose("Checking Sync Cache")
             with response_cache_phase("get"):
                 cached_result = litellm.cache.get_cache(**new_kwargs)
+            if is_response_without_output(cached_result):
+                verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
+                return CachingHandlerResponse(cached_result=None)
             if cached_result is not None:
                 if "detail" in cached_result:
                     # implies an error occurred
@@ -836,8 +878,19 @@ class LLMCachingHandler:
                         cache_key=self.preset_cache_key,
                         **request_kwargs,
                     )
+        if is_response_without_output(cached_result):
+            verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
+            self._forget_worker_copy()
+            return None
         return cached_result
 
+    def _forget_worker_copy(self) -> None:
+        """Drop this worker's memory copy of a stored response without output, so the next read reaches Redis,
+        where the refill lands: the text completion and messages writers skip the memory tier."""
+        if self.dual_cache is None or self.preset_cache_key is None:
+            return
+        self.dual_cache.in_memory_cache.delete_cache(self.preset_cache_key)
+
     def _convert_cached_result_to_model_response(
         self,
         cached_result: Any,
@@ -1063,6 +1116,9 @@ class LLMCachingHandler:
 
         if litellm.cache is None:
             return
+        if is_response_without_output(result):
+            verbose_logger.debug("LiteLLM Cache: not caching a response with no output")
+            return
         cache: Final = litellm.cache
 
         new_kwargs: Final = kwargs.copy()
@@ -1113,6 +1169,11 @@ class LLMCachingHandler:
         """
         Sync internal method to add the result to the cache
         """
+        if litellm.cache is None:
+            return
+        if is_response_without_output(result):
+            verbose_logger.debug("LiteLLM Cache: not caching a response with no output")
+            return
 
         new_kwargs: Final = kwargs.copy()
         new_kwargs.update(
@@ -1121,8 +1182,6 @@ class LLMCachingHandler:
                 args,
             )
         )
-        if litellm.cache is None:
-            return
 
         if self._should_store_result_in_cache(original_function=self.original_function, kwargs=new_kwargs):
             with response_cache_phase("set"):
@@ -1201,13 +1260,13 @@ class LLMCachingHandler:
                 is_async=True,
             )
         )
-        # if a complete_streaming_response is assembled, add it to the cache
-        if complete_streaming_response is not None:
-            await self.async_set_cache(
-                result=complete_streaming_response,
-                original_function=self.original_function,
-                kwargs=self.request_kwargs,
-            )
+        if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response):
+            return
+        await self.async_set_cache(
+            result=complete_streaming_response,
+            original_function=self.original_function,
+            kwargs=self.request_kwargs,
+        )
 
     def _sync_add_streaming_response_to_cache(self, processed_chunk: ModelResponse):
         """
@@ -1224,12 +1283,12 @@ class LLMCachingHandler:
             )
         )
 
-        # if a complete_streaming_response is assembled, add it to the cache
-        if complete_streaming_response is not None:
-            self.sync_set_cache(
-                result=complete_streaming_response,
-                kwargs=self.request_kwargs,
-            )
+        if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response):
+            return
+        self.sync_set_cache(
+            result=complete_streaming_response,
+            kwargs=self.request_kwargs,
+        )
 
     def _update_litellm_logging_obj_environment(
         self,

litellm/constants.py

@@ -1807,6 +1807,7 @@ SPEND_LOG_PARTITION_INTERVAL: Final = os.getenv("SPEND_LOG_PARTITION_INTERVAL",
 SPEND_LOG_PARTITION_PRECREATE_AHEAD: Final = int(os.getenv("SPEND_LOG_PARTITION_PRECREATE_AHEAD", 7))
 SPEND_LOG_WRITE_BATCH_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_BYTES", 2_000_000)))
 SPEND_LOG_WRITE_BATCH_MAX_ROWS: Final = max(1, int(os.getenv("SPEND_LOG_WRITE_BATCH_MAX_ROWS", "100")))
+SPEND_ROLLUP_LOCK_TIMEOUT_MS: Final = max(1, int(os.getenv("SPEND_ROLLUP_LOCK_TIMEOUT_MS", "5000")))
 SPEND_LOG_QUEUE_SIZE_THRESHOLD: Final = int(os.getenv("SPEND_LOG_QUEUE_SIZE_THRESHOLD", 100))
 SPEND_LOG_QUEUE_MAX_BYTES: Final = max(1, int(os.getenv("SPEND_LOG_QUEUE_MAX_BYTES", "64000000")))
 SPEND_LOG_QUEUE_POLL_INTERVAL: Final = float(os.getenv("SPEND_LOG_QUEUE_POLL_INTERVAL", 2.0))

litellm/cost_calculator.py

@@ -2568,6 +2568,20 @@ def _batch_rate(
     return fallback if rate is None else rate
 
 
+def _batch_or_half(batch_rate: float | None, standard_rate: float | None, default: float = 0.0) -> float:
+    if batch_rate is not None:
+        return batch_rate
+    if standard_rate is not None:
+        return standard_rate / 2
+    return default
+
+
+def _completion_image_tokens(usage: Usage) -> int:
+    details: Final = usage.completion_tokens_details
+    image_tokens: Final = (details.image_tokens or 0) if details is not None else 0
+    return min(image_tokens, usage.completion_tokens)
+
+
 def batch_cost_calculator(
     usage: Usage,
     model: str,
@@ -2607,13 +2621,14 @@ def batch_cost_calculator(
             "output_cost_per_token",
         )
     ):
-        # model_info was provided (e.g. deployment metadata with only id/db_model)
-        # but carries no pricing fields. Fall back to the global pricing table so
-        # that standard model pricing is used instead of silently returning $0.
+        deployment_info: Final = model_info
         try:
             global_info: Final = litellm.get_model_info(model=model, custom_llm_provider=custom_llm_provider)
             if global_info:
-                model_info = global_info
+                model_info = {
+                    **global_info,
+                    **{key: value for key, value in deployment_info.items() if value is not None},
+                }
         except Exception:
             pass
 
@@ -2647,12 +2662,15 @@ def batch_cost_calculator(
 
         cache_creation_cost: Final = model_info.get("cache_creation_input_token_cost") or input_cost_per_token
         total_prompt_cost += cache_creation_tokens * cache_creation_cost / 2
-    if batch_rates.output is not None:
-        total_completion_cost = usage.completion_tokens * batch_rates.output
-    elif output_cost_per_token:
-        total_completion_cost = (
-            usage.completion_tokens * (output_cost_per_token) / 2
-        )  # batch cost is usually half of the regular token cost
+    text_rate: Final = _batch_or_half(batch_rates.output, output_cost_per_token)
+    image_rate: Final = _batch_or_half(
+        model_info.get("output_cost_per_image_token_batches"),
+        model_info.get("output_cost_per_image_token"),
+        default=text_rate,
+    )
+    image_tokens: Final = _completion_image_tokens(usage)
+    text_tokens: Final = usage.completion_tokens - image_tokens
+    total_completion_cost = text_tokens * text_rate + image_tokens * image_rate
 
     uplift: Final = _get_regional_uplift_multiplier(model_info, data_residency)
     if uplift != 1.0:

litellm/integrations/opentelemetry.py

@@ -32,6 +32,7 @@ from litellm.integrations.otel.model.db_endpoint import db_span_attributes
 from litellm.integrations.otel.model.metadata import flatten_metadata
 from litellm.integrations.otel.model.semconv import LiteLLM, Metric
 from litellm.integrations.otel.plumbing.otlp_tls import resolve_otlp_http_tls
+from litellm.integrations.otel.routing import routing_decision_attributes
 from litellm.litellm_core_utils.core_helpers import is_batch_line_item_event
 from litellm.litellm_core_utils.internal_call_metadata import is_unbilled_non_inference_call_from_params
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
@@ -2413,6 +2414,8 @@ class OpenTelemetry(OTELGenAISemconvMixin, CustomLogger):
             metadata: Final = standard_logging_payload["metadata"]
             for key, value in metadata.items():
                 self.safe_set_attribute(span=span, key=f"metadata.{key}", value=value)
+            decision: Final = metadata.get("routing_decision")
+            span.set_attributes(routing_decision_attributes(decision))
 
             # get hidden params
             hidden_params: Final = getattr(standard_logging_payload, "hidden_params", None) or (

litellm/integrations/otel/logger.py

@@ -54,6 +54,7 @@ from litellm.integrations.otel.model.spans import SpanRole, span_role_for_servic
 from litellm.integrations.otel.model.utils import to_ns
 from litellm.integrations.otel.plumbing.context import (
     active_phase,
+    active_phase_span,
     is_recordable_span,
     mcp_message_transport_span,
     post_response_root,
@@ -778,6 +779,11 @@ class OpenTelemetryV2(CustomLogger):
         if is_recordable_span(span):
             span.add_event(name, attributes)
 
+    def set_phase_attributes(self, attributes: Mapping[str, str | int | float | bool]) -> None:
+        span: Final = active_phase_span()
+        if span is not None and is_recordable_span(span):
+            span.set_attributes(attributes)
+
     async def async_pre_call_hook(
         self,
         user_api_key_dict: "UserAPIKeyAuth",
@@ -1042,6 +1048,12 @@ def phase_event(name: str, attributes: Mapping[str, str | int] | None = None) ->
         logger.add_phase_event(name, attributes)
 
 
+def phase_attributes(attributes: Mapping[str, str | int | float | bool]) -> None:
+    logger: Final = _registered_v2_logger()
+    if logger is not None:
+        logger.set_phase_attributes(attributes)
+
+
 def build_otel_v2_logger(
     config: OpenTelemetryV2Config,
     callback_name: str | None = None,

litellm/integrations/otel/mappers/genai.py

@@ -174,7 +174,7 @@ class GenAIMapper:
                 return {}
 
     def _llm_call(self, data: LLMCallSpanData) -> AttributeMap:
-        attrs: Final = collect(self._LLM_CALL_ATTRS, data)
+        attrs: Final = {**collect(self._LLM_CALL_ATTRS, data), **data.routing_attributes}
         if data.tools:
             attrs[LiteLLM.TOOLS_DECLARED] = len(data.tools)
             attrs.update(

litellm/integrations/otel/model/payloads.py

@@ -30,6 +30,7 @@ from litellm.integrations.otel.model.utils import (
     as_str_mapping,
     as_str_tuple,
 )
+from litellm.integrations.otel.routing import RoutingAttributeValue, routing_decision_attributes
 
 # ``RequestIdentity`` and the request-metadata translation now live in
 # :mod:`metadata`; re-exported here so existing ``model.payloads`` imports keep
@@ -436,6 +437,7 @@ class LLMCallSpanData:
     trace: TraceControls = field(default_factory=TraceControls)
     session_id: str | None = None
     embedding_output: EmbeddingOutput | None = None
+    routing_attributes: Mapping[str, RoutingAttributeValue] = field(default_factory=lambda: MappingProxyType({}))
 
     @classmethod
     def from_standard_logging_payload(
@@ -453,6 +455,8 @@ class LLMCallSpanData:
         # model split, the response model, api base, and identity all come from
         # here rather than being re-derived from the raw payload dicts.
         context: Final = RequestContext.from_standard_logging_payload(payload)
+        metadata: Final = payload.get("metadata")
+        decision: Final = metadata["routing_decision"] if metadata and "routing_decision" in metadata else None
         # Normalize ``response`` to a dict once so the content/id reads below are a
         # plain ``.get`` — no repeated ``isinstance`` guards.
         raw_response: Final = payload.get("response")
@@ -496,6 +500,7 @@ class LLMCallSpanData:
             trace=trace or TraceControls(),
             session_id=session_id or None,
             embedding_output=embedding_output if capture_content else None,
+            routing_attributes=routing_decision_attributes(decision),
         )
 
 

litellm/integrations/otel/routing.py

@@ -0,0 +1,57 @@
+from collections.abc import Mapping
+from types import MappingProxyType
+from typing import Final, TypeAlias
+
+RoutingAttributeValue: TypeAlias = str | int | float | bool
+
+_ROUTING_SCALAR_FIELDS: Final = frozenset(
+    {
+        "router_model_name",
+        "router_type",
+        "router_config_id",
+        "router_config_updated_at",
+        "router_config_fingerprint",
+        "routed_model",
+        "cause",
+        "tier",
+        "tier_label",
+        "request_type",
+        "score",
+        "classifier_model",
+        "classifier_cost",
+        "classifier_failure_reason",
+        "classifier_error_type",
+        "classifier_confidence",
+        "classifier_primary_rule",
+        "classifier_capability_boundary",
+        "classifier_p_solve",
+        "classifier_calibrated_p_solve",
+        "classifier_calibration_version",
+        "classifier_efficient_p_solve",
+        "classifier_capable_p_solve",
+        "classifier_calibrated_efficient_p_solve",
+        "classifier_calibrated_capable_p_solve",
+        "classifier_max_quality_gap",
+        "classifier_prompt_version",
+        "classifier_threshold",
+        "escalated",
+        "context_escalated",
+        "context_escalation_original_tier",
+        "reasoning_override_min_score",
+        "conversation_continuing",
+        "savings_baseline_model",
+        "savings_baseline_deployment_id",
+    }
+)
+
+
+def routing_decision_attributes(decision: Mapping[str, object] | None) -> Mapping[str, RoutingAttributeValue]:
+    if not isinstance(decision, Mapping):
+        return MappingProxyType({})
+    return MappingProxyType(
+        {
+            f"litellm.routing.{key}": value
+            for key in _ROUTING_SCALAR_FIELDS
+            if isinstance(value := decision.get(key), (str, int, float, bool))
+        }
+    )

litellm/integrations/otel/runtime.py

@@ -7,7 +7,9 @@ V2 is not the active logger — so a call site can wrap a request phase or seed
 identity unconditionally.
 """
 
-from collections.abc import Callable, Iterator, Mapping
+from __future__ import annotations
+
+from collections.abc import Callable, Generator, Mapping
 from contextlib import AbstractContextManager, contextmanager
 from functools import cache
 from typing import TYPE_CHECKING, Final, TypeAlias
@@ -16,11 +18,18 @@ if TYPE_CHECKING:
     from opentelemetry.trace import Span
 
 PhaseEventAttributes: TypeAlias = Mapping[str, str | int]
+PhaseAttributes: TypeAlias = Mapping[str, str | int | float | bool]
 
 
 @cache
 def _otel_runtime() -> (
-    "tuple[Callable[[str], AbstractContextManager[Span | None]], Callable[..., None], Callable[[str, PhaseEventAttributes | None], None]] | None"
+    tuple[
+        Callable[[str], AbstractContextManager[Span | None]],
+        Callable[..., None],
+        Callable[[str, PhaseEventAttributes | None], None],
+        Callable[[PhaseAttributes], None],
+    ]
+    | None
 ):
     """Resolve the SDK-backed hooks once and cache the outcome, absence included.
 
@@ -32,11 +41,11 @@ def _otel_runtime() -> (
         from litellm.integrations.otel import logger
     except Exception:
         return None
-    return (logger.phase_span, logger.seed_request_identity, logger.phase_event)
+    return (logger.phase_span, logger.seed_request_identity, logger.phase_event, logger.phase_attributes)
 
 
 @contextmanager
-def phase_span(name: str) -> "Iterator[Span | None]":
+def phase_span(name: str) -> Generator[Span | None]:
     """Run a request phase inside a live active span so its DB/service calls nest.
 
     Yields ``None`` (a plain no-op) when the OTel SDK is unavailable or V2 is not
@@ -64,3 +73,10 @@ def seed_request_identity(user_api_key_dict: object, model: object = None) -> No
     if runtime is None:
         return
     runtime[1](user_api_key_dict, model=model)
+
+
+def phase_attributes(attributes: PhaseAttributes) -> None:
+    runtime: Final = _otel_runtime()
+    if runtime is None:
+        return
+    runtime[3](attributes)

litellm/litellm_core_utils/litellm_logging.py

@@ -728,6 +728,7 @@ class Logging(LiteLLMLoggingBaseClass):
         # enqueue closure here instead of firing it immediately.
         self._defer_async_logging: bool = False
         self._enqueue_deferred_logging: Callable[[], None] | None = None
+        self._async_success_scheduled: bool = False
         self._on_detached_stream_failure: Callable[[Exception], Awaitable[None]] | None = None
         self.shadow_eval_request_snapshot: GuardrailRequestSnapshot | None = None
 
@@ -2270,6 +2271,15 @@ class Logging(LiteLLMLoggingBaseClass):
         except Exception:
             return True
 
+    def claim_async_success_log(self) -> bool:
+        """One async success log per request. The innermost @client wrapper always exits first,
+        since the outer one is awaiting it, so it claims the log here and the outer wrapper gets
+        False and schedules nothing."""
+        if self._async_success_scheduled:
+            return False
+        self._async_success_scheduled = True
+        return True
+
     def mark_logging_complete(
         self,
         event_type: Literal["async_success", "sync_success", "async_failure", "sync_failure"],

litellm/llms/anthropic/pass_through/messages/response_cache.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.caching.caching_handler import create_cache_write_task
+from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
 from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
     AnthropicMessagesStreamingResponse,
     BaseAnthropicMessagesStreamingIterator,
@@ -13,6 +13,7 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
     _is_provider_error_chunk,
     aclose_if_supported,
 )
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY
 
 if TYPE_CHECKING:
     from litellm.caching.caching_handler import LLMCachingHandler
@@ -20,8 +21,6 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import AllMessageValues
     from litellm.types.utils import ModelResponseStream
 
-CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events"
-
 _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({})
 
 _SSE_EVENT_BOUNDARY: Final = re.compile(r"(?<=\n\n)")
@@ -114,6 +113,9 @@ class AnthropicMessagesStreamCacheWriter:
             verbose_logger.exception("Anthropic Messages stream cache write failed: %s", e)
             return
         cached_payload: Final = {CACHED_STREAM_EVENTS_KEY: events}
+        if is_response_without_output(cached_payload):
+            verbose_logger.debug("LiteLLM Cache: not caching a stream with no content blocks")
+            return
         dual_cache: Final = self.caching_handler.dual_cache
 
         async def _write() -> None:

litellm/model_prices_and_context_window_backup.json

@@ -12611,7 +12611,7 @@
     "azure_ai/grok-4.6": {
         "cache_read_input_token_cost": 5e-07,
         "cache_read_input_token_cost_above_200k_tokens": 1e-06,
-        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token": 2e-06,
         "input_cost_per_token_above_200k_tokens": 4e-06,
         "litellm_provider": "azure_ai",
         "max_input_tokens": 200000,
@@ -29288,6 +29288,7 @@
         "mode": "image_generation",
         "output_cost_per_image": 0.134,
         "output_cost_per_image_token": 0.00012,
+        "output_cost_per_image_token_batches": 6e-05,
         "output_cost_per_token": 1.2e-05,
         "rpm": 1000,
         "tpm": 4000000,
@@ -29379,6 +29380,7 @@
         "mode": "image_generation",
         "output_cost_per_image": 0.045,
         "output_cost_per_image_token": 6e-05,
+        "output_cost_per_image_token_batches": 3e-05,
         "output_cost_per_token": 3e-06,
         "output_cost_per_token_batches": 1.5e-06,
         "rpm": 1000,
@@ -80419,8 +80421,7 @@
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/images/generations",
-            "/v1/images/edits",
-            "/v1/batch"
+            "/v1/images/edits"
         ],
         "supported_modalities": [
             "text",
@@ -80533,5 +80534,27 @@
         "supports_vision": true,
         "supports_web_search": true,
         "web_search_billing_unit": "per_query"
+    },
+    "azure_ai/grok-4.7": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "deprecation_date": "2027-10-05",
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/developers/models/grok-4.7",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
     }
 }

litellm/proxy/db/db_spend_update_writer.py

@@ -75,6 +75,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
 )
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 from litellm.proxy.db.model_usage_rollup import build_model_usage_transaction
+from litellm.proxy.db.rollup_lock_timeout import apply_rollup_lock_timeout
 from litellm.proxy.route_llm_request import ROUTE_ENDPOINT_MAPPING
 from litellm.proxy.spend_tracking.compression_savings import (
     extract_compression_saved_tokens,
@@ -292,6 +293,7 @@ async def _spend_update_tx(
 ) -> AsyncGenerator[_SpendTransaction]:
     tx: Final[_SpendTransactionManager] = prisma_client.db.tx(timeout=timedelta(seconds=60))
     async with db_span(call_type, table), tx as transaction:
+        await apply_rollup_lock_timeout(transaction)
         yield transaction
 
 
@@ -2032,11 +2034,17 @@ class DBSpendUpdateWriter:
         start_time: float,
         proxy_logging_obj: ProxyLogging,
     ) -> None:
-        """Retry a failed spend-update transaction on connection errors or deadlocks, else re-raise."""
+        """Retry a failed spend-update transaction on connection errors, deadlocks or a rollup
+        ``lock_timeout`` (55P03), else re-raise. All three roll the transaction back before any
+        increment applied, so re-sending the same batch cannot double-count."""
         from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
         from litellm.proxy.utils import _raise_failed_update_spend_exception
 
-        is_retryable = isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) or PrismaDBExceptionHandler.is_deadlock_error(e)
+        is_retryable = (
+            isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
+            or PrismaDBExceptionHandler.is_deadlock_error(e)
+            or PrismaDBExceptionHandler.is_lock_timeout_error(e)
+        )
         if not is_retryable or attempt >= n_retry_times:
             _raise_failed_update_spend_exception(e=e, start_time=start_time, proxy_logging_obj=proxy_logging_obj)
         verbose_proxy_logger.warning(

litellm/proxy/db/exception_handler.py

@@ -31,6 +31,8 @@ _CONNECTION_CAPACITY_PHRASES: Final = (
     "remaining connection slots are reserved",
 )
 _PRISMA_POOL_TIMEOUT_CODE: Final = "P2024"
+_LOCK_TIMEOUT_SQLSTATE: Final = "55P03"
+_LOCK_TIMEOUT_PHRASE: Final = "canceling statement due to lock timeout"
 
 
 def _exception_chain(e: BaseException) -> Iterator[BaseException]:
@@ -266,6 +268,21 @@ class PrismaDBExceptionHandler:
         error_message: Final = str(e).lower()
         return any(phrase in error_message for phrase in _CONNECTION_CAPACITY_PHRASES)
 
+    @staticmethod
+    def is_lock_timeout_error(e: Exception) -> bool:
+        """True iff Postgres cancelled a statement for exceeding ``lock_timeout``
+        (SQLSTATE 55P03). The statement never acquired its lock, so it never
+        applied and the transaction rolled back: the rows it carried are safe to
+        re-send, and the pooled connection is free again rather than pinned
+        behind the holder."""
+        import prisma
+
+        if not isinstance(e, _exception_types(prisma.errors.PrismaError)):
+            return False
+        if PrismaDBExceptionHandler.postgres_sqlstate(e) == _LOCK_TIMEOUT_SQLSTATE:
+            return True
+        return _LOCK_TIMEOUT_PHRASE in str(e).lower()
+
     @staticmethod
     def postgres_sqlstate(e: Exception) -> str | None:
         """The SQLSTATE Postgres attached to a failed statement, as prisma surfaces it, or None."""

litellm/proxy/db/model_usage_rollup.py

@@ -9,6 +9,7 @@ from itertools import groupby
 from typing import TYPE_CHECKING, Final, Protocol
 
 from pydantic import TypeAdapter, ValidationError
+from typing_extensions import LiteralString
 
 from litellm.constants import (
     INTERNAL_CALL_ORIGIN_METADATA_KEY,
@@ -17,6 +18,7 @@ from litellm.constants import (
 )
 from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES, SpendLogsPayload
 from litellm.proxy.db.model_insights_tasks import load_model_insight_tasks
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting
 
 if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient
@@ -32,6 +34,8 @@ class _UpsertTable(Protocol):
 class _ModelUsageBatch(Protocol):
     litellm_dailymodelusage: _UpsertTable
 
+    def execute_raw(self, query: LiteralString, *args: object) -> None: ...
+
 
 class _ModelUsageBatchManager(Protocol):
     async def __aenter__(self) -> _ModelUsageBatch: ...
@@ -132,6 +136,7 @@ async def flush_model_usage_transactions(
     for attempt in range(n_retry_times + 1):
         try:
             async with _model_usage_batch(prisma_client) as batcher:
+                batcher.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting())
                 for key, grouped in groupby(ordered, key=lambda transaction: transaction.key):
                     entries = tuple(grouped)
                     spend = sum(entry.spend for entry in entries)

litellm/proxy/db/prisma_query_span.py

@@ -39,6 +39,7 @@ _ROOT_FIELD: Final = re.compile(r"result:\s*(\w+)")
 _RAW_SQL: Final = re.compile(r'query:\s*"((?:[^"\\]|\\.)*)"')
 _LEADING_KEYWORD: Final = re.compile(r"(?:\\[nrt]|\s|\()*(\w+)")
 _SETTING: Final = re.compile(r"(?:\\[nrt]|\s)*SET\s+(?:LOCAL\s+|SESSION\s+)?([A-Za-z_.]+)", re.IGNORECASE)
+_SET_CONFIG: Final = re.compile(r"(?:\\[nrt]|\s)*SELECT\s+set_config\(\s*'([A-Za-z_.]+)'", re.IGNORECASE)
 _CATALOG: Final = re.compile(r"\bpg_\w+|\bto_regclass\b|\binformation_schema\b|\bcurrent_setting\s*\(|^\s*SHOW\b")
 _PROBE: Final = re.compile(r"(?:\\[nrt]|\s)*SELECT\s+\d+\s*;?(?:\\[nrt]|\s)*$", re.IGNORECASE)
 _CTE_WRITE: Final = re.compile(r"\b(UPDATE|INSERT|DELETE)\s+(?:INTO\s+|FROM\s+)?(?:\\?\")", re.IGNORECASE)
@@ -90,9 +91,13 @@ def sql_relation(sql: str) -> str | None:
 
 def sql_operation(sql: str) -> tuple[str | None, str | None]:
     """``(verb, target)`` for a raw statement: the SQL verb from its leading keyword and the
-    relation it names, or for ``SET`` the setting it changes."""
+    relation it names, or for ``SET`` (and its parameterizable twin ``SELECT set_config``)
+    the setting it changes."""
     if _PROBE.match(sql):
         return "ping", None
+    set_config: Final = _SET_CONFIG.match(sql)
+    if set_config is not None:
+        return "set", set_config.group(1).lower()
     keyword: Final = _LEADING_KEYWORD.match(sql)
     leading: Final = keyword.group(1).upper() if keyword is not None else ""
     cte_write: Final = _CTE_WRITE.search(sql) if leading == "WITH" else None

litellm/proxy/db/rollup_lock_timeout.py

@@ -0,0 +1,34 @@
+"""The row-lock budget every spend rollup transaction runs under.
+
+A rollup upsert that waits on a row another pod holds keeps its pooled
+connection for as long as it waits. ``lock_timeout`` bounds that wait, so a
+long holder costs the fleet one aborted statement per waiter instead of a
+connection pinned until the chain drains. The aborted statement never applied,
+so the caller requeues its rows (SQLSTATE 55P03, see
+``PrismaDBExceptionHandler.is_lock_timeout_error``).
+
+``set_config(..., true)`` is ``SET LOCAL``: it lasts for the enclosing
+transaction only, and takes the value as a bind parameter, which ``SET``
+cannot. Issued as the first statement of every rollup transaction.
+"""
+
+from typing import Final, Protocol
+
+from typing_extensions import LiteralString
+
+from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
+
+ROLLUP_LOCK_TIMEOUT_SQL: Final = "SELECT set_config('lock_timeout', $1::text, true)"
+
+
+def rollup_lock_timeout_setting() -> str:
+    return f"{SPEND_ROLLUP_LOCK_TIMEOUT_MS}ms"
+
+
+class _RawExecutor(Protocol):
+    async def execute_raw(self, query: LiteralString, *args: object) -> int: ...
+
+
+async def apply_rollup_lock_timeout(transaction: _RawExecutor) -> None:
+    """Bound the row-lock waits of the open ``transaction`` for the rest of its life."""
+    _ = await transaction.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting())

litellm/proxy/db/spend_log_tool_index.py

@@ -15,15 +15,18 @@ from __future__ import annotations
 
 import asyncio
 import random
-from collections.abc import Sequence
+from collections.abc import Mapping, Sequence
 from dataclasses import dataclass
 from datetime import datetime, timezone
 from itertools import groupby
-from typing import TYPE_CHECKING, Any, Final
+from typing import TYPE_CHECKING, Any, Final, Protocol
+
+from typing_extensions import LiteralString
 
 from litellm.constants import SPEND_LOG_WRITE_BATCH_MAX_BYTES, SPEND_LOG_WRITE_BATCH_MAX_ROWS
 from litellm.proxy._types import DB_RETRY_SAFE_ERROR_TYPES
 from litellm.proxy.db.db_span import db_span
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting
 from litellm.proxy.db.spend_log_batching import spend_log_write_batches
 from litellm.repositories.table_repositories import SpendLogToolIndexRepository
 
@@ -99,6 +102,27 @@ def build_tool_usage_transaction(
     )
 
 
+class _ToolRollupTable(Protocol):
+    def upsert(self, *, where: Mapping[str, object], data: Mapping[str, object]) -> None: ...
+
+
+class _ToolRollupBatch(Protocol):
+    litellm_dailytoolspend: _ToolRollupTable
+
+    def execute_raw(self, query: LiteralString, *args: object) -> None: ...
+
+
+class _ToolRollupBatchManager(Protocol):
+    async def __aenter__(self) -> _ToolRollupBatch: ...
+
+    async def __aexit__(self, exc_type: object, exc_value: object, traceback: object) -> bool | None: ...
+
+
+def _tool_rollup_batch(prisma_client: PrismaClient) -> _ToolRollupBatchManager:
+    batch: Final[_ToolRollupBatchManager] = prisma_client.db.batch_()
+    return batch
+
+
 async def flush_tool_usage_transactions(
     prisma_client: PrismaClient,
     transactions: Sequence[ToolUsageTransaction],
@@ -140,8 +164,9 @@ async def flush_tool_usage_transactions(
                     await index_table.create_many(data=statement_rows, skip_duplicates=True)
             async with (
                 db_span("commit_daily_tool_spend", "LiteLLM_DailyToolSpend"),
-                prisma_client.db.batch_() as batcher,
+                _tool_rollup_batch(prisma_client) as batcher,
             ):
+                batcher.execute_raw(ROLLUP_LOCK_TIMEOUT_SQL, rollup_lock_timeout_setting())
                 for (date_key, tool_name), grouped in groupby(per_tool_day, key=lambda entry: (entry[0], entry[1])):
                     entries = tuple(grouped)
                     spend = sum(entry[2] for entry in entries)

litellm/proxy/lens/inference.py

@@ -1,4 +1,5 @@
 import asyncio
+import sys
 from collections.abc import AsyncGenerator, Callable, Coroutine
 from contextlib import asynccontextmanager
 from datetime import datetime, timedelta, timezone
@@ -25,6 +26,11 @@ from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import AllMessageValues
 from litellm.types.utils import CostPerToken, ModelResponse
 
+if sys.version_info >= (3, 11):
+    from asyncio import timeout
+else:
+    from async_timeout import timeout
+
 BUDGET_LEASE: Final = timedelta(minutes=5)
 BUDGET_RENEW_INTERVAL: Final = 30.0
 BUDGET_WAIT_TIMEOUT: Final = 60.0
@@ -317,7 +323,7 @@ async def renew_budget_reservation(
     while True:
         await asyncio.sleep(BUDGET_RENEW_INTERVAL)
         try:
-            async with asyncio.timeout(BUDGET_RENEW_INTERVAL):
+            async with timeout(BUDGET_RENEW_INTERVAL):
                 if (
                     await repo.update(
                         lens_id, lambda e: renew_reservation(e, reservation_id, datetime.now(timezone.utc))
@@ -325,7 +331,7 @@ async def renew_budget_reservation(
                     is None
                 ):
                     raise HTTPException(503, "Could not renew analysis budget reservation")
-        except TimeoutError as error:
+        except (TimeoutError, asyncio.TimeoutError) as error:
             raise HTTPException(503, "Analysis budget reservation renewal timed out") from error
 
 
@@ -363,15 +369,15 @@ async def reserved_budget(
     repo: LensRepository, lens_id: str, reservation_id: str, reserve: Callable[[Lens], Lens], admitted: asyncio.Event
 ) -> AsyncGenerator[None]:
     try:
-        async with asyncio.timeout(float(litellm.request_timeout)):
+        async with timeout(float(litellm.request_timeout)):
             try:
-                async with asyncio.timeout(BUDGET_WAIT_TIMEOUT):
+                async with timeout(BUDGET_WAIT_TIMEOUT):
                     await wait_for_reservation(repo, lens_id, reservation_id, reserve)
-            except TimeoutError as error:
+            except (TimeoutError, asyncio.TimeoutError) as error:
                 raise HTTPException(504, "Analysis request timed out waiting for budget") from error
             admitted.set()
             yield
-    except TimeoutError as error:
+    except (TimeoutError, asyncio.TimeoutError) as error:
         raise HTTPException(504, "Analysis request timed out waiting for budget or model output") from error
 
 

litellm/proxy/management/teams/authz.py

@@ -44,6 +44,12 @@ class TeamAccess:
         return await self.org_roles.is_org_admin(caller.user_id, team.organization_id)
 
 
+def is_team_member(user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable) -> bool:
+    return user_api_key_dict.user_id is not None and any(
+        member.user_id == user_api_key_dict.user_id for member in team_obj.members_with_roles
+    )
+
+
 def is_team_admin(user_api_key_dict: UserAPIKeyAuth, team_obj: LiteLLM_TeamTable) -> bool:
     return any(
         member.user_id is not None and member.user_id == user_api_key_dict.user_id and member.role == "admin"

litellm/proxy/management/users/service.py

@@ -10,13 +10,20 @@ if TYPE_CHECKING:
     from litellm.proxy.utils import PrismaClient, ProxyLogging
 
 
-def holds_org_admin(user: LiteLLM_UserTable | None, organization_id: str) -> bool:
-    return user is not None and any(
-        membership.organization_id == organization_id and membership.user_role == LitellmUserRoles.ORG_ADMIN.value
+def org_admin_org_ids(user: LiteLLM_UserTable | None) -> frozenset[str]:
+    if user is None:
+        return frozenset()
+    return frozenset(
+        membership.organization_id
         for membership in user.organization_memberships or []
+        if membership.user_role == LitellmUserRoles.ORG_ADMIN.value
     )
 
 
+def holds_org_admin(user: LiteLLM_UserTable | None, organization_id: str) -> bool:
+    return organization_id in org_admin_org_ids(user)
+
+
 @dataclass(frozen=True, slots=True)
 class PrismaOrgRoles:
     prisma_client: PrismaClient | None

litellm/proxy/management_endpoints/model_management_endpoints.py

@@ -126,10 +126,11 @@ from litellm.router_strategy.complexity_router.config import resolve_complexity_
 from litellm.router_utils.auto_router_model_naming import (
     GATED_AUTO_ROUTER_CAPABILITIES,
     STRATEGY_ROUTER_PARAM_FIELDS,
+    GatedAutoRouterCapability,
     capability_limit_violation,
     carries_complexity_router_settings,
     count_capability_routers,
-    gated_capability_of,
+    gated_capabilities_of,
     is_complexity_router_model,
     validate_complexity_router_config_placement,
     validate_complexity_router_config_write,
@@ -559,6 +560,20 @@ def _raise_on_tuning_quota_violation(
         raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}")
 
 
+async def _claimed_capability_violation(
+    tables: _TxModelTables,
+    *,
+    capability: GatedAutoRouterCapability,
+    config_rows: Sequence[Mapping[str, object]],
+    model_id: str | None,
+    limit: int | None,
+) -> str | None:
+    rows: Final = await tables.query_raw(_CAPABILITY_DB_ROWS_SQL[capability.key], model_id or "")
+    db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model"))))
+    held: Final = db_held + count_capability_routers(config_rows, capability=capability)
+    return capability_limit_violation(capability=capability, held=held + 1, limit=limit)
+
+
 @asynccontextmanager
 async def _auto_router_capability_slot(
     prisma_client: PrismaClient,
@@ -591,11 +606,11 @@ async def _auto_router_capability_slot(
     )
 
     limit: Final = _license_check.auto_router_capability_limit()
-    capability: Final = gated_capability_of(effective_params)
+    capabilities: Final = gated_capabilities_of(effective_params)
     baselines: Final = heuristic_v1_tuning_baselines
     tuning_candidate: Final = _tuning_candidate(effective_params, model_id=model_id)
     judges_tuning: Final = baselines is not None and is_mutable_tuned_candidate(tuning_candidate, baselines)
-    if member_write is None and (limit is None or (capability is None and not judges_tuning)):
+    if member_write is None and (limit is None or (not capabilities and not judges_tuning)):
         yield _proxy_model_table(prisma_client)
         return
     transaction_client: Final = _ModelTransactionClient.model_validate(prisma_client.db)
@@ -685,14 +700,12 @@ async def _auto_router_capability_slot(
                 prisma_client=pinned_client,
                 llm_router=llm_router,
             )
-        if capability is not None:
-            rows: Sequence[Mapping[str, object]] = await tx_ctx.query_raw(
-                _CAPABILITY_DB_ROWS_SQL[capability.key], model_id or ""
-            )
-            db_held: Final = sum(1 for row in rows if is_complexity_router_model(_decrypted_model(row.get("model"))))
-            held: Final = db_held + count_capability_routers(config_rows, capability=capability)
-            violation: Final = capability_limit_violation(capability=capability, held=held + 1, limit=limit)
-            if violation is not None:
+        for capability in capabilities:
+            if (
+                violation := await _claimed_capability_violation(
+                    tables, capability=capability, config_rows=config_rows, model_id=model_id, limit=limit
+                )
+            ) is not None:
                 raise HTTPException(
                     status_code=status.HTTP_403_FORBIDDEN, detail=f"{violation} {AUTO_ROUTER_LICENSE_REMEDY}"
                 )

litellm/proxy/management_helpers/auto_router_availability.py

@@ -12,7 +12,7 @@ from litellm.router_utils.auto_router_model_naming import (
     capability_limit_violation,
     classify_strategy_router_model,
     count_capability_routers,
-    gated_capability_of,
+    gated_capabilities_of,
 )
 from litellm.router_utils.auto_router_tuning_baseline import (
     is_mutable_tuned_candidate,
@@ -91,8 +91,8 @@ def auto_router_availability(
 ) -> AutoRouterAvailabilityResponse:
     existing_params: Final = None if existing is None else existing.get("litellm_params")
     candidate_params: Final = candidate.get("litellm_params")
-    owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
-    claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
+    owned: Final = gated_capabilities_of(existing_params) if isinstance(existing_params, Mapping) else ()
+    claimed: Final = gated_capabilities_of(candidate_params) if isinstance(candidate_params, Mapping) else ()
     counts: Final = tuple(
         (capability, count_capability_routers(others, capability=capability))
         for capability in GATED_AUTO_ROUTER_CAPABILITIES
@@ -103,15 +103,16 @@ def auto_router_availability(
             key=capability.key,
             limit=limit,
             remaining=None if limit is None else max(0, limit - held),
-            used_by_this_router=owned is capability,
+            used_by_this_router=capability in owned,
         )
         for capability, held in counts
     )
-    capability_error: Final = next(
+    denied_capability: Final = next(
         (
-            capability_limit_violation(capability=capability, held=held + 1, limit=limit)
+            capability
             for capability, held in counts
-            if capability is claimed
+            if capability in claimed
+            and capability_limit_violation(capability=capability, held=held + 1, limit=limit) is not None
         ),
         None,
     )
@@ -140,8 +141,8 @@ def auto_router_availability(
             ),
         ),
         error=(
-            f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
-            if capability_error is not None and claimed is not None
+            f"{capability_labels[denied_capability.key]} has no available allowance. Choose another option or free an existing allowance."
+            if denied_capability is not None
             else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
             if tuning_error is not None
             else None

litellm/proxy/utils.py

@@ -7645,6 +7645,13 @@ async def update_daily_tag_spend(
         verbose_proxy_logger.error("Error updating daily tag spend: %s", e)
 
 
+def _rollup_batch_never_applied(e: Exception) -> bool:
+    """True when a drained rollup batch provably never reached its rows: Postgres had no
+    connection for it (53300) or cancelled it under ``lock_timeout`` before it took the row
+    lock (55P03). Either way re-sending the batch cannot double-count, so it is requeued."""
+    return PrismaDBExceptionHandler.is_database_capacity_error(e) or PrismaDBExceptionHandler.is_lock_timeout_error(e)
+
+
 async def update_spend_logs_job(
     prisma_client: PrismaClient,
     db_writer_client: AsyncHTTPHandler | None,
@@ -7710,9 +7717,11 @@ async def _run_spend_logs_job(
         )
 
     # Tool usage tracking: drain the request-time queue into the tool index and the
-    # LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for was never
-    # sent, so it is requeued and the job stops; any other failure is dropped because
-    # a replay could double-count the rollup.
+    # LiteLLM_DailyToolSpend rollup. A batch Postgres had no connection for, or whose
+    # rollup statement was cancelled by lock_timeout, never applied, so it is requeued;
+    # any other failure is dropped because a replay could double-count the rollup. Only
+    # a full server (53300) stops the job: a timed-out row belongs to this rollup alone,
+    # the writers below have their own rows and keep flushing.
     async with prisma_client._tool_usage_transactions_lock:
         tool_usage_to_process: Final = prisma_client.tool_usage_transactions[:MAX_LOGS_PER_INTERVAL]
         prisma_client.tool_usage_transactions = prisma_client.tool_usage_transactions[len(tool_usage_to_process) :]
@@ -7724,19 +7733,20 @@ async def _run_spend_logs_job(
             transactions=tool_usage_to_process,
         )
     except Exception as tool_tracking_err:
-        if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err):
+        if _rollup_batch_never_applied(tool_tracking_err):
             async with prisma_client._tool_usage_transactions_lock:
                 prisma_client.tool_usage_transactions = [
                     *tool_usage_to_process,
                     *prisma_client.tool_usage_transactions,
                 ]
             verbose_proxy_logger.warning(
-                "Spend tracking - database out of connections during tool usage flush, "
-                "requeued %s tool usage transactions for the next flush: %s",
+                "Spend tracking - database out of connections or rollup row lock timed out during tool usage "
+                "flush, requeued %s tool usage transactions for the next flush: %s",
                 len(tool_usage_to_process),
                 tool_tracking_err,
             )
-            raise
+            if PrismaDBExceptionHandler.is_database_capacity_error(tool_tracking_err):
+                raise
         else:
             verbose_proxy_logger.error(
                 "Spend tracking - tool usage flush failed; %s tool usage transactions dropped: %s",
@@ -7744,6 +7754,7 @@ async def _run_spend_logs_job(
                 tool_tracking_err,
             )
 
+    # Model usage rollup: same requeue and stop rules as the tool rollup above
     async with prisma_client._model_usage_transactions_lock:
         model_usage_to_process: Final = prisma_client.model_usage_transactions
         prisma_client.model_usage_transactions = []
@@ -7752,11 +7763,28 @@ async def _run_spend_logs_job(
 
         await flush_model_usage_transactions(prisma_client=prisma_client, transactions=model_usage_to_process)
     except Exception as model_usage_err:
-        verbose_proxy_logger.error(
-            "Spend tracking - model usage flush failed; %s model usage transactions dropped: %s",
-            len(model_usage_to_process),
-            model_usage_err,
-        )
+        if _rollup_batch_never_applied(model_usage_err):
+            async with (
+                prisma_client._model_usage_transactions_lock  # pyright: ignore[reportPrivateUsage]  # needed to requeue
+            ):
+                prisma_client.model_usage_transactions = [
+                    *model_usage_to_process,
+                    *prisma_client.model_usage_transactions,
+                ]
+            verbose_proxy_logger.warning(
+                "Spend tracking - database out of connections or rollup row lock timed out during model usage "
+                "flush, requeued %s model usage transactions for the next flush: %s",
+                len(model_usage_to_process),
+                model_usage_err,
+            )
+            if PrismaDBExceptionHandler.is_database_capacity_error(model_usage_err):
+                raise
+        else:
+            verbose_proxy_logger.error(
+                "Spend tracking - model usage flush failed; %s model usage transactions dropped: %s",
+                len(model_usage_to_process),
+                model_usage_err,
+            )
 
     await flush_baseline_accounting(prisma_client)
 

litellm/responses/streaming_iterator.py

@@ -691,8 +691,10 @@ class BaseResponsesAPIStreamingIterator:
         if getattr(completed_response, "type", None) != openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
             return
 
+        from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
+
         response_obj: Final = self._get_completed_response_object()
-        if response_obj is None:
+        if response_obj is None or is_response_without_output(response_obj):
             return
 
         caching_handler: Final[LLMCachingHandler | None] = getattr(self.logging_obj, "_llm_caching_handler", None)
@@ -732,8 +734,6 @@ class BaseResponsesAPIStreamingIterator:
         if cached_response is None:
             return
         if is_async:
-            from litellm.caching.caching_handler import create_cache_write_task
-
             cache_write_task: Final = create_cache_write_task(
                 lambda: cache.async_add_cache(
                     cached_response,

litellm/router.py

@@ -73,7 +73,8 @@ from litellm.constants import (
 )
 from litellm.integrations.custom_guardrail import is_guardrail_intervention
 from litellm.integrations.custom_logger import CustomLogger
-from litellm.integrations.otel.runtime import phase_event, phase_span
+from litellm.integrations.otel.routing import routing_decision_attributes
+from litellm.integrations.otel.runtime import phase_attributes, phase_event, phase_span
 from litellm.litellm_core_utils.asyncify import run_async_function
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
@@ -163,9 +164,10 @@ from litellm.router_utils.add_retry_fallback_headers import (
 )
 from litellm.router_utils.auto_router_model_naming import (
     AUTO_ROUTER_MODEL_PREFIX,
+    STRATEGY_ROUTER_PARAM_FIELDS,
     GatedAutoRouterCapability,
     capability_limit_violation,
-    claimed_capability,
+    claimed_capabilities,
     classify_strategy_router_model,
     count_capability_routers,
 )
@@ -8725,6 +8727,19 @@ class Router:
         breadcrumbs: Final = (*kept_breadcrumbs, attempt_record)
         earlier: Final = request_metadata.get("request_retry_count")
         request_retry_count: Final = (earlier if type(earlier) is int and 0 <= earlier else 0) + 1
+        decision: Final = request_metadata.get("routing_decision")
+        router_name: Final = decision.get("router_model_name") if isinstance(decision, Mapping) else None
+        if isinstance(router_name, str):
+            phase_event(
+                "litellm.routing.retry",
+                {
+                    "litellm.routing.router_model_name": router_name,
+                    "litellm.retry.count": request_retry_count,
+                    "error.type": type(e).__name__,
+                    **({"litellm.deployment.model_group": model_group} if isinstance(model_group, str) else {}),
+                    **({"litellm.deployment.id": deployment_id} if isinstance(deployment_id, str) else {}),
+                },
+            )
         kwargs[_metadata_var]["previous_models"] = breadcrumbs  # rebind-ok: the logging object already holds this dict
         kwargs[_metadata_var]["request_retry_count"] = request_retry_count  # rebind-ok: same dict, read by the cap
         return kwargs
@@ -9492,10 +9507,8 @@ class Router:
         )
 
         complexity_router_config: Final[dict | None] = deployment.litellm_params.complexity_router_config
-        capability: Final = claimed_capability(complexity_router_config)
-        if capability is not None:
-            limit_violation: Final = self.auto_router_capability_violation(capability)
-            if limit_violation is not None:
+        for capability in claimed_capabilities(complexity_router_config):
+            if (limit_violation := self.auto_router_capability_violation(capability)) is not None:
                 raise ValueError(limit_violation)
 
         default_model: Final = (
@@ -9575,9 +9588,29 @@ class Router:
             )
         registry[deployment.model_name] = [
             *registry.get(deployment.model_name, []),
-            TaggedPreRoutingStrategy(tags=tags, strategy=strategy),
+            TaggedPreRoutingStrategy(
+                tags=tags,
+                strategy=strategy,
+                definition_fingerprint=self._routing_definition_fingerprint(deployment),
+                deployment=deployment,
+            ),
         ]
 
+    @staticmethod
+    def _routing_definition_fingerprint(deployment: Deployment) -> str | None:
+        """Identify configured routing fields, excluding external files and live strategy state."""
+        try:
+            definition: Final = json.dumps(
+                deployment.litellm_params.model_dump(
+                    include=STRATEGY_ROUTER_PARAM_FIELDS | {"model", "tags"}, exclude_none=True, warnings=False
+                ),
+                sort_keys=True,
+                default=Router._json_default_stable_id,
+            )
+        except (TypeError, ValueError):
+            return None
+        return Router.generate_model_id(deployment.model_name, {"routing_definition": definition})
+
     @staticmethod
     def _unregister_pre_routing_strategy(
         registry: dict[str, list[TaggedPreRoutingStrategy[_PreRoutingStrategyT]]],
@@ -9656,7 +9689,12 @@ class Router:
                 if adaptive_router is not None:
                     self.adaptive_routers[model_name] = [
                         *self.adaptive_routers.get(model_name, []),
-                        TaggedPreRoutingStrategy(tags=tagged.tags, strategy=adaptive_router),
+                        TaggedPreRoutingStrategy(
+                            tags=tagged.tags,
+                            strategy=adaptive_router,
+                            definition_fingerprint=tagged.definition_fingerprint,
+                            deployment=tagged.deployment,
+                        ),
                     ]
 
         self._sync_adaptive_router_hooks()
@@ -14227,6 +14265,7 @@ class Router:
         spend metadata is stamped before routing and the response carries the tier group the
         strategy picked.
         """
+        self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
         requested_registered_model_name: Final = self._get_model_from_alias(model=model) or model
         registered_model_name: Final = await self._resolve_claude_code_session_router(
             model=model,
@@ -14250,7 +14289,6 @@ class Router:
         )
         if selected_strategy is None:
             await arm_compaction(request_kwargs, None)
-            self._record_routing_decision(request_kwargs=request_kwargs, routing_decision=None)
             self._stamp_or_clear_metadata_key(
                 request_kwargs=request_kwargs, key=SESSION_DEPLOYMENT_AFFINITY_TTL_METADATA_KEY, value=None
             )
@@ -14262,6 +14300,26 @@ class Router:
         from litellm.proxy.auth.auto_router_checks import authorize_member_auto_router_inference
         from litellm.router_strategy.complexity_router.complexity_router import ComplexityRouter
 
+        marker: Final = self._selected_strategy_marker_deployment(
+            model=registered_model_name, strategy_tags=selected_strategy.tags, request_kwargs=request_kwargs
+        )
+        definition: Final = selected_strategy.deployment
+        config_id: Final = definition.model_info.id if definition else None
+        updated_at: Final = definition.model_info.updated_at if definition else None
+        router_kind: Final = classify_strategy_router_model(definition.litellm_params.model) if definition else None
+        provenance: Final[StandardLoggingRoutingDecision] = {
+            "router_model_name": definition.model_name if definition else registered_model_name,
+            **({"router_type": router_kind} if router_kind is not None else {}),
+            **({"router_config_id": config_id} if config_id is not None else {}),
+            **({"router_config_updated_at": updated_at.isoformat()} if updated_at is not None else {}),
+            **(
+                {"router_config_fingerprint": selected_strategy.definition_fingerprint}
+                if selected_strategy.definition_fingerprint is not None
+                else {}
+            ),
+        }
+        phase_attributes(routing_decision_attributes(provenance))
+
         reject_recursive_compactor(registered_model_name)
         await arm_compaction(
             request_kwargs,
@@ -14285,11 +14343,7 @@ class Router:
         )
 
         await authorize_member_auto_router_inference(
-            deployment=self._selected_strategy_marker_deployment(
-                model=registered_model_name,
-                strategy_tags=selected_strategy.tags,
-                request_kwargs=request_kwargs,
-            ),
+            deployment=marker,
             request_kwargs=request_kwargs,
             llm_router=self,
         )
@@ -14327,13 +14381,18 @@ class Router:
             input=input,
             specific_deployment=specific_deployment,
         )
+        traced_routed: Final = (
+            routed.model_copy(update={"routing_decision": {**provenance, **routed.routing_decision}})
+            if routed is not None and routed.routing_decision is not None
+            else routed
+        )
         # Routing-only compression must not leak into the response: the model call and
         # deployment-context filtering key off this field. Compared by value, since
         # pydantic rebuilds the list rather than keeping the object passed in.
         pre_routing_hook_response: Final = (
-            routed.model_copy(update={"messages": messages})
-            if routed is not None and routing_messages is not None and routed.messages == routing_messages
-            else routed
+            traced_routed.model_copy(update={"messages": messages})
+            if traced_routed is not None and routing_messages is not None and traced_routed.messages == routing_messages
+            else traced_routed
         )
         self._record_routing_decision(
             request_kwargs=request_kwargs,
@@ -14484,6 +14543,7 @@ class Router:
         """
         from litellm.types.router import BaselineRouteStamp
 
+        phase_attributes(routing_decision_attributes(routing_decision))
         baseline_model: Final = routing_decision.get("savings_baseline_model") if routing_decision else None
         baseline_id: Final = routing_decision.get("savings_baseline_deployment_id") if routing_decision else None
         router_name: Final = routing_decision.get("router_model_name") if routing_decision else None

litellm/router_strategy/auto_router/auto_router.py

@@ -4,12 +4,13 @@ Auto-Routing Strategy that works with a Semantic Router Config
 
 import asyncio
 from collections.abc import Mapping, Sequence
-from typing import TYPE_CHECKING, Any, Final, Optional
+from typing import TYPE_CHECKING, Any, Final, NamedTuple, Optional
 
 from pydantic import ConfigDict
 
 from litellm._logging import verbose_router_logger
 from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS
+from litellm.exceptions import Timeout as LiteLLMTimeout
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.internal_call_metadata import (
     effective_turn_off_message_logging,
@@ -17,7 +18,12 @@ from litellm.litellm_core_utils.internal_call_metadata import (
     parent_session_kwargs,
 )
 from litellm.types.llms.base import LiteLLMBaseModel
-from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN
+from litellm.types.utils import (
+    AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
+    ClassifierFailureReason,
+    RoutingDecisionCause,
+    StandardLoggingRoutingDecision,
+)
 
 if TYPE_CHECKING:
     from semantic_router.routers import SemanticRouter
@@ -41,6 +47,12 @@ class _CallerMetadata(LiteLLMBaseModel):
     litellm_metadata: Mapping[str, object] | None = None
 
 
+class _SemanticMatchOutcome(NamedTuple):
+    route_name: str | None = None
+    failure_reason: ClassifierFailureReason | None = None
+    error_type: str | None = None
+
+
 class AutoRouter(CustomLogger):
     DEFAULT_AUTO_SYNC_VALUE = "local"
 
@@ -72,6 +84,7 @@ class AutoRouter(CustomLogger):
 
         from litellm.router_strategy.auto_router.litellm_encoder import LiteLLMRouterEncoder
 
+        self.model_name: Final = model_name
         self.auto_router_config_path: str | None = auto_router_config_path
         self.auto_router_config: str | None = auto_router_config
         self.auto_sync_value = self.DEFAULT_AUTO_SYNC_VALUE
@@ -205,20 +218,38 @@ class AutoRouter(CustomLogger):
         if resolved_messages is None:
             return None
 
-        routelayer = await self._ensure_routelayer()
+        routelayer: Final = await self._ensure_routelayer()
 
         message_content: Final = self._extract_text_from_messages(resolved_messages)
-        route_name: Final = await self._matched_route_name(routelayer, message_content, request_kwargs)
+        outcome: Final = await self._match_route(routelayer, message_content, request_kwargs)
+        routed_model: Final = outcome.route_name or self.default_model
+        cause: Final[RoutingDecisionCause] = (
+            "semantic_error"
+            if outcome.failure_reason is not None
+            else "semantic_match"
+            if outcome.route_name
+            else "semantic_no_match"
+        )
+        decision: Final[StandardLoggingRoutingDecision] = {
+            "router_model_name": self.model_name,
+            "router_type": "semantic",
+            "routed_model": routed_model,
+            "cause": cause,
+            "classifier_model": self.embedding_model,
+            **({"classifier_failure_reason": outcome.failure_reason} if outcome.failure_reason is not None else {}),
+            **({"classifier_error_type": outcome.error_type} if outcome.error_type is not None else {}),
+        }
 
         return PreRoutingHookResponse(
-            model=route_name or self.default_model,
+            model=routed_model,
             messages=messages,
+            routing_decision=decision,
         )
 
-    async def _matched_route_name(
+    async def _match_route(
         self, routelayer: "SemanticRouter", text: str, request_kwargs: Mapping[str, object]
-    ) -> str | None:
-        """Name of the route `text` matches, or None when nothing matched or the match failed.
+    ) -> _SemanticMatchOutcome:
+        """Matched route or the reason a default model will serve the request.
 
         `text` is embedded here rather than by `routelayer(text=...)` so the caller's metadata reaches
         `aembedding()` and the embedding's spend lands on the key/team that sent the request;
@@ -247,10 +278,16 @@ class AutoRouter(CustomLogger):
             verbose_router_logger.warning(
                 "AutoRouter: semantic routing failed (%s), falling back to default model %s", e, self.default_model
             )
-            return None
+            failure: Final = e.__cause__ if isinstance(e, ValueError) and e.__cause__ is not None else e
+            return _SemanticMatchOutcome(
+                failure_reason="timeout"
+                if isinstance(failure, (TimeoutError, asyncio.TimeoutError, LiteLLMTimeout))
+                else "classifier_error",
+                error_type=type(failure).__name__,
+            )
         verbose_router_logger.debug("route_choice: %s", route_choice)
         if isinstance(route_choice, RouteChoice):
-            return route_choice.name
+            return _SemanticMatchOutcome(route_name=route_choice.name)
         if isinstance(route_choice, list) and route_choice:
-            return route_choice[0].name
-        return None
+            return _SemanticMatchOutcome(route_name=route_choice[0].name)
+        return _SemanticMatchOutcome()

litellm/router_strategy/complexity_router/README.md

@@ -381,7 +381,7 @@ Each dimension contributes its weight once when any matcher hits the current ask
 
 The API and YAML store exactly the weights written. A `dimension_weights` map and inline custom weights are read literally, missing recognized built-in names score zero, and nothing renormalizes the vector, so a total other than 1 is legal and scores accordingly. The dashboard's heuristic scoring editor is the one place that rebalances: editing one weight there holds it and redistributes the remainder across the other active dimensions in the draft, then Save sends the resulting explicit values, which the backend stores and scores as written. Opening a router, applying a preset, editing matchers, changing `scoring_mode`, or saving unrelated fields never normalizes existing weights
 
-Only `heuristic`, `heuristic_first` and `hybrid` accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
+Only `heuristic`, `heuristic_first` and `hybrid` using heuristic v1 accept custom dimensions. Each name must be a unique ASCII identifier starting with a letter, at most 64 characters, and cannot reuse a built-in dimension name or a key in `dimension_weights`. Set its weight inline, greater than zero and at most one
 
 Patterns are checked at configuration time against a grammar whose worst case stays a few milliseconds on 2048 characters. Every quantifier needs an explicit upper bound of at most 64 and must repeat a single character or character class, so `\s{1,4}` is accepted while `\s+`, `(a|aa){0,12}` and `(?:ab){0,64}` are refused. Backreferences, lookarounds, atomic groups and possessive quantifiers are refused as well. Each pattern is then costed: alternation branches and repeat lengths multiply the ways the engine can retry, and every later piece of the pattern is charged once per path that can reach it, so `a?a?a?a?a?a?a?a?` followed by a long fixed tail is refused even though each quantifier is small. The budget is 2048 work units per pattern and 8192 across the router. An invalid or over-budget pattern fails the write with a message naming the pattern and the rule it broke
 
@@ -555,7 +555,7 @@ on by default; set `classifier_llm_config.circuit_breaker_enabled: false` to dis
 fallback is the local heuristic scorer, so a classifier outage does not repeat its timeout across
 every turn or session handled by the router process.
 
-A request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
+With the default heuristic v1, a request short-circuits, meaning it routes on the scorer's own tier with no classifier call, when
 two things hold: the scorer landed at or below `heuristic_first_max_tier`, and it produced at least
 one signal. Everything else goes to the classifier, which then decides as it normally would.
 
@@ -576,6 +576,13 @@ except that the heuristic outcome is the one already computed rather than a seco
 Spend logs record `routing_decision.cause` as `heuristic_first_short_circuit` when the classifier
 was skipped, and `llm_classifier` when it ran, so the two are told apart per request.
 
+Set `local_heuristic: heuristic_v2` to chain the trained predictor instead. Its selected tier must
+meet `heuristic_v2_success_threshold` and remain at or below `heuristic_first_max_tier` to skip the judge
+
+Omitting `local_heuristic` keeps v1. Both choices preserve the configured classifier failure fallback;
+the heuristic fallback uses the selected local scorer. V2 short-circuits and fallbacks also record
+`heuristic_v2_forecast` with the probabilities and success threshold used for the decision
+
 ### Hybrid
 
 `classifier_type: hybrid` also scores locally first, but it asks a different question than
@@ -601,7 +608,7 @@ model_list:
           REASONING: o1-preview
 ```
 
-A request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin`
+With the default heuristic v1, a request routes on the scorer's own tier when its score is further than `hybrid_boundary_margin`
 from every active boundary. Everything else goes to the classifier: a score inside the band, where a
 hair's difference would have named the adjacent tier and its model pool, and a prompt where no
 dimension fired at all, which has no opinion to be confident about. `hybrid_boundary_margin` is
@@ -619,6 +626,10 @@ Operator-defined tier sets (`tier_definitions`) are not supported here, for the
 not supported under heuristic-first: the scorer only produces the built-in tiers. Classifier failure
 behaves exactly as it does under `classifier_type: llm`.
 
+For `local_heuristic: heuristic_v2`, the selected tier must meet `heuristic_v2_success_threshold`.
+The judge decides when any probability at or below the selected tier is within `hybrid_boundary_margin`
+of that threshold, or when no tier meets the threshold. A zero margin still defers exact-threshold predictions
+
 ### Reasoning Override
 
 If 2+ reasoning markers are detected in the user message, the request is promoted to the REASONING tier even when the weighted score maps lower, so complex reasoning tasks get the appropriate model. The promotion requires the score to reach `reasoning_override_min_score`, which tracks `tier_boundaries.simple_medium` unless set, so stock phrases on an otherwise trivial prompt cannot buy the top tier. Set it to `0` to promote on the markers alone.

litellm/router_strategy/complexity_router/complexity_router.py

@@ -42,6 +42,7 @@ from litellm.constants import (
     SESSION_ID_GENERATED_METADATA_KEY,
 )
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.otel.runtime import phase_event
 from litellm.litellm_core_utils.classifier_logging import masked_originating_request
 from litellm.litellm_core_utils.core_helpers import (
     _get_parent_otel_span_from_kwargs,
@@ -77,6 +78,7 @@ from litellm.types.llms.openai import (
 )
 from litellm.types.utils import (
     AUTOROUTER_CLASSIFIER_CALL_ORIGIN,
+    ClassifierFailureReason,
     ModelResponse,
     RoutingDecisionCause,
     StandardLoggingHeuristicV2Forecast,
@@ -112,6 +114,7 @@ from .config import (
     CustomDimension,
     OpenSourceClassifierConfig,
     TierDefinition,
+    configured_local_heuristic,
 )
 from .jev_classifier import (
     DEFAULT_JEV_INSTRUCTIONS,
@@ -1084,10 +1087,21 @@ class ClassificationOutcome(NamedTuple):
     llm_v2_forecast: LLMV2Decision | None = None
     jev_verdict: JevVerdict | None = None
     heuristic_v2_forecast: StandardLoggingHeuristicV2Forecast | None = None
+    classifier_failure_reason: ClassifierFailureReason | None = None
+    classifier_error_type: str | None = None
 
 
-def _with_signal(outcome: ClassificationOutcome, signal: str | None) -> ClassificationOutcome:
-    return outcome if signal is None else outcome._replace(signals=(*outcome.signals, signal))
+def _with_classifier_failure(
+    outcome: ClassificationOutcome,
+    failure_reason: ClassifierFailureReason,
+    error_type: str | None,
+    signal: str | None,
+) -> ClassificationOutcome:
+    return outcome._replace(
+        signals=(*outcome.signals, signal) if signal is not None else outcome.signals,
+        classifier_failure_reason=failure_reason,
+        classifier_error_type=error_type,
+    )
 
 
 def _with_llm_v2_forecast(
@@ -1220,6 +1234,12 @@ def _is_classifier_timeout(exc: BaseException) -> bool:
     return isinstance(exc, LiteLLMTimeout)
 
 
+def _classifier_failure_reason(exc: Exception) -> ClassifierFailureReason:
+    if _is_classifier_timeout(exc):
+        return "timeout"
+    return "invalid_response" if isinstance(exc, ValidationError) else "classifier_error"
+
+
 def _allowed(models: tuple[str, ...], fit_filter: frozenset[str] | None) -> tuple[str, ...]:
     return models if fit_filter is None else tuple(model for model in models if model in fit_filter)
 
@@ -1488,7 +1508,10 @@ class ComplexityRouter(CustomLogger):
                 resolve_tier_artifact(self.config.heuristic_v2_artifact),
                 routing_threshold=self.config.heuristic_v2_success_threshold,
             )
-            if self.config.classifier_type == "heuristic_v2"
+            if configured_local_heuristic(
+                {"classifier_type": self.config.classifier_type, "local_heuristic": self.config.local_heuristic}
+            )
+            == "heuristic_v2"
             else None
         )
 
@@ -1831,6 +1854,8 @@ class ComplexityRouter(CustomLogger):
         escalated: bool = False,
         classifier_model: str | None = None,
         classifier_cost: float | None = None,
+        classifier_failure_reason: ClassifierFailureReason | None = None,
+        classifier_error_type: str | None = None,
         conversation_continuing: bool = True,
         tier_litellm_params: Mapping[str, object] | None = None,
         context_escalation_original_tier: ComplexityTier | str | None = None,
@@ -1849,6 +1874,12 @@ class ComplexityRouter(CustomLogger):
             routed_model=routed_model,
             cause=cause,
             conversation_continuing=conversation_continuing,
+            **(
+                {"classifier_failure_reason": classifier_failure_reason}
+                if classifier_failure_reason is not None
+                else {}
+            ),
+            **({"classifier_error_type": classifier_error_type} if classifier_error_type is not None else {}),
         )
         if (baseline := self.savings_baseline) is not None:
             decision["savings_baseline_model"] = baseline.model
@@ -1940,10 +1971,31 @@ class ComplexityRouter(CustomLogger):
         if self.config.classifier_type == "capability" and self.config.classifier_llm_config is not None:
             return await self._capability_classifier_outcome(prompt, request_kwargs, messages)
         if self.config.classifier_type not in ("llm", "llm_v2") or self.config.classifier_llm_config is None:
-            tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
-            return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
+            return self._classify_locally(prompt, system_prompt)
         return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages)
 
+    def _classify_locally(self, prompt: str, system_prompt: str | None) -> ClassificationOutcome:
+        if self._tier_success_predictor is not None:
+            return self._classify_with_heuristic_v2(prompt)
+        tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
+        return ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
+
+    def _local_outcome_is_decided(self, outcome: ClassificationOutcome, margin: float | None = None) -> bool:
+        forecast: Final = outcome.heuristic_v2_forecast
+        if forecast is None:
+            return bool(outcome.signals) and (
+                margin is None or (outcome.score is not None and not self._is_near_tier_boundary(outcome.score, margin))
+            )
+        threshold: Final = forecast["threshold"]
+        if forecast["probabilities"][forecast["predicted_tier"]] < threshold:
+            return False
+        if margin is None:
+            return True
+        deciding_tiers: Final = TIER_SEVERITY_ORDER[
+            : TIER_SEVERITY_ORDER.index(ComplexityTier(forecast["predicted_tier"])) + 1
+        ]
+        return all(abs(forecast["probabilities"][tier.value] - threshold) > margin for tier in deciding_tiers)
+
     def _classify_with_heuristic_v2(self, prompt: str) -> ClassificationOutcome:
         predictor: Final = self._tier_success_predictor
         if predictor is None:
@@ -1981,27 +2033,27 @@ class ComplexityRouter(CustomLogger):
         """Score locally, and only pay for the classifier call when the scorer did not confidently
         place the request at or below heuristic_first_max_tier.
 
-        Confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0,
+        For v1, confidence is `signals`, not `score`. A prompt where no dimension fired scores exactly 0.0,
         which is below simple_medium and so lands SIMPLE by default rather than by evidence, and a
         threshold check alone would hand that traffic to the cheapest model without ever consulting
         the classifier. Scores also go negative when simple indicators fire, so a score threshold
-        would reject exactly the trivial prompts this path exists to serve.
+        would reject exactly the trivial prompts this path exists to serve. V2 requires its predicted
+        tier to meet the configured success threshold.
 
         A turn carrying images the classifier would see is never decided cheaply: the scorer reads
         text alone, so its confidence describes a request it has only partly seen, and a trivial
         caption beside a screenshot is exactly the misrouting vision classification exists to stop.
         """
-        tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
-        scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
+        scored: Final = self._classify_locally(prompt, system_prompt)
         threshold: Final = self.config.heuristic_first_max_tier
         decided_cheaply: Final = (
             threshold is not None
-            and bool(signals)
+            and self._local_outcome_is_decided(scored)
             and not self._classifier_image_parts(messages)
-            and self._active_tier_severity(tier) <= self._active_tier_severity(threshold)
+            and self._active_tier_severity(scored.tier) <= self._active_tier_severity(threshold)
         )
         if decided_cheaply:
-            return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="heuristic_first_short_circuit")
+            return scored._replace(cause="heuristic_first_short_circuit")
         return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
 
     async def _classify_hybrid(
@@ -2015,21 +2067,20 @@ class ComplexityRouter(CustomLogger):
 
         Where heuristic_first asks how CHEAP the scorer's tier is, this asks how DECIDED it is, so a
         confident score keeps its tier at every tier including the most expensive one. Two things make
-        a score undecided: landing within hybrid_boundary_margin of an active boundary, where a
+        a v1 score undecided: landing within hybrid_boundary_margin of an active boundary, where a
         hair's difference in score would have named the adjacent tier and its model pool, and firing
         no dimension at all, which scores 0.0 and lands SIMPLE by default rather than by evidence.
+        V2 compares its selected and lower-tier probabilities against the success threshold.
         """
-        tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
-        scored: Final = ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause)
+        scored: Final = self._classify_locally(prompt, system_prompt)
         margin: Final = self.config.hybrid_boundary_margin
         decided: Final = (
             margin is not None
-            and bool(signals)
+            and self._local_outcome_is_decided(scored, margin)
             and not self._classifier_image_parts(messages)
-            and not self._is_near_tier_boundary(score, margin)
         )
         if decided:
-            return ClassificationOutcome(tier=tier, score=score, signals=signals, cause="hybrid_short_circuit")
+            return scored._replace(cause="hybrid_short_circuit")
         return await self._llm_classifier_outcome(prompt, system_prompt, request_kwargs, messages, scored=scored)
 
     def _classifier_image_parts(
@@ -2070,7 +2121,9 @@ class ComplexityRouter(CustomLogger):
         permit: Final = breaker.acquire_permit() if breaker is not None else None
         if breaker is not None and permit is None:
             return self._capability_classifier_failure_outcome(
-                "capability classifier circuit is open", signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL
+                "capability classifier circuit is open",
+                failure_reason="circuit_open",
+                signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
             )
         try:
             tier, classifier_cost, forecast = await self._classify_with_capability_llm(prompt, request_kwargs, messages)
@@ -2094,9 +2147,20 @@ class ComplexityRouter(CustomLogger):
         except Exception as e:  # noqa: BLE001 -- every unavailable or invalid judge verdict must fail closed
             if breaker is not None and permit is not None:
                 breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
-            return self._capability_classifier_failure_outcome(f"capability classifier failed ({type(e).__name__})")
+            return self._capability_classifier_failure_outcome(
+                f"capability classifier failed ({type(e).__name__})",
+                failure_reason=_classifier_failure_reason(e),
+                error_type=type(e).__name__,
+            )
 
-    def _capability_classifier_failure_outcome(self, reason: str, signal: str | None = None) -> ClassificationOutcome:
+    def _capability_classifier_failure_outcome(
+        self,
+        reason: str,
+        *,
+        failure_reason: ClassifierFailureReason,
+        error_type: str | None = None,
+        signal: str | None = None,
+    ) -> ClassificationOutcome:
         """Fail closed to the configured capable tier without consulting another taxonomy."""
         capability: Final = self.config.capability_classifier_config
         if capability is None:
@@ -2117,6 +2181,8 @@ class ComplexityRouter(CustomLogger):
             score=None,
             signals=signals,
             cause="capability_classifier_fallback",
+            classifier_failure_reason=failure_reason,
+            classifier_error_type=error_type,
         )
 
     async def _llm_classifier_outcome(
@@ -2140,6 +2206,7 @@ class ComplexityRouter(CustomLogger):
                 prompt,
                 system_prompt,
                 scored,
+                failure_reason="circuit_open",
                 signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
             )
         try:
@@ -2169,7 +2236,12 @@ class ComplexityRouter(CustomLogger):
             if breaker is not None and permit is not None:
                 breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
             return self._classifier_failure_outcome(
-                f"LLM classifier failed ({type(e).__name__})", prompt, system_prompt, scored
+                f"LLM classifier failed ({type(e).__name__})",
+                prompt,
+                system_prompt,
+                scored,
+                failure_reason=_classifier_failure_reason(e),
+                error_type=type(e).__name__,
             )
 
     async def _jev_classifier_outcome(
@@ -2182,11 +2254,16 @@ class ComplexityRouter(CustomLogger):
         config: Final = self.config.opensource_classifier_config
         client: Final = self._jev_client
         if config is None or client is None:
-            return self._classifier_failure_outcome("jev classifier is not configured", prompt, system_prompt)
+            return self._classifier_failure_outcome(
+                "jev classifier is not configured", prompt, system_prompt, failure_reason="not_configured"
+            )
         marker_pairs: Final = self._reminder_markers_for_request(request_kwargs or EMPTY_MAPPING)
         if _encrypted_classifier_task(request_kwargs, marker_pairs) is not None:
             return self._classifier_failure_outcome(
-                "jev classifier does not support encrypted agent tasks", prompt, system_prompt
+                "jev classifier does not support encrypted agent tasks",
+                prompt,
+                system_prompt,
+                failure_reason="unsupported_input",
             )
         breaker: Final = self._classifier_circuit_breaker
         permit: Final = breaker.acquire_permit() if breaker is not None else None
@@ -2195,6 +2272,7 @@ class ComplexityRouter(CustomLogger):
                 "jev classifier circuit is open",
                 prompt,
                 system_prompt,
+                failure_reason="circuit_open",
                 signal=_CLASSIFIER_CIRCUIT_OPEN_SIGNAL,
             )
         criteria: Final[Mapping[str, str]] = (
@@ -2264,7 +2342,11 @@ class ComplexityRouter(CustomLogger):
             if breaker is not None and permit is not None:
                 breaker.record_failure(permit, is_timeout=_is_classifier_timeout(e))
             return self._classifier_failure_outcome(
-                f"jev classifier failed ({type(e).__name__})", prompt, system_prompt
+                f"jev classifier failed ({type(e).__name__})",
+                prompt,
+                system_prompt,
+                failure_reason=_classifier_failure_reason(e),
+                error_type=type(e).__name__,
             )
 
     def _classifier_failure_outcome(
@@ -2273,6 +2355,9 @@ class ComplexityRouter(CustomLogger):
         prompt: str,
         system_prompt: str | None,
         scored: ClassificationOutcome | None = None,
+        *,
+        failure_reason: ClassifierFailureReason,
+        error_type: str | None = None,
         signal: str | None = None,
     ) -> ClassificationOutcome:
         """The outcome when the LLM classifier or classifier plugin produced no usable tier:
@@ -2283,36 +2368,44 @@ class ComplexityRouter(CustomLogger):
         v2: Final = self.config.llm_v2_config
         if v2 is not None:
             verbose_router_logger.warning("ComplexityRouter: %s, routing to llm_v2 capable tier", reason)
-            return _with_signal(
+            return _with_classifier_failure(
                 ClassificationOutcome(
                     tier=ComplexityTier(v2.capable_tier),
                     score=None,
                     signals=("llm-v2:fallback-capable",),
                     cause="llm_v2_fallback",
                 ),
+                failure_reason,
+                error_type,
                 signal,
             )
         fallback_tier: Final = self.config.fallback_tier
         if fallback_tier is not None:
             verbose_router_logger.warning("ComplexityRouter: %s, routing to fallback_tier %s", reason, fallback_tier)
-            return _with_signal(
+            return _with_classifier_failure(
                 ClassificationOutcome(
                     tier=fallback_tier,
                     score=None,
                     signals=(f"classifier-fallback:{fallback_tier}",),
                     cause="classifier_fallback",
                 ),
+                failure_reason,
+                error_type,
                 signal,
             )
         verbose_router_logger.warning(
             "ComplexityRouter: %s, falling back to %s", reason, self.config.classifier_fallback
         )
         if self.config.classifier_fallback == "default_model":
-            return _with_signal(self._default_model_fallback_outcome(), signal)
+            return _with_classifier_failure(self._default_model_fallback_outcome(), failure_reason, error_type, signal)
         if scored is not None:
-            return _with_signal(scored, signal)
-        tier, score, signals, cause = self._score_and_classify(prompt, system_prompt)
-        return _with_signal(ClassificationOutcome(tier=tier, score=score, signals=signals, cause=cause), signal)
+            return _with_classifier_failure(scored, failure_reason, error_type, signal)
+        return _with_classifier_failure(
+            self._classify_locally(prompt, system_prompt),
+            failure_reason,
+            error_type,
+            signal,
+        )
 
     async def _classify_with_plugin(
         self,
@@ -2326,7 +2419,9 @@ class ComplexityRouter(CustomLogger):
 
         plugin: Final = self.config.classifier_plugin
         if plugin is None:
-            return self._classifier_failure_outcome("classifier_plugin is not set", prompt, system_prompt)
+            return self._classifier_failure_outcome(
+                "classifier_plugin is not set", prompt, system_prompt, failure_reason="not_configured"
+            )
         kwargs: Final = request_kwargs if request_kwargs is not None else EMPTY_MAPPING
         pools: Final = self._tier_pools()
         try:
@@ -2342,29 +2437,48 @@ class ComplexityRouter(CustomLogger):
             verdict: Final = await asyncio.wait_for(
                 plugin.classify(context), timeout=self.config.classifier_plugin_timeout_ms / 1000
             )
-        except asyncio.TimeoutError:
+        except asyncio.TimeoutError as e:
             return self._classifier_failure_outcome(
-                f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms", prompt, system_prompt
+                f"classifier plugin timed out after {self.config.classifier_plugin_timeout_ms}ms",
+                prompt,
+                system_prompt,
+                failure_reason="timeout",
+                error_type=type(e).__name__,
             )
         except Exception as e:  # noqa: BLE001 -- an operator hook can fail in arbitrary ways (network, bug); any failure must fall back rather than fail the request
-            return self._classifier_failure_outcome(f"classifier plugin failed ({e})", prompt, system_prompt)
+            return self._classifier_failure_outcome(
+                f"classifier plugin failed ({e})",
+                prompt,
+                system_prompt,
+                failure_reason=_classifier_failure_reason(e),
+                error_type=type(e).__name__,
+            )
         if verdict is None:
-            return self._classifier_failure_outcome("classifier plugin declined to classify", prompt, system_prompt)
+            return self._classifier_failure_outcome(
+                "classifier plugin declined to classify", prompt, system_prompt, failure_reason="declined"
+            )
         if not isinstance(verdict, str):
             return self._classifier_failure_outcome(
                 f"classifier plugin returned a non-string verdict of type {type(verdict).__name__}",
                 prompt,
                 system_prompt,
+                failure_reason="invalid_response",
             )
         tier: Final = self.config.resolve_classified_tier(verdict)
         if tier is None:
             return self._classifier_failure_outcome(
-                f"classifier plugin returned unknown tier {verdict!r}", prompt, system_prompt
+                f"classifier plugin returned unknown tier {verdict!r}",
+                prompt,
+                system_prompt,
+                failure_reason="invalid_response",
             )
         tier_key: Final = _tier_name(tier)
         if not pools.get(tier_key):
             return self._classifier_failure_outcome(
-                f"classifier plugin returned tier {tier_key!r}, which has no models configured", prompt, system_prompt
+                f"classifier plugin returned tier {tier_key!r}, which has no models configured",
+                prompt,
+                system_prompt,
+                failure_reason="invalid_response",
             )
         return ClassificationOutcome(
             tier=tier,
@@ -2628,9 +2742,13 @@ class ComplexityRouter(CustomLogger):
             verdict: Final = LLMV2Verdict.model_validate_json(extract_classifier_json(content))
         except ValidationError as error:
             _log_rejected_classifier_verdict(error, content, request_kwargs)
-            return self._classifier_failure_outcome("Invalid LLM V2 forecast", prompt, system_prompt)._replace(
-                classifier_cost=classifier_cost
-            )
+            return self._classifier_failure_outcome(
+                "Invalid LLM V2 forecast",
+                prompt,
+                system_prompt,
+                failure_reason="invalid_response",
+                error_type=type(error).__name__,
+            )._replace(classifier_cost=classifier_cost)
         decision: Final = v2.classify(verdict)
         return ClassificationOutcome(
             tier=ComplexityTier(v2.efficient_tier if decision.use_efficient else v2.capable_tier),
@@ -4071,6 +4189,10 @@ class ComplexityRouter(CustomLogger):
         try:
             semantic_tier: Final = await self._semantic_tier_override(user_message, request_kwargs)
         except Exception as e:  # noqa: BLE001 -- embedding call can fail many ways (timeout, provider/network/parse error); any failure must fall back to scoring, never fail the request
+            phase_event(
+                "litellm.router.semantic_keyword_fallback",
+                {"litellm.router.failure_reason": _classifier_failure_reason(e), "error.type": type(e).__name__},
+            )
             verbose_router_logger.warning(
                 "ComplexityRouter: semantic keyword matching failed (%s), falling back to complexity scoring", e
             )
@@ -4705,6 +4827,8 @@ class ComplexityRouter(CustomLogger):
                     conversation_continuing=conversation_continuing,
                     cause=outcome.cause,
                     signals=outcome.signals,
+                    classifier_failure_reason=outcome.classifier_failure_reason,
+                    classifier_error_type=outcome.classifier_error_type,
                     escalation_keyword=escalation_keyword,
                     escalated=False,
                     tier_litellm_params=fallback_tier_params,
@@ -4817,6 +4941,8 @@ class ComplexityRouter(CustomLogger):
             escalated=escalated,
             classifier_model=classifier_model,
             classifier_cost=outcome.classifier_cost,
+            classifier_failure_reason=outcome.classifier_failure_reason,
+            classifier_error_type=outcome.classifier_error_type,
             tier_litellm_params=tier_litellm_params,
             context_escalation_original_tier=context_original_tier,
         )

litellm/router_strategy/complexity_router/config.py

@@ -74,6 +74,15 @@ DEFAULT_CLASSIFICATION_RUBRIC: Final[ClassificationRubric] = ClassificationRubri
 LLM_CLASSIFIER_TYPES: Final[frozenset[str]] = frozenset({"llm", "capability", "llm_v2", "heuristic_first", "hybrid"})
 
 
+def configured_local_heuristic(config: Mapping[str, object]) -> Literal["heuristic", "heuristic_v2"] | None:
+    classifier_type: Final = config.get("classifier_type", "heuristic")
+    if classifier_type == "heuristic" or classifier_type == "heuristic_v2":
+        return classifier_type
+    if classifier_type in ("heuristic_first", "hybrid"):
+        return "heuristic_v2" if config.get("local_heuristic") == "heuristic_v2" else "heuristic"
+    return None
+
+
 TIER_SEVERITY_ORDER: Final[tuple[ComplexityTier, ...]] = (
     ComplexityTier.SIMPLE,
     ComplexityTier.MEDIUM,
@@ -1160,10 +1169,17 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
         default=None,
         description="Experimental joint task-demand and solver-capability forecasting for classifier_type llm_v2.",
     )
+    local_heuristic: Literal["heuristic", "heuristic_v2"] | None = Field(
+        default=None,
+        description=(
+            "Local scorer for heuristic_first or hybrid. Omitted or null keeps heuristic v1; "
+            "heuristic_v2 uses the trained success predictor. Rejected for other classifier types."
+        ),
+    )
     heuristic_v2_artifact: TrainedTierArtifact | Literal["ultrafeedback"] = Field(
         default="ultrafeedback",
         description=(
-            "Success-probability artifact used by classifier_type 'heuristic_v2'. The bundled "
+            "Success-probability artifact used by standalone or chained heuristic_v2. The bundled "
             "UltraFeedback artifact is selected by default; an inline trained artifact may replace it"
         ),
     )
@@ -1173,10 +1189,11 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
         ge=0.0,
         le=1.0,
         description=(
-            "Minimum predicted success probability for classifier_type 'heuristic_v2' to select a tier. "
-            "The first tier meeting this threshold is selected, or REASONING if none meets it. "
+            "Minimum predicted success probability for standalone or chained heuristic_v2 to select a tier. "
+            "The first tier meeting this threshold is selected. When none meets it, standalone heuristic_v2 "
+            "selects REASONING and chained heuristic_v2 defers to the LLM judge. "
             "When omitted or null, uses the artifact's routing_threshold (0.75 for the bundled artifact). "
-            "Other classifier types ignore this setting"
+            "Ignored when heuristic_v2 is not selected"
         ),
     )
     classifier_llm_config: ClassifierLLMConfig | None = Field(
@@ -1203,8 +1220,8 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
             "one skips the LLM classifier and routes straight to that heuristic tier, so the classifier "
             "call is only paid for on traffic the scorer could not place cheaply. The scorer must also "
             "have produced at least one signal: a prompt where no dimension fired scores 0.0 and would "
-            "otherwise land SIMPLE by default rather than by evidence, which is how a chained router "
-            "would silently send unclassified traffic to the cheapest model. Names a built-in tier, and "
+            "otherwise land SIMPLE by default rather than by evidence. With local_heuristic 'heuristic_v2', "
+            "the predicted tier must meet its success threshold. Names a built-in tier, and "
             "may not name the highest one, since that would make the LLM classifier unreachable."
         ),
     )
@@ -1218,7 +1235,9 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
             "this from every active boundary routes on the scorer's own tier with no classifier call, at any "
             "tier, which is what separates 'hybrid' from 'heuristic_first' and its cheap-tier ceiling. A "
             "prompt where no dimension fired still goes to the classifier, since the scorer has no opinion "
-            "to be near a boundary with. 0 escalates only scores sitting exactly on a boundary."
+            "to be near a boundary with. With local_heuristic 'heuristic_v2', a tier must meet its success "
+            "threshold and its probability and all lower-tier probabilities must be further than this margin "
+            "from that threshold. 0 escalates only scores or probabilities exactly on a boundary."
         ),
     )
     classifier_plugin: ClassifierPlugin | None = Field(
@@ -1855,12 +1874,25 @@ class ComplexityRouterConfig(LiteLLMBaseModel):
             raise ValueError("llm_v2 requires one distinct model group in each tier")
         return self
 
+    @model_validator(mode="after")
+    def _validate_local_heuristic(self) -> "ComplexityRouterConfig":
+        if self.local_heuristic is not None and self.classifier_type not in ("heuristic_first", "hybrid"):
+            raise ValueError("local_heuristic requires classifier_type heuristic_first or hybrid")
+        return self
+
     @model_validator(mode="after")
     def _validate_custom_dimensions(self) -> "ComplexityRouterConfig":
         if not self.custom_dimensions:
             return self
-        if self.classifier_type not in ("heuristic", "heuristic_first", "hybrid"):
-            raise ValueError("custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid")
+        if (
+            configured_local_heuristic(
+                {"classifier_type": self.classifier_type, "local_heuristic": self.local_heuristic}
+            )
+            != "heuristic"
+        ):
+            raise ValueError(
+                "custom_dimensions requires classifier_type heuristic, heuristic_first or hybrid using heuristic v1"
+            )
         names: Final = tuple(dimension.name.casefold() for dimension in self.custom_dimensions)
         reserved: Final = frozenset(name.casefold() for name in DEFAULT_DIMENSION_WEIGHTS)
         weighted: Final = frozenset(name.casefold() for name in self.dimension_weights)

litellm/router_utils/auto_router_model_naming.py

@@ -19,6 +19,7 @@ from litellm.router_strategy.complexity_router.config import (
     COMPLEXITY_ROUTER_CONFIG_KEYS,
     DEFAULT_JEV_INSTRUCTIONS,
     LLM_CLASSIFIER_TYPES,
+    configured_local_heuristic,
     normalize_classifier_config_aliases,
 )
 
@@ -186,7 +187,7 @@ def strategy_router_dependencies(
 
 def uses_heuristic_v2_classifier(complexity_router_config: object) -> bool:
     """Whether this complexity config classifies with the bundled heuristic_v2 model."""
-    return _mapping(complexity_router_config).get("classifier_type") == "heuristic_v2"
+    return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic_v2"
 
 
 def defines_custom_tiers(complexity_router_config: object) -> bool:
@@ -237,8 +238,8 @@ class GatedAutoRouterCapability:
     stored ``litellm_params`` (``{config}`` is the caller's expression for the normalized
     ``complexity_router_config`` jsonb, substituted as many times as the predicate needs); they live
     on one record so they cannot drift apart. ``subject`` and ``remedy`` build the shared refusal
-    message. A validated config claims at most one capability: gated classifier types cannot be
-    combined with operator-defined tiers or classifier prompts.
+    message. A router chaining heuristic_v2 with an operator-written judge prompt claims both
+    capabilities.
     """
 
     key: str
@@ -250,10 +251,14 @@ class GatedAutoRouterCapability:
 
 HEURISTIC_V2_CAPABILITY: Final = GatedAutoRouterCapability(
     key="heuristic_v2",
-    subject="with classifier_type 'heuristic_v2'",
-    remedy="Use classifier_type 'heuristic' for this router or remove an existing heuristic_v2 router.",
+    subject="using the heuristic_v2 classifier",
+    remedy="Choose the heuristic v1 classifier for this router or remove an existing heuristic_v2 router.",
     uses=uses_heuristic_v2_classifier,
-    sql_config_predicate="{config} ->> 'classifier_type' = 'heuristic_v2'",
+    sql_config_predicate=(
+        "{config} ->> 'classifier_type' = 'heuristic_v2' OR "
+        "({config} ->> 'classifier_type' IN ('heuristic_first', 'hybrid') AND "
+        "{config} ->> 'local_heuristic' = 'heuristic_v2')"
+    ),
 )
 
 CAPABILITY_CLASSIFIER_CAPABILITY: Final = GatedAutoRouterCapability(
@@ -307,20 +312,19 @@ GATED_AUTO_ROUTER_CAPABILITIES: Final = (
 )
 
 
-def claimed_capability(complexity_router_config: object) -> GatedAutoRouterCapability | None:
-    """The licensed capability this complexity config claims, or None."""
-    return next(
-        (capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config)),
-        None,
+def claimed_capabilities(complexity_router_config: object) -> tuple[GatedAutoRouterCapability, ...]:
+    """The licensed capabilities this complexity config claims."""
+    return tuple(
+        capability for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(complexity_router_config)
     )
 
 
-def gated_capability_of(litellm_params: Mapping[str, object]) -> GatedAutoRouterCapability | None:
-    """The licensed capability this deployment claims, or None unless it is a complexity router."""
+def gated_capabilities_of(litellm_params: Mapping[str, object]) -> tuple[GatedAutoRouterCapability, ...]:
+    """The licensed capabilities this deployment claims, or empty unless it is a complexity router."""
     model: Final = litellm_params.get("model")
     if not is_complexity_router_model(model if isinstance(model, str) else None):
-        return None
-    return claimed_capability(litellm_params.get("complexity_router_config"))
+        return ()
+    return claimed_capabilities(litellm_params.get("complexity_router_config"))
 
 
 def count_capability_routers(
@@ -328,7 +332,9 @@ def count_capability_routers(
 ) -> int:
     """How many of ``deployments`` (router model_list entries or config.yaml rows) claim ``capability``."""
     return sum(
-        1 for deployment in deployments if gated_capability_of(_mapping(deployment.get("litellm_params"))) is capability
+        1
+        for deployment in deployments
+        if capability in gated_capabilities_of(_mapping(deployment.get("litellm_params")))
     )
 
 

litellm/router_utils/auto_router_tuning_baseline.py

@@ -8,7 +8,7 @@ from typing import Final
 
 from pydantic import ValidationError
 
-from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
+from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig, configured_local_heuristic
 
 # v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
 TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
@@ -30,7 +30,6 @@ HEURISTIC_V1_TUNING_FIELDS: Final = (
 
 _TUNING_FIELD_SET: Final = frozenset(HEURISTIC_V1_TUNING_FIELDS)
 
-_V1_SCORING_CLASSIFIER_TYPES: Final = frozenset({"heuristic", "heuristic_first", "hybrid"})
 _AUTO_ROUTER_COMPLEXITY_PREFIX: Final = "auto_router/complexity_router"
 _EMPTY: Final[Mapping[str, object]] = MappingProxyType({})
 _EMPTY_TAGS: Final[tuple[str, ...]] = ()
@@ -71,7 +70,7 @@ DEFAULT_TUNING_FINGERPRINT: Final = tuning_fingerprint(_EMPTY)
 
 def uses_heuristic_v1(complexity_router_config: object) -> bool:
     """Whether a config's primary classifier path is the heuristic-v1 scorer."""
-    return _mapping(complexity_router_config).get("classifier_type", "heuristic") in _V1_SCORING_CLASSIFIER_TYPES
+    return configured_local_heuristic(_mapping(complexity_router_config)) == "heuristic"
 
 
 def router_identity(deployment: Mapping[str, object]) -> str | None:

litellm/tracing/types.py

@@ -1,6 +1,6 @@
 from collections.abc import Sequence
 
-from typing_extensions import ReadOnly, TypedDict
+from typing_extensions import NotRequired, ReadOnly, TypedDict
 
 
 class SpendLogRecord(TypedDict):
@@ -8,7 +8,7 @@ class SpendLogRecord(TypedDict):
 
     request_id: ReadOnly[str]
     response_id: ReadOnly[str]
-    provider_request_id: ReadOnly[str]
+    provider_request_id: NotRequired[ReadOnly[str]]
     litellm_call_id: ReadOnly[str]
     call_type: ReadOnly[str]
     api_key: ReadOnly[str]

litellm/types/caching.py

@@ -140,6 +140,8 @@ class HealthCheckCacheParams(LiteLLMBaseModel):
 
 EMBEDDING_CACHE_FORMAT_VERSION: Final = 2
 
+CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events"
+
 
 class CachedEmbedding(TypedDict):
     """Type definition for cached embedding objects"""

litellm/types/router.py

@@ -1214,6 +1214,8 @@ class TaggedPreRoutingStrategy(Generic[_PreRoutingStrategyT_co]):
 
     tags: tuple[str, ...]
     strategy: _PreRoutingStrategyT_co
+    definition_fingerprint: str | None = None
+    deployment: Deployment | None = None
 
 
 @dataclass(frozen=True, slots=True)

litellm/types/utils.py

@@ -376,6 +376,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
     output_cost_per_image: float | None
     output_cost_per_pixel: ReadOnly[float | None]
     output_cost_per_image_token: float | None
+    output_cost_per_image_token_batches: ReadOnly[float | None]
     output_cost_per_video_token: float | None  # for gemini omni models with video output
     output_vector_size: int | None
     output_cost_per_reasoning_token: float | None
@@ -3060,6 +3061,14 @@ RoutingDecisionCause = Literal[
     "keyword",
     "quality_tier",
     "bandit",
+    "semantic_match",
+    "semantic_no_match",
+    "semantic_error",
+]
+
+
+ClassifierFailureReason = Literal[
+    "timeout", "circuit_open", "not_configured", "unsupported_input", "invalid_response", "declined", "classifier_error"
 ]
 
 
@@ -3092,7 +3101,10 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     """Per-request provenance for a pre-routing strategy (auto-router) decision."""
 
     router_model_name: str
-    router_type: Literal["complexity", "adaptive", "quality"]
+    router_type: ReadOnly[Literal["complexity", "adaptive", "quality", "semantic"]]
+    router_config_id: ReadOnly[str]
+    router_config_updated_at: ReadOnly[str]
+    router_config_fingerprint: ReadOnly[str]
     routed_model: str
     cause: RoutingDecisionCause
     tier: str
@@ -3104,6 +3116,8 @@ class StandardLoggingRoutingDecision(TypedDict, total=False):
     escalation_keyword: str
     classifier_model: str
     classifier_cost: float
+    classifier_failure_reason: ReadOnly[ClassifierFailureReason]
+    classifier_error_type: ReadOnly[str]
     classifier_probabilities: ReadOnly[Mapping[str, float]]
     classifier_confidence: ReadOnly[float]
     heuristic_v2_forecast: ReadOnly[StandardLoggingHeuristicV2Forecast]
@@ -3142,6 +3156,9 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
     {
         "router_model_name",
         "router_type",
+        "router_config_id",
+        "router_config_updated_at",
+        "router_config_fingerprint",
         "routed_model",
         "cause",
         "tier",
@@ -3150,6 +3167,8 @@ DERIVED_ROUTING_DECISION_FIELDS: Final[frozenset[str]] = frozenset(
         "score",
         "classifier_model",
         "classifier_cost",
+        "classifier_failure_reason",
+        "classifier_error_type",
         "classifier_probabilities",
         "classifier_confidence",
         "heuristic_v2_forecast",
@@ -3823,6 +3842,7 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
     output_cost_per_character_above_128k_tokens: float | None = None
     output_cost_per_image: float | None = None
     output_cost_per_image_token: float | None = None
+    output_cost_per_image_token_batches: float | None = None
     output_cost_per_video_token: float | None = None
     output_cost_per_reasoning_token: float | None = None
     output_cost_per_reasoning_token_flex: float | None = None

litellm/utils.py

@@ -1373,11 +1373,14 @@ def _schedule_async_success_logging(
 
     Nested @client wrappers (Anthropic Messages over the chat adapter, chat over the Responses
     bridge) each exit through here with the same logging object and their own shape of the same
-    response. The immediate path already logs one request once, since the first task marks
-    ``has_logged_async_success`` and the later ones skip. The deferred slot keeps the same
-    first-wins rule: the innermost wrapper's provider-shaped result is the one the spend log
-    reads usage from, and a later wrapper never swaps in its client-shaped translation.
+    response. The innermost wrapper always exits first, since the outer one is awaiting it, so it
+    claims the async success log here, synchronously, and the outer wrapper returns without
+    enqueuing: one request queues one handler, whatever the handler tasks do later. The deferred
+    slot keeps the same innermost-wins rule: the provider-shaped result is the one the spend log
+    reads usage from, and the outer wrapper never swaps in its client-shaped translation.
     """
+    if not logging_obj.claim_async_success_log():
+        return
 
     def _enqueue_async_logging() -> None:
         asyncio.create_task(
@@ -6268,6 +6271,7 @@ def _get_model_info_helper(
                 output_cost_per_image=_model_info.get("output_cost_per_image", None),
                 output_cost_per_pixel=_model_info.get("output_cost_per_pixel", None),
                 output_cost_per_image_token=_model_info.get("output_cost_per_image_token", None),
+                output_cost_per_image_token_batches=_model_info.get("output_cost_per_image_token_batches", None),
                 output_cost_per_video_token=_model_info.get("output_cost_per_video_token", None),
                 output_vector_size=_model_info.get("output_vector_size", None),
                 citation_cost_per_token=_model_info.get("citation_cost_per_token", None),

model_prices_and_context_window.json

@@ -12611,7 +12611,7 @@
     "azure_ai/grok-4.6": {
         "cache_read_input_token_cost": 5e-07,
         "cache_read_input_token_cost_above_200k_tokens": 1e-06,
-        "input_cost_per_token": 1.25e-06,
+        "input_cost_per_token": 2e-06,
         "input_cost_per_token_above_200k_tokens": 4e-06,
         "litellm_provider": "azure_ai",
         "max_input_tokens": 200000,
@@ -29288,6 +29288,7 @@
         "mode": "image_generation",
         "output_cost_per_image": 0.134,
         "output_cost_per_image_token": 0.00012,
+        "output_cost_per_image_token_batches": 6e-05,
         "output_cost_per_token": 1.2e-05,
         "rpm": 1000,
         "tpm": 4000000,
@@ -29379,6 +29380,7 @@
         "mode": "image_generation",
         "output_cost_per_image": 0.045,
         "output_cost_per_image_token": 6e-05,
+        "output_cost_per_image_token_batches": 3e-05,
         "output_cost_per_token": 3e-06,
         "output_cost_per_token_batches": 1.5e-06,
         "rpm": 1000,
@@ -80419,8 +80421,7 @@
         "supported_endpoints": [
             "/v1/chat/completions",
             "/v1/images/generations",
-            "/v1/images/edits",
-            "/v1/batch"
+            "/v1/images/edits"
         ],
         "supported_modalities": [
             "text",
@@ -80533,5 +80534,27 @@
         "supports_vision": true,
         "supports_web_search": true,
         "web_search_billing_unit": "per_query"
+    },
+    "azure_ai/grok-4.7": {
+        "cache_read_input_token_cost": 5e-07,
+        "cache_read_input_token_cost_above_200k_tokens": 1e-06,
+        "deprecation_date": "2027-10-05",
+        "input_cost_per_token": 2e-06,
+        "input_cost_per_token_above_200k_tokens": 4e-06,
+        "litellm_provider": "azure_ai",
+        "max_input_tokens": 500000,
+        "max_output_tokens": 500000,
+        "max_tokens": 500000,
+        "mode": "chat",
+        "output_cost_per_token": 6e-06,
+        "output_cost_per_token_above_200k_tokens": 1.2e-05,
+        "source": "https://docs.x.ai/developers/models/grok-4.7",
+        "supports_function_calling": true,
+        "supports_prompt_caching": true,
+        "supports_reasoning": true,
+        "supports_response_schema": true,
+        "supports_tool_choice": true,
+        "supports_vision": true,
+        "supports_web_search": true
     }
 }

model_prices_and_context_window.schema.json

@@ -710,6 +710,10 @@
           "type": "number",
           "minimum": 0
         },
+        "output_cost_per_image_token_batches": {
+          "type": "number",
+          "minimum": 0
+        },
         "output_cost_per_pixel": {
           "type": "number",
           "minimum": 0

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm"
-version = "1.105.0"
+version = "1.106.0"
 description = "Library to easily interface with LLM API providers"
 readme = "README.md"
 requires-python = ">=3.10, <3.15"
@@ -29,6 +29,7 @@ dependencies = [
     "click>=8.0.0,<9.0",
     "jinja2>=3.1.6,<4.0",
     "aiohttp>=3.14.2,<4.0",
+    "async-timeout>=4.0.3,<6.0; python_version < '3.11'",
     "pydantic>=2.11.0,<3.0.0; python_version < '3.14'",
     "pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
     "pydantic-settings>=2.14.1,<3.0",
@@ -76,8 +77,8 @@ proxy = [
     "mcp>=2.2.0,<3",
     "httpx2>=2.5.0,<3",
     "pydantic>=2.12.0,<3",
-    "litellm-proxy-extras==0.4.105",
-    "litellm-enterprise==0.1.73",
+    "litellm-proxy-extras==0.4.106",
+    "litellm-enterprise==0.1.74",
     "RestrictedPython>=8.5,<9.0",
     "rich>=13.9.4,<14.0",
     "InquirerPy>=0.3.4,<1.0",
@@ -371,7 +372,7 @@ litellm-enterprise = { workspace = true }
 members = ["enterprise", "litellm-proxy-extras"]
 
 [tool.commitizen]
-version = "1.105.0"
+version = "1.106.0"
 version_files = [
     "pyproject.toml:^version",
 ]

scripts/seed_tracing_fixtures.py

@@ -108,7 +108,13 @@ def response_ids(rows: tuple[SpendLogRecord, ...]) -> Iterator[str]:
 def response_pattern(rows: tuple[SpendLogRecord, ...]) -> re.Pattern[str]:
     identities: Final = sorted(
         frozenset(
-            identity for identity in chain(response_ids(rows), (row["litellm_call_id"] for row in rows)) if identity
+            identity
+            for identity in chain(
+                response_ids(rows),
+                (row["litellm_call_id"] for row in rows),
+                (row.get("provider_request_id", "") for row in rows),
+            )
+            if identity
         ),
         key=len,
         reverse=True,
@@ -119,8 +125,11 @@ def response_pattern(rows: tuple[SpendLogRecord, ...]) -> re.Pattern[str]:
 def rebased_response(value: str, namespace: str, pattern: re.Pattern[str]) -> str:
     decoded: Final = managed_response(value)
     if decoded is None:
+        if value.startswith(("msg_", "req_")):
+            prefix, suffix = value.split("_", 1)
+            return f"{prefix}_seed-{namespace}-{suffix}"
         return f"seed-{namespace}-{value}"
-    payload: Final = pattern.sub(lambda match: f"seed-{namespace}-{match.group()}", decoded)
+    payload: Final = pattern.sub(lambda match: rebased_response(match.group(), namespace, pattern), decoded)
     return "resp_" + base64.b64encode(payload.encode()).decode()
 
 
@@ -194,9 +203,7 @@ def rebase(
     if isinstance(value, dict):
         attribute_key: Final = value.get("key")
         attribute_value: Final = value.get("value")
-        session_id: Final = (
-            attribute_value.get("stringValue") if isinstance(attribute_value, dict) else None
-        )
+        session_id: Final = attribute_value.get("stringValue") if isinstance(attribute_value, dict) else None
         if (
             isinstance(attribute_key, str)
             and attribute_key in TRACE_ID_ATTRIBUTES
@@ -431,6 +438,7 @@ WHERE t.TraceId IN {{trace_ids:Array(String)}}
 SELECT s.* REPLACE (
     {clickhouse_call_id("s.request_id")} AS request_id,
     {clickhouse_call_id("s.response_id")} AS response_id,
+    {clickhouse_call_id("s.provider_request_id")} AS provider_request_id,
     {clickhouse_call_id("s.litellm_call_id")} AS litellm_call_id,
     {clickhouse_hash("s.trace_id", trace_salt, 32)} AS trace_id,
     {clickhouse_hash("s.session_id", trace_salt, 32)} AS session_id,

tests/code_coverage_tests/unbounded_in_baseline.txt

@@ -1,7 +1,6 @@
 # Grandfathered findings of check_unbounded_in_lists.py: path::scope::kind::subject::occurrence.
 # Fix a site and delete its line; regenerate with `check_unbounded_in_lists.py --update-baseline`.
 enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py CheckResponsesCost.check_responses_cost prisma id.in `[job.id for job in completed_jobs]` 0
-enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py list_projects prisma team_id.in `user_team_ids` 0
 litellm/integrations/shadow_eval_logger.py ShadowEvalLogger._active_jobs prisma job_id.in `[str(record.id) for record in records]` 0
 litellm/llms/litellm_proxy/skills/handler.py LiteLLMSkillsHandler.list_skills prisma created_by.in `owner_scopes` 0
 litellm/proxy/_experimental/mcp_server/db.py get_mcp_servers prisma server_id.in `server_ids` 0

tests/integration/_support/manifest.py

@@ -18,6 +18,7 @@ OWNED_DIRECTORIES: Final = frozenset(
         "compatibility",
         "sdk",
         "cost_calculation",
+        "caching",
         "security",
     }
 )

tests/integration/caching/response_cache_case.py

@@ -0,0 +1,364 @@
+"""Scripted provider answers, upstream observations and Redis helpers shared by the response cache cells.
+
+The bodies carry ``$UNIQUE_ID`` so every upstream answer has its own id: an answer served from the
+cache repeats the id of the request that filled it, an answer that reached the upstream again does not.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import os
+import re
+import uuid
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import Final
+
+import httpx
+from pydantic import JsonValue
+from redis import Redis
+
+from tests.integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value
+from tests.integration._support.upstream import CONTROL_URL, ScenarioHandle, delete_scenario, register_scenario
+from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
+
+CACHE_KEY: Final = re.compile(r"^[0-9a-f]{64}$")
+PROVIDER_KEY: Final = "integration-provider-key"
+CHAT_MODEL: Final = "openai/gpt-4o-mini"
+TEXT_MODEL: Final = "openai/gpt-3.5-turbo-instruct"
+MESSAGES_MODEL: Final = "anthropic/claude-haiku-4-5"
+
+CHAT: Final[dict[str, JsonValue]] = {
+    "id": "chatcmpl-$UNIQUE_ID",
+    "object": "chat.completion",
+    "created": 1,
+    "model": "gpt-4o-mini",
+    "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+TEXT: Final[dict[str, JsonValue]] = {
+    "id": "cmpl-$UNIQUE_ID",
+    "object": "text_completion",
+    "created": 1,
+    "model": "gpt-3.5-turbo-instruct",
+    "choices": [{"text": "scripted", "index": 0, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+RESPONSE: Final[dict[str, JsonValue]] = {
+    "id": "resp_$UNIQUE_ID",
+    "object": "response",
+    "created_at": 1,
+    "status": "completed",
+    "model": "gpt-4o-mini",
+    "output": [
+        {
+            "id": "msg_$UNIQUE_ID",
+            "type": "message",
+            "status": "completed",
+            "role": "assistant",
+            "content": [{"type": "output_text", "text": "scripted", "annotations": []}],
+        }
+    ],
+    "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+}
+MESSAGE: Final[dict[str, JsonValue]] = {
+    "id": "msg-$UNIQUE_ID",
+    "type": "message",
+    "role": "assistant",
+    "model": "claude-haiku-4-5",
+    "content": [{"type": "text", "text": "scripted"}],
+    "stop_reason": "end_turn",
+    "stop_sequence": None,
+    "usage": {"input_tokens": 1, "output_tokens": 1},
+}
+EMBEDDING: Final[dict[str, JsonValue]] = {
+    "object": "list",
+    "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
+    "model": "text-embedding-$UNIQUE_ID",
+    "usage": {"prompt_tokens": 1, "total_tokens": 1},
+}
+RERANK: Final[dict[str, JsonValue]] = {
+    "id": "rerank-$UNIQUE_ID",
+    "results": [{"index": 0, "relevance_score": 0.5}],
+    "meta": {},
+}
+TRANSCRIPT: Final[dict[str, JsonValue]] = {"text": "scripted $UNIQUE_ID"}
+
+CHAT_CHUNK: Final = (
+    'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":'
+)
+CHAT_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        CHAT_CHUNK + '[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{"content":"streamed "},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{"content":"response"},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{},"finish_reason":"stop"}]}',
+        "data: [DONE]",
+    ),
+)
+CHAT_STREAM_WITHOUT_CHOICES: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(CHAT_CHUNK + "[]}", "data: [DONE]"),
+)
+TEXT_CHUNK: Final = (
+    'data: {"id":"cmpl-$UNIQUE_ID","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":'
+)
+TEXT_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        TEXT_CHUNK + '[{"text":"streamed ","index":0,"finish_reason":null}]}',
+        TEXT_CHUNK + '[{"text":"response","index":0,"finish_reason":null}]}',
+        TEXT_CHUNK + '[{"text":"","index":0,"finish_reason":"stop"}]}',
+        "data: [DONE]",
+    ),
+)
+TEXT_STREAM_WITHOUT_CHOICES: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(TEXT_CHUNK + "[]}", "data: [DONE]"),
+)
+RESPONSE_CREATED: Final = (
+    "event: response.created\n"
+    'data: {"type":"response.created","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+    '"created_at":1,"status":"in_progress","model":"gpt-4o-mini","output":[],"usage":null}}'
+)
+RESPONSE_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        RESPONSE_CREATED,
+        (
+            "event: response.output_item.added\n"
+            'data: {"type":"response.output_item.added","output_index":0,'
+            '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"in_progress","role":"assistant","content":[]}}'
+        ),
+        (
+            "event: response.content_part.added\n"
+            'data: {"type":"response.content_part.added","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}'
+        ),
+        (
+            "event: response.output_text.delta\n"
+            'data: {"type":"response.output_text.delta","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"delta":"streamed response"}'
+        ),
+        (
+            "event: response.output_text.done\n"
+            'data: {"type":"response.output_text.done","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"text":"streamed response"}'
+        ),
+        (
+            "event: response.content_part.done\n"
+            'data: {"type":"response.content_part.done","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"part":{"type":"output_text","text":"streamed response","annotations":[]}}'
+        ),
+        (
+            "event: response.output_item.done\n"
+            'data: {"type":"response.output_item.done","output_index":0,'
+            '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"completed","role":"assistant",'
+            '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}}'
+        ),
+        (
+            "event: response.completed\n"
+            'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+            '"created_at":1,"status":"completed","model":"gpt-4o-mini",'
+            '"output":[{"id":"msg_$UNIQUE_ID","type":"message","status":"completed","role":"assistant",'
+            '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}],'
+            '"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}'
+        ),
+    ),
+)
+RESPONSE_STREAM_WITHOUT_OUTPUT: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        RESPONSE_CREATED,
+        (
+            "event: response.completed\n"
+            'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+            '"created_at":1,"status":"completed","model":"gpt-4o-mini","output":[],'
+            '"usage":{"input_tokens":1,"output_tokens":0,"total_tokens":1}}}'
+        ),
+    ),
+)
+MESSAGE_START: Final = (
+    "event: message_start\n"
+    'data: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant",'
+    '"model":"claude-haiku-4-5","content":[],"stop_reason":null,"stop_sequence":null,'
+    '"usage":{"input_tokens":1,"output_tokens":0}}}'
+)
+MESSAGE_END: Final = (
+    (
+        "event: message_delta\n"
+        'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},'
+        '"usage":{"output_tokens":2}}'
+    ),
+    'event: message_stop\ndata: {"type":"message_stop"}',
+)
+MESSAGE_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        MESSAGE_START,
+        (
+            "event: content_block_start\n"
+            'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}'
+        ),
+        (
+            "event: content_block_delta\n"
+            'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"streamed response"}}'
+        ),
+        'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
+        *MESSAGE_END,
+    ),
+)
+MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS: Final = SseResponse(
+    content_type="text/event-stream", frames=(MESSAGE_START, *MESSAGE_END)
+)
+
+
+class Upstream:
+    """The owned upstream's observations, drained once per read and kept, filtered per deployment."""
+
+    def __init__(self, url: str = CONTROL_URL) -> None:
+        self._url: Final = url.rstrip("/")
+        self._seen: tuple[dict[str, JsonValue], ...] = ()
+
+    def received(self, identity: str) -> tuple[dict[str, JsonValue], ...]:
+        with httpx.Client(timeout=10, trust_env=False) as client:
+            payload: Final = JSON_OBJECT.validate_json(client.get(f"{self._url}/__observations").content)
+        requests: Final = payload["requests"]
+        assert isinstance(requests, list)
+        self._seen = (
+            *self._seen,
+            *(object_value(item) for item in requests),
+        )  # rebind-ok: drained observations accumulate
+        return tuple(item for item in self._seen if f"/{identity}/" in string_value(item.get("path", "")))
+
+    def calls(self, identity: str) -> int:
+        """The LLM calls the deployment took; the proxy's boot-time ``GET /v1/models`` discovery of a config
+        deployment is not one."""
+        return sum(1 for item in self.received(identity) if item.get("method") == "POST")
+
+
+def json_response(body: Mapping[str, JsonValue], status: int = 200) -> JsonResponse:
+    return JsonResponse(content_type="application/json", body=dict(body), status=status)
+
+
+def slowed(stream: SseResponse, frame_delay_ms: int) -> SseResponse:
+    return stream.model_copy(update={"frame_delay_ms": frame_delay_ms})
+
+
+def emptied(body: Mapping[str, JsonValue], field: str) -> dict[str, JsonValue]:
+    return {**body, field: []}
+
+
+def scenario_id() -> str:
+    return f"cache-{uuid.uuid4().hex[:12]}"
+
+
+def scripted(scenario: Scenario, response: JsonResponse | SseResponse) -> ScenarioHandle:
+    handle: Final = register_scenario(scenario_id(), response, control_url=scenario.gateway.upstream_url)
+    scenario.cleanups.callback(delete_scenario, handle)
+    return handle
+
+
+@contextmanager
+def scripted_in_process(response: JsonResponse | SseResponse) -> Iterator[ScenarioHandle]:
+    handle: Final = register_scenario(scenario_id(), response)
+    try:
+        yield handle
+    finally:
+        delete_scenario(handle)
+
+
+def rescript(handle: ScenarioHandle, response: JsonResponse | SseResponse) -> None:
+    register_scenario(handle.scenario_id, response, control_url=handle.control_url)
+
+
+def prompt() -> str:
+    return f"cache probe {uuid.uuid4().hex}"
+
+
+def redis_store() -> Redis:
+    return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]))
+
+
+def entry_key(store: Redis, needle: str) -> str | None:
+    """The response cache key (a bare sha256) whose stored value names ``needle``, or ``None``."""
+    for raw_key in store.scan_iter(count=1000):
+        key: Final = raw_key.decode()
+        if not CACHE_KEY.match(key) or store.type(key) != b"string":
+            continue
+        value: Final = store.get(key)
+        if value is not None and needle.encode() in value:
+            return key
+    return None
+
+
+def await_entry(store: Redis, needle: str) -> str:
+    key: Final = eventually(lambda: entry_key(store, needle), lambda found: found is not None)
+    assert key is not None
+    return key
+
+
+def _decode_entry(raw: bytes) -> tuple[dict[str, JsonValue], bool]:
+    """The stored entry and whether it is JSON; the sync SDK path stores ``str(dict)`` instead."""
+    text: Final = raw.decode()
+    if text.startswith('{"'):
+        return object_value(json.loads(text)), True
+    return object_value(ast.literal_eval(text)), False
+
+
+SHORT_TTL_SECONDS: Final = 2
+SHORT_TTL_CACHE_CONTROL: Final[dict[str, JsonValue]] = {"ttl": SHORT_TTL_SECONDS}
+STALE_ENTRY_TTL_SECONDS: Final = 60
+STALE_ID_SUFFIX: Final = "-stale"
+
+
+def flip_entry_to_empty(store: Redis, key: str, field: str) -> str:
+    """Rewrite the stored response so ``field`` is ``[]`` and its id carries ``-stale``, keeping the encoding.
+
+    Returns the stale id. The proxy worker that wrote the entry keeps a copy in its own memory for the entry's ttl,
+    so a cell fills the entry with ``SHORT_TTL_CACHE_CONTROL`` and reads the rewritten entry once that copy lapsed.
+    """
+    raw: Final = store.get(key)
+    assert raw is not None, key
+    entry, is_json = _decode_entry(raw)
+    response: Final = entry["response"]
+    stored: Final = object_value(json.loads(response) if isinstance(response, str) else response)
+    assert field in stored, f"{field!r} missing from the cached response {sorted(stored)}"
+    stale_id: Final = f"{string_value(stored['id'])}{STALE_ID_SUFFIX}"
+    flipped: Final[dict[str, JsonValue]] = {**stored, field: [], "id": stale_id}
+    rewritten: Final[dict[str, JsonValue]] = {
+        **entry,
+        "response": json.dumps(flipped) if isinstance(response, str) else flipped,
+    }
+    store.set(key, json.dumps(rewritten) if is_json else str(rewritten), ex=STALE_ENTRY_TTL_SECONDS)
+    return stale_id
+
+
+def chat_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]:
+    return {"model": model, "messages": [{"role": "user", "content": text}], **extra}
+
+
+def message_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]:
+    return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}], **extra}
+
+
+def response_id(response: httpx.Response) -> str:
+    assert response.status_code == 200, response.text
+    return string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+
+def choices(response: httpx.Response) -> list[JsonValue]:
+    assert response.status_code == 200, response.text
+    found: Final = JSON_OBJECT.validate_json(response.content)["choices"]
+    assert isinstance(found, list), response.text
+    return found
+
+
+def content_of_first_choice(response: httpx.Response) -> JsonValue:
+    return object_value(object_value(choices(response)[0])["message"])["content"]
+
+
+def text_of_first_choice(response: httpx.Response) -> JsonValue:
+    return object_value(choices(response)[0])["text"]

tests/integration/caching/test_response_cache_chaos.py

@@ -0,0 +1,503 @@
+"""Chaos rows for the response cache on an owned two-worker proxy.
+
+Each owned proxy carries its own deployments pointed at scripted upstream scenarios, so both workers know
+them from boot. A 24-request burst (chat, text completions, Responses and Anthropic Messages, two of each
+streamed at 500 ms per frame so they are still in flight when the outage lands) runs while the cache store
+is stopped (C1), paused (C2) or while one worker has just been SIGKILLed (C3): every request answers 200
+with content and lands exactly once in the spend log. After recovery an upstream answer with no choices
+still reaches the upstream again on the next identical request. C4 runs the in-memory cache mode, where
+each worker keeps its own store and the cached object is read back without a JSON round trip.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import uuid
+from collections.abc import Callable, Iterator, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path
+from typing import Final
+
+import httpx
+import psutil
+import pytest
+import yaml
+from pydantic import JsonValue
+from redis import Redis
+
+from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, string_value
+from tests.integration._support.database import read_rows
+from tests.integration._support.process import owned_proxy_process
+from tests.integration._support.redis_process import owned_redis
+from tests.integration._support.upstream import ScenarioHandle
+from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    CHAT_STREAM,
+    MESSAGE,
+    MESSAGE_STREAM,
+    MESSAGES_MODEL,
+    PROVIDER_KEY,
+    RESPONSE,
+    RESPONSE_STREAM,
+    TEXT,
+    TEXT_MODEL,
+    TEXT_STREAM,
+    Upstream,
+    chat_body,
+    choices,
+    content_of_first_choice,
+    emptied,
+    json_response,
+    message_body,
+    prompt,
+    rescript,
+    response_id,
+    scripted,
+    slowed,
+)
+
+RecordProperty = Callable[[str, object], None]
+
+WORKERS: Final = 2
+REMOVE_FROM_ENVIRONMENT: Final = ("DATABASE_URL_READ_REPLICA",)
+BURST_TIMEOUT_SECONDS: Final = 60
+CHAOS_AFTER_ANSWERS: Final = 4
+STREAM_FRAME_DELAY_MS: Final = 500
+PER_ENDPOINT: Final = 6
+STREAMS_PER_ENDPOINT: Final = 2
+LOCAL_BATCH: Final = 20
+SPEND_SQL: Final = 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
+
+
+@dataclass(frozen=True, slots=True)
+class Deployment:
+    name: str
+    litellm_model: str
+    handle: ScenarioHandle
+
+    def entry(self) -> dict[str, JsonValue]:
+        return {
+            "model_name": self.name,
+            "litellm_params": {
+                "model": self.litellm_model,
+                "api_base": self.handle.api_base(),
+                "api_key": PROVIDER_KEY,
+            },
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class Fleet:
+    chat: Deployment
+    chat_stream: Deployment
+    text: Deployment
+    text_stream: Deployment
+    responses: Deployment
+    responses_stream: Deployment
+    messages: Deployment
+    messages_stream: Deployment
+    flip: Deployment
+
+    def burst_members(self) -> tuple[Deployment, ...]:
+        return (
+            self.chat,
+            self.chat_stream,
+            self.text,
+            self.text_stream,
+            self.responses,
+            self.responses_stream,
+            self.messages,
+            self.messages_stream,
+        )
+
+    def entries(self) -> list[dict[str, JsonValue]]:
+        return [deployment.entry() for deployment in (*self.burst_members(), self.flip)]
+
+
+@dataclass(frozen=True, slots=True)
+class Call:
+    label: str
+    path: str
+    body: dict[str, JsonValue]
+
+    @property
+    def streamed(self) -> bool:
+        return self.label.startswith("stream-")
+
+
+@dataclass(frozen=True, slots=True)
+class Answer:
+    call: Call
+    status: int | None
+    text: str
+
+    @property
+    def has_content(self) -> bool:
+        marker: Final = "streamed" if self.call.streamed else "scripted"
+        return self.status == 200 and marker in self.text
+
+    def describe(self) -> str:
+        return f"{self.call.label}: {self.status} {self.text[:160]!r}"
+
+
+def _deployment(scenario: Scenario, kind: str, litellm_model: str, response: JsonResponse | SseResponse) -> Deployment:
+    return Deployment(f"chaos-{kind}-{uuid.uuid4().hex[:8]}", litellm_model, scripted(scenario, response))
+
+
+def _fleet(scenario: Scenario) -> Fleet:
+    return Fleet(
+        chat=_deployment(scenario, "chat", CHAT_MODEL, json_response(CHAT)),
+        chat_stream=_deployment(scenario, "chat-stream", CHAT_MODEL, slowed(CHAT_STREAM, STREAM_FRAME_DELAY_MS)),
+        text=_deployment(scenario, "text", TEXT_MODEL, json_response(TEXT)),
+        text_stream=_deployment(scenario, "text-stream", TEXT_MODEL, slowed(TEXT_STREAM, STREAM_FRAME_DELAY_MS)),
+        responses=_deployment(scenario, "responses", CHAT_MODEL, json_response(RESPONSE)),
+        responses_stream=_deployment(
+            scenario, "responses-stream", CHAT_MODEL, slowed(RESPONSE_STREAM, STREAM_FRAME_DELAY_MS)
+        ),
+        messages=_deployment(scenario, "messages", MESSAGES_MODEL, json_response(MESSAGE)),
+        messages_stream=_deployment(
+            scenario, "messages-stream", MESSAGES_MODEL, slowed(MESSAGE_STREAM, STREAM_FRAME_DELAY_MS)
+        ),
+        flip=_deployment(scenario, "flip", CHAT_MODEL, json_response(emptied(CHAT, "choices"))),
+    )
+
+
+def _config(directory: Path, fleet: Fleet, cache_params: dict[str, JsonValue] | None = None) -> Path:
+    """The rig's proxy config with the fleet as its model list and, when given, another response cache."""
+    rig: Final = yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text())
+    assert isinstance(rig, dict)
+    litellm_settings: Final = rig["litellm_settings"]
+    assert isinstance(litellm_settings, dict)
+    config: Final = {
+        **rig,
+        "model_list": fleet.entries(),
+        "litellm_settings": {**litellm_settings, **({"cache_params": cache_params} if cache_params else {})},
+    }
+    path: Final = directory / f"proxy_config-{uuid.uuid4().hex[:8]}.yaml"
+    path.write_text(yaml.safe_dump(config, sort_keys=False))
+    return path
+
+
+def _overrides() -> dict[str, str]:
+    return {"DATABASE_URL": os.environ["DATABASE_URL"]}
+
+
+def _burst_calls(fleet: Fleet, per_endpoint: int, streams_per_endpoint: int) -> tuple[Call, ...]:
+    def calls() -> Iterator[Call]:
+        for index in range(per_endpoint):
+            streamed: Final = index < streams_per_endpoint
+            prefix: Final = "stream" if streamed else "plain"
+            extra: Final[dict[str, JsonValue]] = {"stream": True} if streamed else {}
+            chat: Final = fleet.chat_stream if streamed else fleet.chat
+            text: Final = fleet.text_stream if streamed else fleet.text
+            responses: Final = fleet.responses_stream if streamed else fleet.responses
+            messages: Final = fleet.messages_stream if streamed else fleet.messages
+            yield Call(f"{prefix}-chat-{index}", "/v1/chat/completions", chat_body(chat.name, prompt(), **extra))
+            yield Call(f"{prefix}-text-{index}", "/v1/completions", {"model": text.name, "prompt": prompt(), **extra})
+            yield Call(
+                f"{prefix}-responses-{index}", "/v1/responses", {"model": responses.name, "input": prompt(), **extra}
+            )
+            yield Call(f"{prefix}-messages-{index}", "/v1/messages", message_body(messages.name, prompt(), **extra))
+
+    return tuple(calls())
+
+
+class Burst:
+    """Every call submitted at once against ``target``; ``chaos_point`` is set once ``CHAOS_AFTER_ANSWERS``
+    have answered (or failed), while the slowed streams are still in flight."""
+
+    def __init__(self, target: str, key: str) -> None:
+        self._target: Final = target
+        self._key: Final = key
+        self._lock: Final = threading.Lock()
+        self._answers = 0  # rebind-ok: counter behind _lock
+        self._futures: dict[str, Future[Answer]] = {}
+        self.chaos_point: Final = threading.Event()
+
+    def start(self, pool: ThreadPoolExecutor, calls: Sequence[Call]) -> None:
+        assert not self._futures, "burst already started"
+        self._futures.update((call.label, pool.submit(self._send, call)) for call in calls)
+        assert self.chaos_point.wait(BURST_TIMEOUT_SECONDS), (
+            f"fewer than {CHAOS_AFTER_ANSWERS} requests answered within {BURST_TIMEOUT_SECONDS}s"
+        )
+
+    def _send(self, call: Call) -> Answer:
+        try:
+            with httpx.Client(base_url=self._target, timeout=BURST_TIMEOUT_SECONDS, trust_env=False) as client:
+                response: Final = client.post(
+                    call.path, json=call.body, headers={"Authorization": f"Bearer {self._key}"}
+                )
+            answer = Answer(call, response.status_code, response.text)
+        except httpx.HTTPError as error:
+            answer = Answer(call, None, f"{type(error).__name__}: {error}"[:200])
+        with self._lock:
+            self._answers += 1
+            if self._answers >= CHAOS_AFTER_ANSWERS:
+                self.chaos_point.set()
+        return answer
+
+    def answered(self) -> int:
+        with self._lock:
+            return self._answers
+
+    def pending_streams(self) -> tuple[str, ...]:
+        return tuple(
+            label for label, future in self._futures.items() if label.startswith("stream-") and not future.done()
+        )
+
+    def outcomes(self) -> tuple[Answer, ...]:
+        return tuple(future.result(timeout=BURST_TIMEOUT_SECONDS + 30) for future in self._futures.values())
+
+
+def _assert_all_answered_with_content(outcomes: Sequence[Answer]) -> None:
+    failures: Final = tuple(answer.describe() for answer in outcomes if not answer.has_content)
+    assert not failures, (
+        f"{len(failures)} of {len(outcomes)} burst requests lack a 200 with content:\n  " + "\n  ".join(failures)
+    )
+
+
+def _plain_ids(outcomes: Sequence[Answer]) -> frozenset[str]:
+    """The ids of the non-streamed chat, text and Messages answers, which the spend log records verbatim."""
+    return frozenset(
+        string_value(JSON_OBJECT.validate_json(answer.text)["id"])
+        for answer in outcomes
+        if not answer.call.streamed and answer.call.path != "/v1/responses"
+    )  # comprehension-ok: one filter over the burst
+
+
+def _spend_request_ids(key: str) -> frozenset[str]:
+    return frozenset(
+        string_value(row["request_id"]) for row in read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),))
+    )
+
+
+def _assert_every_burst_request_landed_once(key: str, outcomes: Sequence[Answer]) -> None:
+    rows: Final = eventually(lambda: _spend_request_ids(key), lambda ids: len(ids) >= len(outcomes), seconds=70)
+    assert len(rows) == len(outcomes), f"{len(rows)} spend rows for {len(outcomes)} burst requests: {sorted(rows)}"
+    missing: Final = _plain_ids(outcomes) - rows
+    assert not missing, f"burst answers without a spend row: {sorted(missing)}"
+
+
+def _chat(target: str, key: str, body: dict[str, JsonValue]) -> httpx.Response:
+    with httpx.Client(base_url=target, timeout=30, trust_env=False) as client:
+        return client.post("/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"})
+
+
+def _assert_empty_answer_reaches_upstream_again(target: str, key: str, upstream: Upstream, flip: Deployment) -> None:
+    """Through ``target`` after the outage: the scripted empty answer is never served again once the
+    upstream answers for real, and the real answer is the one served from the cache afterwards."""
+    request: Final = chat_body(flip.name, prompt())
+    empty: Final = _chat(target, key, request)
+    assert choices(empty) == [], empty.text
+    assert upstream.calls(flip.handle.scenario_id) == 1
+    rescript(flip.handle, json_response(CHAT))
+    refilled: Final = _chat(target, key, request)
+    assert content_of_first_choice(refilled) == "scripted", f"the empty answer was served: {refilled.text}"
+    assert upstream.calls(flip.handle.scenario_id) == 2
+    hit: Final = eventually(lambda: _chat(target, key, request), lambda r: response_id(r) == response_id(refilled))
+    assert content_of_first_choice(hit) == "scripted", hit.text
+    assert upstream.calls(flip.handle.scenario_id) >= 2
+
+
+def _workers(root: psutil.Process) -> tuple[psutil.Process, ...]:
+    """uvicorn's worker children of the owned proxy root, spawned through ``multiprocessing.spawn``."""
+
+    def spawned() -> Iterator[psutil.Process]:
+        for child in root.children():
+            try:
+                cmdline = child.cmdline()
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+            if any("multiprocessing.spawn" in part for part in cmdline):
+                yield child
+
+    return tuple(sorted(spawned(), key=lambda process: process.pid))
+
+
+def _cache_ping(target: str, key: str) -> httpx.Response:
+    with httpx.Client(base_url=target, timeout=30, trust_env=False) as client:
+        return client.get("/cache/ping", headers={"Authorization": f"Bearer {key}"})
+
+
+def _cache_status(response: httpx.Response) -> str:
+    assert response.status_code == 200, f"/cache/ping: {response.status_code} {response.text}"
+    return string_value(JSON_OBJECT.validate_json(response.content)["status"])
+
+
+def _proxy_url(gateway: Gateway) -> str:
+    return str(gateway.client.base_url).rstrip("/")
+
+
+@pytest.mark.timeout(300)
+def test_redis_stopped_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario, owned_redis(tmp_path) as store:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                {
+                    **_overrides(),
+                    "REDIS_HOST": store.host,
+                    "REDIS_PORT": str(store.port),
+                    "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "0",
+                },
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT))
+            pending: Final = burst.pending_streams()
+            record_property("answered_before_outage", burst.answered())
+            assert pending, "no stream was in flight when Redis stopped"
+            store.stop()
+            down: Final = _cache_ping(target, gateway.key)
+            assert down.status_code == 503, f"/cache/ping with Redis stopped: {down.status_code} {down.text}"
+            assert "Service Unhealthy" in down.text, down.text
+            outcomes: Final = burst.outcomes()
+            store.start()
+            recovered: Final = eventually(
+                lambda: _cache_ping(target, gateway.key), lambda r: r.status_code == 200, seconds=60
+            )
+            assert _cache_status(recovered) == "healthy"
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+@pytest.mark.timeout(300)
+def test_redis_paused_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario, owned_redis(tmp_path) as store:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                {**_overrides(), "REDIS_HOST": store.host, "REDIS_PORT": str(store.port)},
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+            Redis(host=store.host, port=store.port) as control,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT))
+            pending: Final = burst.pending_streams()
+            record_property("answered_before_pause", burst.answered())
+            assert pending, "no stream was in flight when Redis was paused"
+            assert control.client_pause(2000) is True
+            outcomes: Final = burst.outcomes()
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+@pytest.mark.timeout(300)
+def test_worker_killed_before_burst_leaves_the_survivor_serving_and_never_pins_an_empty_answer(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                _overrides(),
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            root: Final = psutil.Process(owned.process.pid)
+            before: Final = _workers(root)
+            assert len(before) == WORKERS, [process.pid for process in before]
+            victim: Final = before[0]
+            victim.kill()
+            victim.wait(timeout=10)
+            with httpx.Client(base_url=target, timeout=15, trust_env=False) as fresh:
+                readiness: Final = fresh.get("/health/readiness")
+            assert readiness.status_code == 200, f"/health/readiness with worker {victim.pid} dead: {readiness.text}"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT // 2, STREAMS_PER_ENDPOINT // 2))
+            outcomes: Final = burst.outcomes()
+            respawned: Final = eventually(
+                lambda: tuple(process.pid for process in _workers(root)),
+                lambda pids: len(pids) == WORKERS and victim.pid not in pids,
+                seconds=60,
+            )
+            record_property(
+                "worker_pids", {"before": [process.pid for process in before], "killed": victim.pid, "after": respawned}
+            )
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+def _batch(target: str, key: str, body: dict[str, JsonValue], size: int) -> tuple[httpx.Response, ...]:
+    """``size`` identical requests, each on a fresh connection so both workers take traffic."""
+    return tuple(_chat(target, key, body) for _ in range(size))
+
+
+@pytest.mark.timeout(300)
+def test_in_memory_cache_mode_never_serves_an_empty_answer_from_a_worker_store(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[fleet.flip.name])
+        with owned_proxy_process(
+            gateway,
+            tmp_path,
+            _overrides(),
+            config=_config(tmp_path, fleet, cache_params={"type": "local"}),
+            remove_environment=REMOVE_FROM_ENVIRONMENT,
+            workers=WORKERS,
+        ) as owned:
+            target: Final = _proxy_url(owned.gateway)
+            request: Final = chat_body(fleet.flip.name, prompt())
+            first_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            assert all(choices(response) == [] for response in first_batch), [r.text for r in first_batch]
+            assert upstream.calls(fleet.flip.handle.scenario_id) == LOCAL_BATCH, (
+                "an empty answer was served from a worker store"
+            )
+            rescript(fleet.flip.handle, json_response(CHAT))
+            second_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            served_empty: Final = tuple(response.text for response in second_batch if choices(response) == [])
+            assert not served_empty, (
+                f"{len(served_empty)} of {LOCAL_BATCH} answers still empty after the upstream recovered"
+            )
+            assert all(content_of_first_choice(response) == "scripted" for response in second_batch)
+            after_second: Final = upstream.calls(fleet.flip.handle.scenario_id)
+            assert LOCAL_BATCH < after_second <= 2 * LOCAL_BATCH, after_second
+            third_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            assert all(content_of_first_choice(response) == "scripted" for response in third_batch)
+            assert upstream.calls(fleet.flip.handle.scenario_id) - after_second <= WORKERS, (
+                "the real answer is not cached"
+            )

tests/integration/caching/test_response_cache_empty_output.py

@@ -0,0 +1,813 @@
+"""The response cache never stores an answer with no output and never serves a stored one.
+
+Every cell scripts the owned upstream for one deployment, drives the rig proxy through the client a
+user would hold (OpenAI SDK, Anthropic SDK, raw httpx) and reads three things: what the caller got,
+how often the upstream was called for that deployment (an answer served from the cache never reaches
+it), and what landed in Redis or the spend log. An upstream answer carrying no choices, no output items
+or no content blocks reaches the upstream again on the next identical request, and the real answer
+that follows is the one the cache keeps. Answers with output, errors, ``no-cache`` requests, the
+``/cache/delete`` workaround and the other cached call types are unchanged.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from functools import partial
+from hashlib import sha256
+from typing import Final, TypeAlias, TypeVar
+
+import httpx
+import pytest
+from anthropic import Anthropic, AsyncAnthropic
+from openai import AsyncOpenAI, OpenAI
+from pydantic import JsonValue
+
+from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value
+from tests.integration._support.database import read_rows
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    CHAT_STREAM,
+    CHAT_STREAM_WITHOUT_CHOICES,
+    EMBEDDING,
+    MESSAGE,
+    MESSAGE_STREAM,
+    MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS,
+    MESSAGES_MODEL,
+    RERANK,
+    RESPONSE,
+    RESPONSE_STREAM,
+    RESPONSE_STREAM_WITHOUT_OUTPUT,
+    SHORT_TTL_CACHE_CONTROL,
+    TEXT,
+    TEXT_MODEL,
+    TEXT_STREAM_WITHOUT_CHOICES,
+    TRANSCRIPT,
+    Upstream,
+    await_entry,
+    chat_body,
+    choices,
+    content_of_first_choice,
+    emptied,
+    flip_entry_to_empty,
+    json_response,
+    message_body,
+    prompt,
+    redis_store,
+    rescript,
+    response_id,
+    scripted,
+    text_of_first_choice,
+)
+
+MISSING: Final = object()
+EMPTY_STREAMS: Final = 6
+REPLAY_ATTEMPTS: Final = 8
+SETTLE_SECONDS: Final = 15
+T = TypeVar("T")
+_Answer: TypeAlias = tuple[frozenset[str], str, tuple[str, ...]]
+SPEND_SQL: Final = (
+    'SELECT request_id, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s ORDER BY "startTime", request_id'
+)
+
+
+def _proxy_root(gateway: Gateway) -> str:
+    return str(gateway.client.base_url).rstrip("/")
+
+
+def _openai(gateway: Gateway, key: str | None = None) -> OpenAI:
+    return OpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0)
+
+
+def _async_openai(gateway: Gateway, key: str | None = None) -> AsyncOpenAI:
+    return AsyncOpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0)
+
+
+def _anthropic(gateway: Gateway) -> Anthropic:
+    return Anthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0)
+
+
+def _async_anthropic(gateway: Gateway) -> AsyncAnthropic:
+    return AsyncAnthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0)
+
+
+def _chat(gateway: Gateway, body: Mapping[str, JsonValue], *, key: str | None = None) -> httpx.Response:
+    return gateway.request("POST", "/v1/chat/completions", body, key=key)
+
+
+def _spend_rows(key: str) -> list[dict[str, JsonValue]]:
+    return read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),))
+
+
+def _assert_prompt_reached_upstream(upstream: Upstream, identity: str, text: str, times: int) -> None:
+    received: Final = upstream.received(identity)
+    assert len(received) == times, f"{identity}: {len(received)} upstream calls, expected {times}"
+    assert all(text in json.dumps(item["body"]) for item in received), received
+
+
+def _served_without_upstream(upstream: Upstream, identity: str, call: Callable[[], T]) -> T | None:
+    before: Final = upstream.calls(identity)
+    answer: Final = call()
+    return None if upstream.calls(identity) > before else answer
+
+
+@dataclass(frozen=True, slots=True)
+class Streamed:
+    ids: frozenset[str]
+    text: str
+    kinds: tuple[str, ...]
+    reached_upstream: bool
+
+
+def _streams(stream: Callable[[], _Answer], upstream: Upstream, identity: str, attempts: int) -> Iterator[Streamed]:
+    """Stream up to ``attempts`` times, stopping after the first answer served without an upstream call."""
+    for _ in range(attempts):
+        before: Final = upstream.calls(identity)
+        ids, text, kinds = stream()
+        answer: Final = Streamed(ids, text, kinds, upstream.calls(identity) > before)
+        yield answer
+        if not answer.reached_upstream:
+            return
+
+
+def _never_replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str) -> tuple[Streamed, ...]:
+    """``EMPTY_STREAMS`` identical streams, every one reaching the upstream and carrying ids of its own."""
+    answers: Final = tuple(_streams(stream, upstream, identity, EMPTY_STREAMS))
+    assert len(answers) == EMPTY_STREAMS and all(answer.reached_upstream for answer in answers), (
+        f"a stream was replayed from the cache: {answers}"
+    )
+    distinct: Final = frozenset[str]().union(*(answer.ids for answer in answers))
+    assert sum(len(answer.ids) for answer in answers) == len(distinct), answers
+    return answers
+
+
+def _replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str, produced: frozenset[str]) -> Streamed:
+    """The first of up to ``REPLAY_ATTEMPTS`` identical streams served without an upstream call.
+
+    A streamed request can reach the upstream once per proxy worker before a replay is served: the proxy adds
+    ``stream_options`` to a stream before keying it only when its own router already resolves the model, and the
+    worker that did not take the ``/model/new`` call resolves it once its router syncs. The replay carries ids an
+    upstream answer produced.
+    """
+    answers: Final = tuple(_streams(stream, upstream, identity, REPLAY_ATTEMPTS))
+    replay: Final = answers[-1]
+    assert not replay.reached_upstream, (
+        f"no stream was replayed from the cache in {REPLAY_ATTEMPTS} attempts: {answers}"
+    )
+    seen: Final = produced.union(*(answer.ids for answer in answers[:-1]))
+    assert replay.ids <= seen, (replay.ids, seen)
+    return replay
+
+
+def test_chat_empty_choices_reach_upstream_again_and_the_real_answer_is_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _openai(gateway) as client:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        empty: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}])
+        assert empty.choices == [], empty.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1)
+        rescript(handle, json_response(CHAT))
+        real: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}])
+        assert real.id != empty.id and real.choices[0].message.content == "scripted", real.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2)
+        await_entry(store, real.id)
+        hit: Final = client.chat.completions.with_raw_response.create(
+            model=model, messages=[{"role": "user", "content": text}]
+        )
+        served: Final = hit.parse()
+        assert served.id == real.id and served.choices[0].message.content == "scripted", hit.text
+        assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_async_chat_spend_rows_record_the_empty_answer_and_the_refill_as_misses(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        key: Final = scenario.key(models=[model])
+
+        async def drive() -> tuple[str, str, str]:
+            async with _async_openai(gateway, key) as client:
+                messages: Final = [{"role": "user", "content": text}]
+                empty = await client.chat.completions.create(model=model, messages=messages)
+                assert empty.choices == [], empty.model_dump_json()
+                rescript(handle, json_response(CHAT))
+                real = await client.chat.completions.create(model=model, messages=messages)
+                assert real.choices[0].message.content == "scripted", real.model_dump_json()
+                await_entry(store, real.id)
+                served = await client.chat.completions.create(model=model, messages=messages)
+                return empty.id, real.id, served.id
+
+        empty_id, real_id, served_id = asyncio.run(drive())
+        assert served_id == real_id != empty_id
+        assert upstream.calls(handle.scenario_id) == 2
+        rows: Final = eventually(lambda: _spend_rows(key), lambda found: len(found) == 3, seconds=70)
+        by_id: Final = {string_value(row["request_id"]): row for row in rows}
+        assert set(by_id) >= {empty_id, real_id}, sorted(by_id)
+        assert by_id[empty_id]["cache_hit"] != "True" and by_id[real_id]["cache_hit"] != "True", rows
+        hit_row: Final = next(row for request_id, row in by_id.items() if request_id.startswith(f"{real_id}_cache_hit"))
+        assert hit_row["cache_hit"] == "True" and float(str(hit_row["spend"])) == 0, rows
+
+
+def test_text_completion_empty_choices_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(TEXT, "choices")))
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "prompt": prompt()}
+        empty: Final = gateway.request("POST", "/v1/completions", body)
+        assert choices(empty) == [], empty.text
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(TEXT))
+        real: Final = gateway.request("POST", "/v1/completions", body)
+        assert text_of_first_choice(real) == "scripted" and response_id(real) != response_id(empty), real.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(real))
+        hit: Final = gateway.request("POST", "/v1/completions", body)
+        assert response_id(hit) == response_id(real) and text_of_first_choice(hit) == "scripted", hit.text
+        assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_responses_empty_output_reaches_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(RESPONSE, "output")))
+        model: Final = scenario.model(api_base=handle.api_base())
+
+        async def drive() -> None:
+            async with _async_openai(gateway) as client:
+                empty = await client.responses.create(model=model, input=text)
+                assert empty.output == [], empty.model_dump_json()
+                assert upstream.calls(handle.scenario_id) == 1
+                rescript(handle, json_response(RESPONSE))
+                real = await client.responses.create(model=model, input=text)
+                assert real.output_text == "scripted", real.model_dump_json()
+                assert upstream.calls(handle.scenario_id) == 2
+                await_entry(store, real.output[0].id)
+                hit = await client.responses.with_raw_response.create(model=model, input=text)
+                served = hit.parse()
+                assert served.output[0].id == real.output[0].id and served.output_text == "scripted", hit.text
+                assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+                assert upstream.calls(handle.scenario_id) == 2
+
+        asyncio.run(drive())
+
+
+def test_messages_empty_content_reaches_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _anthropic(gateway) as client:
+        handle: Final = scripted(scenario, json_response(emptied(MESSAGE, "content")))
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        messages: Final = [{"role": "user", "content": text}]
+        empty: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert empty.content == [], empty.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1)
+        rescript(handle, json_response(MESSAGE))
+        real: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert real.id != empty.id and real.content[0].type == "text", real.model_dump_json()
+        assert real.content[0].text == "scripted"
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2)
+        await_entry(store, real.id)
+        served: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert served.id == real.id and served.content[0].type == "text", served.model_dump_json()
+        assert served.content[0].text == "scripted"
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_chat_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL))
+        assert content_of_first_choice(filled) == "scripted", filled.text
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices")
+        request: Final = chat_body(model, text)
+        refilled: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS
+        )
+        assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}"
+        assert content_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS
+        )
+        assert content_of_first_choice(hit) == "scripted", hit.text
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_text_completion_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(TEXT))
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        short_lived: Final[dict[str, JsonValue]] = {"model": model, "prompt": text, "cache": SHORT_TTL_CACHE_CONTROL}
+        filled: Final = gateway.request("POST", "/v1/completions", short_lived)
+        assert text_of_first_choice(filled) == "scripted", filled.text
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices")
+        request: Final[dict[str, JsonValue]] = {"model": model, "prompt": text}
+        refilled: Final = eventually(
+            lambda: gateway.request("POST", "/v1/completions", request),
+            lambda answer: response_id(answer) != response_id(filled),
+            SETTLE_SECONDS,
+        )
+        assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}"
+        assert text_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: gateway.request("POST", "/v1/completions", request),
+            lambda answer: response_id(answer) == response_id(refilled),
+            SETTLE_SECONDS,
+        )
+        assert text_of_first_choice(hit) == "scripted", hit.text
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_responses_entry_with_empty_output_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _openai(gateway) as client:
+        handle: Final = scripted(scenario, json_response(RESPONSE))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = client.responses.create(model=model, input=text, extra_body={"cache": SHORT_TTL_CACHE_CONTROL})
+        assert filled.output_text == "scripted", filled.model_dump_json()
+        flip_entry_to_empty(store, await_entry(store, filled.output[0].id), "output")
+        refilled: Final = eventually(
+            lambda: client.responses.create(model=model, input=text),
+            lambda answer: not answer.output or answer.output[0].id != filled.output[0].id,
+            SETTLE_SECONDS,
+        )
+        assert refilled.output, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert refilled.output_text == "scripted", refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.output[0].id)
+        hit: Final = eventually(
+            lambda: client.responses.create(model=model, input=text),
+            lambda answer: bool(answer.output) and answer.output[0].id == refilled.output[0].id,
+            SETTLE_SECONDS,
+        )
+        assert hit.output_text == "scripted", hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_messages_entry_with_empty_content_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _anthropic(gateway) as client:
+        handle: Final = scripted(scenario, json_response(MESSAGE))
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        messages: Final = [{"role": "user", "content": text}]
+        filled: Final = client.messages.create(
+            model=model, max_tokens=16, messages=messages, extra_body={"cache": SHORT_TTL_CACHE_CONTROL}
+        )
+        assert filled.content[0].type == "text" and filled.content[0].text == "scripted", filled.model_dump_json()
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, filled.id), "content")
+        refilled: Final = eventually(
+            lambda: client.messages.create(model=model, max_tokens=16, messages=messages),
+            lambda answer: answer.id != filled.id,
+            SETTLE_SECONDS,
+        )
+        assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert refilled.content[0].type == "text" and refilled.content[0].text == "scripted", refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.id)
+        hit: Final = eventually(
+            lambda: client.messages.create(model=model, max_tokens=16, messages=messages),
+            lambda answer: answer.id == refilled.id,
+            SETTLE_SECONDS,
+        )
+        assert hit.content[0].type == "text" and hit.content[0].text == "scripted", hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def _streamed_chat(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The chunk ids, the streamed text and the chunk kinds, after the whole stream was read."""
+    with (
+        _openai(gateway) as client,
+        client.chat.completions.create(
+            model=model, messages=[{"role": "user", "content": text}], stream=True
+        ) as stream,
+    ):
+        chunks: Final = tuple(stream)
+    ids: Final = frozenset(chunk.id for chunk in chunks)
+    streamed: Final = "".join(
+        choice.delta.content or "" for chunk in chunks for choice in chunk.choices
+    )  # comprehension-ok: flattens chunk choices
+    return ids, streamed, tuple(chunk.object for chunk in chunks)
+
+
+def test_streamed_chat_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, CHAT_STREAM_WITHOUT_CHOICES)
+        model: Final = scenario.model(api_base=handle.api_base())
+        empty: Final = _never_replayed(partial(_streamed_chat, gateway, model, text), upstream, handle.scenario_id)
+        assert all(answer.text == "" for answer in empty), empty
+
+
+def _streamed_text(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The chunk ids, the streamed text and the chunk kinds of a text completion, after the whole stream was read."""
+    with _openai(gateway) as client, client.completions.create(model=model, prompt=text, stream=True) as stream:
+        chunks: Final = tuple(stream)
+    ids: Final = frozenset(chunk.id for chunk in chunks)
+    streamed: Final = "".join(
+        choice.text or "" for chunk in chunks for choice in chunk.choices
+    )  # comprehension-ok: flattens chunk choices
+    return ids, streamed, tuple(chunk.object for chunk in chunks)
+
+
+def test_streamed_text_completion_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, TEXT_STREAM_WITHOUT_CHOICES)
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        empty: Final = _never_replayed(partial(_streamed_text, gateway, model, text), upstream, handle.scenario_id)
+        assert all(answer.text == "" for answer in empty), empty
+
+
+def test_streamed_chat_with_content_is_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, CHAT_STREAM)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_chat, gateway, model, text)
+        first_ids, first_text, _ = stream()
+        assert first_text == "streamed response" and len(first_ids) == 1, first_ids
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response", replay
+
+
+def _streamed_response(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The completed response's output item ids, its text and the event kinds, after the whole stream was read."""
+
+    async def read() -> _Answer:
+        async with (
+            _async_openai(gateway) as client,
+            await client.responses.create(model=model, input=text, stream=True) as stream,
+        ):
+            events: Final = [event async for event in stream]
+        completed: Final = tuple(event for event in events if event.type == "response.completed")
+        assert len(completed) == 1, [event.type for event in events]
+        ids: Final = frozenset(item.id for item in completed[0].response.output)
+        return ids, completed[0].response.output_text, tuple(event.type for event in events)
+
+    return asyncio.run(read())
+
+
+def test_streamed_responses_completing_without_output_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, RESPONSE_STREAM_WITHOUT_OUTPUT)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_response, gateway, model, text)
+        empty: Final = _never_replayed(stream, upstream, handle.scenario_id)
+        assert all(answer.ids == frozenset() and answer.text == "" for answer in empty), empty
+        rescript(handle, RESPONSE_STREAM)
+        real_ids, real_text, _ = stream()
+        assert real_text == "streamed response", real_ids
+        assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the empty stream was replayed"
+        await_entry(store, next(iter(real_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids)
+        assert replay.text == "streamed response", replay
+
+
+def test_streamed_responses_with_output_are_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, RESPONSE_STREAM)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_response, gateway, model, text)
+        first_ids, first_text, _ = stream()
+        assert first_text == "streamed response" and len(first_ids) == 1, first_ids
+        assert all(item.startswith(f"msg_{handle.scenario_id}-") for item in first_ids), first_ids
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response", replay
+
+
+def _streamed_message(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The message id, the streamed text and the event kinds, after the whole stream was read."""
+
+    async def read() -> _Answer:
+        async with _async_anthropic(gateway) as client:
+            stream: Final = await client.messages.create(
+                model=model, max_tokens=16, messages=[{"role": "user", "content": text}], stream=True
+            )
+            events: Final = [event async for event in stream]
+        starts: Final = tuple(event for event in events if event.type == "message_start")
+        assert len(starts) == 1, [event.type for event in events]
+        deltas: Final = tuple(event for event in events if event.type == "content_block_delta")
+        streamed: Final = "".join(delta.delta.text for delta in deltas if delta.delta.type == "text_delta")
+        return frozenset({starts[0].message.id}), streamed, tuple(event.type for event in events)
+
+    return asyncio.run(read())
+
+
+def test_streamed_messages_without_content_blocks_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS)
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        stream: Final = partial(_streamed_message, gateway, model, text)
+        empty: Final = _never_replayed(stream, upstream, handle.scenario_id)
+        assert all("content_block_start" not in answer.kinds and answer.text == "" for answer in empty), empty
+        rescript(handle, MESSAGE_STREAM)
+        real_ids, real_text, real_kinds = stream()
+        assert real_text == "streamed response" and "content_block_start" in real_kinds, real_kinds
+        assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the stream without content blocks was replayed"
+        await_entry(store, next(iter(real_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids)
+        assert replay.text == "streamed response" and replay.kinds == real_kinds, replay
+
+
+def test_streamed_messages_with_content_are_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, MESSAGE_STREAM)
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        stream: Final = partial(_streamed_message, gateway, model, text)
+        first_ids, first_text, first_kinds = stream()
+        assert first_text == "streamed response" and "content_block_start" in first_kinds, first_kinds
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response" and replay.kinds == first_kinds, replay
+
+
+def test_embeddings_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(EMBEDDING))
+        model: Final = scenario.model(model="openai/text-embedding-3-small", api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "input": [prompt()]}
+        first: Final = gateway.post("/v1/embeddings", body)
+        assert first["object"] == "list" and first["data"], first
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = eventually(
+            lambda: _served_without_upstream(
+                upstream, handle.scenario_id, lambda: gateway.post("/v1/embeddings", body)
+            ),
+            lambda answer: answer is not None,
+            SETTLE_SECONDS,
+        )
+        assert second is not None and second["data"] == first["data"], second
+
+
+@pytest.mark.parametrize(
+    "choice",
+    (
+        pytest.param(
+            {"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "content_filter"},
+            id="blocked-empty-content",
+        ),
+        pytest.param(
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+                    ],
+                },
+                "finish_reason": "tool_calls",
+            },
+            id="tool-calls-only",
+        ),
+    ),
+)
+def test_chat_answers_with_a_choice_but_no_text_are_still_cached(
+    gateway: Gateway, choice: dict[str, JsonValue]
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response({**CHAT, "choices": [choice]}))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt())
+        first: Final = _chat(gateway, request)
+        assert object_value(choices(first)[0])["finish_reason"] == choice["finish_reason"], first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, response_id(first))
+        second: Final = _chat(gateway, request)
+        assert response_id(second) == response_id(first) and choices(second) == choices(first), second.text
+        assert upstream.calls(handle.scenario_id) == 1
+
+
+def test_no_cache_requests_reach_upstream_every_time(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt(), cache={"no-cache": True})
+        ids: Final = tuple(response_id(_chat(gateway, request)) for _ in range(3))
+        assert len(set(ids)) == 3, ids
+        assert upstream.calls(handle.scenario_id) == 3
+
+
+@pytest.mark.parametrize("status", (500, 429))
+def test_upstream_errors_reach_the_caller_and_are_never_cached(gateway: Gateway, status: int) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(
+            scenario, json_response({"error": {"message": "Controlled provider failure", "type": "api_error"}}, status)
+        )
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt())
+        first: Final = _chat(gateway, request)
+        assert first.status_code == status and "Controlled provider failure" in first.text, first.text
+        after_first: Final = upstream.calls(handle.scenario_id)
+        assert after_first >= 1
+        second: Final = _chat(gateway, request)
+        assert second.status_code == status and "Controlled provider failure" in second.text, second.text
+        assert upstream.calls(handle.scenario_id) > after_first, "the error was served from the cache"
+
+
+def test_cache_delete_on_the_entry_key_makes_the_next_request_reach_upstream(gateway: Gateway) -> None:
+    """``/cache/delete`` clears the Redis entry; a worker's in-memory copy of it serves until the entry's ttl lapses."""
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL))
+        key: Final = await_entry(store, response_id(filled))
+        assert gateway.post("/cache/delete", {"keys": [key]}) == {"status": "success"}
+        assert store.get(key) is None
+        request: Final = chat_body(model, text)
+        refilled: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS
+        )
+        assert content_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS
+        )
+        assert hit.headers.get("x-litellm-cache-key") == key, dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def _shape(body: Mapping[str, JsonValue], field: str, value: object) -> dict[str, JsonValue]:
+    if value is MISSING:
+        return {name: item for name, item in body.items() if name != field}
+    assert isinstance(value, (str, int)) or value is None
+    return {**body, field: value}
+
+
+def _request_for(path: str, model: str, text: str) -> dict[str, JsonValue]:
+    if path == "/v1/responses":
+        return {"model": model, "input": text}
+    if path == "/v1/messages":
+        return message_body(model, text)
+    return chat_body(model, text)
+
+
+@pytest.mark.parametrize(
+    ("path", "body", "litellm_model", "field", "value"),
+    (
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", None, id="chat-choices-null"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", MISSING, id="chat-choices-missing"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", "x", id="chat-choices-string"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", 5, id="chat-choices-int"),
+    ),
+)
+def test_malformed_upstream_answers_are_an_error_and_never_cached(
+    gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(_shape(body, field, value)))
+        control: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(model=litellm_model, api_base=handle.api_base())
+        control_model: Final = scenario.model(api_base=control.api_base())
+        key: Final = scenario.key(models=[model, control_model])
+        request: Final = _request_for(path, model, prompt())
+        first: Final = gateway.request("POST", path, request, key=key)
+        assert first.status_code == 500, first.text
+        assert "error" in first.text.lower(), first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = gateway.request("POST", path, request, key=key)
+        assert second.status_code == 500, second.text
+        assert upstream.calls(handle.scenario_id) == 2, "the malformed answer was served from the cache"
+        unrelated: Final = _chat(gateway, chat_body(control_model, prompt()), key=key)
+        assert content_of_first_choice(unrelated) == "scripted", unrelated.text
+
+
+@pytest.mark.parametrize(
+    ("path", "body", "litellm_model", "field", "value"),
+    (
+        pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", None, id="responses-output-null"),
+        pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", MISSING, id="responses-output-missing"),
+        pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", None, id="messages-content-null"),
+        pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", MISSING, id="messages-content-missing"),
+    ),
+)
+def test_answers_without_the_output_field_reach_upstream_again(
+    gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(_shape(body, field, value)))
+        model: Final = scenario.model(model=litellm_model, api_base=handle.api_base())
+        request: Final = _request_for(path, model, prompt())
+        first: Final = gateway.request("POST", path, request)
+        assert first.status_code == 200, first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = gateway.request("POST", path, request)
+        assert second.status_code == 200, second.text
+        assert upstream.calls(handle.scenario_id) == 2, "the answer without output was served from the cache"
+
+
+def test_per_request_ttl_on_an_empty_answer_does_not_pin_it(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt(), cache={"ttl": 30})
+        empty: Final = _chat(gateway, request)
+        assert choices(empty) == [], empty.text
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(CHAT))
+        real: Final = _chat(gateway, request)
+        assert content_of_first_choice(real) == "scripted", real.text
+        assert upstream.calls(handle.scenario_id) == 2, "the empty answer was pinned for the request's ttl"
+        key: Final = await_entry(store, response_id(real))
+        assert 0 < store.ttl(key) <= 30, store.ttl(key)
+
+
+def test_rerank_results_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(RERANK))
+        model: Final = scenario.model(model="cohere/rerank-v4.0", api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "query": prompt(), "documents": ["first", "second"]}
+        first: Final = gateway.post("/v1/rerank", body)
+        assert string_value(first["id"]).startswith(f"rerank-{handle.scenario_id}"), first
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, string_value(first["id"]))
+        second: Final = gateway.post("/v1/rerank", body)
+        assert second["id"] == first["id"] and second["results"] == first["results"], second
+        assert upstream.calls(handle.scenario_id) == 1
+
+
+def test_transcriptions_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(TRANSCRIPT))
+        model: Final = scenario.model(model="openai/gpt-4o-mini-transcribe", api_base=handle.api_base())
+        audio: Final = (f"{uuid.uuid4().hex}.wav", b"RIFF" + uuid.uuid4().bytes, "audio/wav")
+        first: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio})
+        assert first.status_code == 200, first.text
+        transcript: Final = string_value(JSON_OBJECT.validate_json(first.content)["text"])
+        assert transcript.startswith(f"scripted {handle.scenario_id}-"), first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, transcript)
+        second: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio})
+        assert second.status_code == 200 and JSON_OBJECT.validate_json(second.content)["text"] == transcript, (
+            second.text
+        )
+        assert upstream.calls(handle.scenario_id) == 1

tests/integration/cost_calculation/cost_tracking_case.py

@@ -49,6 +49,7 @@ class CostMapEntry(BaseModel):
     output_cost_per_token: float | None = None
     input_cost_per_token_batches: float | None = None
     output_cost_per_token_batches: float | None = None
+    output_cost_per_image_token_batches: float | None = None
     input_cost_per_token_above_128k_tokens: float | None = None
     output_cost_per_token_above_128k_tokens: float | None = None
     output_vector_size: int | None = None
@@ -583,14 +584,24 @@ def data_errors() -> tuple[str, ...]:
         name for name in {case.name for case in _ALL_CASES} if sum(case.name == name for case in _ALL_CASES) > 1
     )
     input_rates: Final = tuple(
-        (entry.input_cost_per_token, model)
+        (entry.litellm_provider, entry.input_cost_per_token, model)
         for model, entry in COST_MAP.items()
         if entry.mode != "realtime"
     )
+    shared_input_rate_details: Final = tuple(
+        (
+            provider,
+            rate,
+            tuple(
+                model
+                for candidate_provider, value, model in input_rates
+                if candidate_provider == provider and value == rate
+            ),
+        )
+        for provider, rate in frozenset((provider, rate) for provider, rate, _ in input_rates if rate is not None)
+    )
     shared_input_rates: Final = sorted(
-        f"{rate}: {tuple(model for value, model in input_rates if value == rate)}"
-        for rate in {value for value, _ in input_rates if value is not None}
-        if sum(value == rate for value, _ in input_rates) > 1
+        f"{rate}: {models}" for _, rate, models in shared_input_rate_details if len(models) > 1
     )
     recount_mismatches: Final = sorted(
         case.name

tests/integration/cost_calculation/cost_tracking_cases.json

@@ -750,6 +750,25 @@
       "output_cost_per_token": 2.16e-06,
       "litellm_provider": "together_ai",
       "mode": "completion"
+    },
+    "gemini/gemini-nano-banana-2.1": {
+      "input_cost_per_token": 1.5e-06,
+      "input_cost_per_token_batches": 7.5e-07,
+      "litellm_provider": "gemini",
+      "max_input_tokens": 131072,
+      "max_output_tokens": 32768,
+      "max_tokens": 32768,
+      "mode": "image_generation",
+      "output_cost_per_image": 0.0336,
+      "output_cost_per_image_token": 3e-05,
+      "output_cost_per_token": 7.5e-06,
+      "output_cost_per_token_batches": 3.75e-06,
+      "search_context_cost_per_query": {
+        "search_context_size_high": 0.014,
+        "search_context_size_low": 0.014,
+        "search_context_size_medium": 0.014
+      },
+      "web_search_billing_unit": "per_query"
     }
   },
   "cases": [
@@ -31279,6 +31298,810 @@
         "min_completion_tokens": 9,
         "max_completion_tokens": 30
       }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-input_text",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "8e41cbb0a701 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer 8e41cbb0a701"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 2000,
+            "candidatesTokenCount": 100,
+            "totalTokenCount": 2100,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 2000
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.00375,
+        "input_cost": 0.003,
+        "output_cost": 0.00075,
+        "prompt_tokens": 2000,
+        "completion_tokens": 100
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-image_input",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "3f6c2d91a40e summarize the attached material in one line and name the city weather"
+              },
+              {
+                "type": "image_url",
+                "image_url": {
+                  "url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+yVWQAAAAASUVORK5CYII=",
+                  "detail": "high"
+                }
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer 3f6c2d91a40e"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 1680,
+            "candidatesTokenCount": 100,
+            "totalTokenCount": 1780,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 560
+              },
+              {
+                "modality": "IMAGE",
+                "tokenCount": 1120
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.00327,
+        "input_cost": 0.00252,
+        "output_cost": 0.00075,
+        "prompt_tokens": 1680,
+        "completion_tokens": 100
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-output_text",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "d05a38c7e219 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer d05a38c7e219"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 4000,
+            "totalTokenCount": 4100,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 4000
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.03015,
+        "input_cost": 0.00015,
+        "output_cost": 0.03,
+        "prompt_tokens": 100,
+        "completion_tokens": 4000
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-reasoning",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "c71e4f0092ba summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "reasoning_effort": "medium",
+        "allowed_openai_params": [
+          "reasoning_effort"
+        ]
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer c71e4f0092ba"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 200,
+            "thoughtsTokenCount": 1800,
+            "totalTokenCount": 2100,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 200
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.01515,
+        "input_cost": 0.00015,
+        "output_cost": 0.015,
+        "prompt_tokens": 100,
+        "completion_tokens": 2000
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-image_output_1k",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "6a9f2c13b847 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "inlineData": {
+                      "mimeType": "image/png",
+                      "data": "aGVsbG8="
+                    }
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 1120,
+            "totalTokenCount": 1220,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "IMAGE",
+                "tokenCount": 1120
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.03375,
+        "input_cost": 0.00015,
+        "output_cost": 0.0336,
+        "prompt_tokens": 100,
+        "completion_tokens": 1120
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-image_output_2k",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "24df708bc351 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "inlineData": {
+                      "mimeType": "image/png",
+                      "data": "aGVsbG8="
+                    }
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 1680,
+            "totalTokenCount": 1780,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "IMAGE",
+                "tokenCount": 1680
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.05055,
+        "input_cost": 0.00015,
+        "output_cost": 0.0504,
+        "prompt_tokens": 100,
+        "completion_tokens": 1680
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-image_output_4k",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "b8a315e4706c summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "inlineData": {
+                      "mimeType": "image/png",
+                      "data": "aGVsbG8="
+                    }
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 2520,
+            "totalTokenCount": 2620,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "IMAGE",
+                "tokenCount": 2520
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.07575,
+        "input_cost": 0.00015,
+        "output_cost": 0.0756,
+        "prompt_tokens": 100,
+        "completion_tokens": 2520
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-mixed_text_image_output",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "91ce62a50d3b summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "allowed_openai_params": []
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer 91ce62a50d3b"
+                  },
+                  {
+                    "inlineData": {
+                      "mimeType": "image/png",
+                      "data": "aGVsbG8="
+                    }
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 1220,
+            "thoughtsTokenCount": 300,
+            "totalTokenCount": 1620,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              },
+              {
+                "modality": "IMAGE",
+                "tokenCount": 1120
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.03675,
+        "input_cost": 0.00015,
+        "output_cost": 0.0366,
+        "prompt_tokens": 100,
+        "completion_tokens": 1520
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-web_search_single_query",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "f407b19d8e26 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "tools": [
+          {
+            "googleSearch": {}
+          }
+        ],
+        "allowed_openai_params": [
+          "web_search_options"
+        ]
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer f407b19d8e26"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0,
+              "groundingMetadata": {
+                "webSearchQueries": [
+                  "query 0"
+                ]
+              }
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 100,
+            "totalTokenCount": 200,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.0149,
+        "input_cost": 0.00015,
+        "output_cost": 0.00075,
+        "tool_usage_cost": 0.014,
+        "prompt_tokens": 100,
+        "completion_tokens": 100
+      }
+    },
+    {
+      "name": "gemini-nano-banana-2.1-web_search_multiple_queries",
+      "covers": "quota_management.spend_tracking.cost_matrix.logs_cost",
+      "model": "gemini/gemini-nano-banana-2.1",
+      "request": {
+        "model": "$MODEL",
+        "messages": [
+          {
+            "role": "system",
+            "content": [
+              {
+                "type": "text",
+                "text": "You are a deterministic pricing-harness assistant. Keep answers to a single short line."
+              }
+            ]
+          },
+          {
+            "role": "user",
+            "content": [
+              {
+                "type": "text",
+                "text": "a36d5f80c297 summarize the attached material in one line and name the city weather"
+              }
+            ]
+          }
+        ],
+        "stream": false,
+        "tools": [
+          {
+            "googleSearch": {}
+          }
+        ],
+        "allowed_openai_params": [
+          "web_search_options"
+        ]
+      },
+      "response": {
+        "content_type": "application/json",
+        "body": {
+          "candidates": [
+            {
+              "content": {
+                "parts": [
+                  {
+                    "text": "scripted answer a36d5f80c297"
+                  }
+                ],
+                "role": "model"
+              },
+              "finishReason": "STOP",
+              "index": 0,
+              "groundingMetadata": {
+                "webSearchQueries": [
+                  "query 0",
+                  "query 1",
+                  "query 2",
+                  "query 3"
+                ]
+              }
+            }
+          ],
+          "usageMetadata": {
+            "promptTokenCount": 100,
+            "candidatesTokenCount": 100,
+            "totalTokenCount": 200,
+            "promptTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ],
+            "candidatesTokensDetails": [
+              {
+                "modality": "TEXT",
+                "tokenCount": 100
+              }
+            ]
+          },
+          "modelVersion": "gemini-nano-banana-2.1"
+        }
+      },
+      "expected": {
+        "spend": 0.0569,
+        "input_cost": 0.00015,
+        "output_cost": 0.00075,
+        "tool_usage_cost": 0.056,
+        "prompt_tokens": 100,
+        "completion_tokens": 100
+      }
     }
   ],
   "batch_cases": [

tests/integration/run.py

@@ -13,7 +13,7 @@ from typing import Final
 GROUPS: Final = MappingProxyType(
     {
         "management": ("management", "authorization", "configuration"),
-        "accounting": ("pricing", "spend"),
+        "accounting": ("pricing", "spend", "caching"),
         "database": ("database",),
         "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation"),
         "extensions": ("observability", "compatibility"),

tests/integration/sdk/test_response_cache_sync_sdk.py

@@ -0,0 +1,125 @@
+"""The sync SDK paths of the response cache, which the proxy never reaches.
+
+``litellm.completion`` and ``litellm.text_completion`` read and write the Redis cache in process. An upstream
+answer with no choices is never stored, a stored entry whose choices were emptied is a miss, and the real answer
+the refill brings back is the one served afterwards.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.caching.caching import Cache
+from litellm.types.caching import LiteLLMCacheType
+from litellm.types.utils import ModelResponse, TextCompletionResponse
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    PROVIDER_KEY,
+    TEXT,
+    TEXT_MODEL,
+    Upstream,
+    await_entry,
+    emptied,
+    flip_entry_to_empty,
+    json_response,
+    prompt,
+    redis_store,
+    rescript,
+    scripted_in_process,
+)
+
+
+@pytest.fixture
+def sdk_cache(monkeypatch: pytest.MonkeyPatch) -> Iterator[Cache]:
+    cache: Final = Cache(type=LiteLLMCacheType.REDIS, host=os.environ["REDIS_HOST"], port=os.environ["REDIS_PORT"])
+    monkeypatch.setattr(litellm, "cache", cache)
+    yield cache
+    monkeypatch.setattr(litellm, "cache", None)
+
+
+def _completion(api_base: str, text: str) -> ModelResponse:
+    response: Final = litellm.completion(
+        model=CHAT_MODEL, messages=[{"role": "user", "content": text}], api_base=api_base, api_key=PROVIDER_KEY
+    )
+    assert isinstance(response, ModelResponse), type(response)
+    return response
+
+
+def _text_completion(api_base: str, text: str) -> TextCompletionResponse:
+    response: Final = litellm.text_completion(model=TEXT_MODEL, prompt=text, api_base=api_base, api_key=PROVIDER_KEY)
+    assert isinstance(response, TextCompletionResponse), type(response)
+    return response
+
+
+def _content(response: ModelResponse) -> object:
+    assert len(response.choices) == 1, response.model_dump_json()
+    return response.choices[0].model_dump()["message"]["content"]
+
+
+def _text(response: TextCompletionResponse) -> object:
+    assert len(response.choices) == 1, response.model_dump_json()
+    return response.choices[0].model_dump()["text"]
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_completion_with_empty_choices_reaches_upstream_again_and_the_real_answer_is_cached() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(emptied(CHAT, "choices"))) as handle:
+        empty: Final = _completion(handle.api_base(), text)
+        assert empty.choices == [], empty.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(CHAT))
+        refilled: Final = _completion(handle.api_base(), text)
+        assert _content(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}"
+        assert refilled.id != empty.id
+        assert upstream.calls(handle.scenario_id) == 2
+        with redis_store() as store:
+            await_entry(store, refilled.id)
+        hit: Final = _completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert _content(hit) == "scripted"
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_completion_treats_a_stale_entry_with_empty_choices_as_a_miss() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(CHAT)) as handle, redis_store() as store:
+        first: Final = _completion(handle.api_base(), text)
+        assert _content(first) == "scripted", first.model_dump_json()
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, first.id), "choices")
+        refilled: Final = _completion(handle.api_base(), text)
+        assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert _content(refilled) == "scripted" and refilled.id != first.id, refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.id)
+        hit: Final = _completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_text_completion_with_empty_choices_reaches_upstream_again() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(emptied(TEXT, "choices"))) as handle:
+        empty: Final = _text_completion(handle.api_base(), text)
+        assert empty.choices == [], empty.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(TEXT))
+        refilled: Final = _text_completion(handle.api_base(), text)
+        assert _text(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}"
+        assert upstream.calls(handle.scenario_id) == 2
+        with redis_store() as store:
+            await_entry(store, refilled.id)
+        hit: Final = _text_completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2

tests/integration/sdk/test_vertex_gemini_image_batch_cost_sdk.py

@@ -0,0 +1,446 @@
+from __future__ import annotations
+
+import json
+import os
+import socket
+import subprocess
+import sys
+import textwrap
+import threading
+from collections.abc import Generator
+from contextlib import contextmanager
+from dataclasses import dataclass
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+from pathlib import Path
+from typing import Final
+from urllib.parse import urlsplit
+
+import pytest
+from integration._support.tls import server_context, write_self_signed_cert
+from integration._support.vertex import service_account_json
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue, TypeAdapter
+
+pytestmark: Final = pytest.mark.timeout(180)
+
+_PROJECT: Final = "scripted-gemini-batch-project"
+_MODEL: Final = "gemini-nano-banana-2.1"
+_BATCH_ID: Final = "scripted-batch"
+_CALLBACK_PREFIX: Final = "CALLBACK:"
+_REGISTRY_PREFIX: Final = "REGISTRY:"
+_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
+_IMAGE_PART: Final = {
+    "inlineData": {
+        "mimeType": "image/png",
+        "data": "aGVsbG8=",
+    }
+}
+
+
+def _pipe(source: socket.socket, destination: socket.socket) -> None:
+    try:
+        for payload in iter(lambda: source.recv(65536), b""):
+            destination.sendall(payload)
+    except OSError:
+        return
+    finally:
+        try:
+            destination.shutdown(socket.SHUT_WR)
+        except OSError:
+            pass
+
+
+@contextmanager
+def _redirect_https_host(host: str, port: int, *, to_port: int) -> Generator[str, None, None]:
+    """Yields an HTTPS_PROXY url that sends host:port to 127.0.0.1:to_port and refuses anything else."""
+    authority: Final = f"{host}:{port}"
+
+    class _RedirectHandler(BaseHTTPRequestHandler):
+        protocol_version = "HTTP/1.1"
+        timeout = 30
+
+        def do_CONNECT(self) -> None:
+            if self.path != authority:
+                self.send_error(403)
+                return
+
+            self.send_response(200, "Connection Established")
+            self.end_headers()
+            try:
+                with socket.create_connection(("127.0.0.1", to_port), timeout=10) as upstream:
+                    client_to_upstream: Final = threading.Thread(
+                        target=_pipe, args=(self.connection, upstream), daemon=True
+                    )
+                    upstream_to_client: Final = threading.Thread(
+                        target=_pipe, args=(upstream, self.connection), daemon=True
+                    )
+                    client_to_upstream.start()
+                    upstream_to_client.start()
+                    client_to_upstream.join(timeout=30)
+                    upstream_to_client.join(timeout=30)
+            except OSError:
+                return
+
+        def log_message(self, format: str, *args: object) -> None:
+            pass
+
+    class _RedirectProxy(ThreadingHTTPServer):
+        daemon_threads = True
+        request_queue_size = 16
+
+    with _RedirectProxy(("127.0.0.1", 0), _RedirectHandler) as server:
+        thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05})
+        thread.start()
+        try:
+            yield f"http://127.0.0.1:{server.server_port}"
+        finally:
+            server.shutdown()
+            thread.join(timeout=6)
+            assert not thread.is_alive(), "CONNECT tunnel server survived cleanup"
+            server.server_close()
+
+
+@dataclass(frozen=True, slots=True)
+class _BatchScenario:
+    name: str
+    prompt_tokens: int
+    prompt_details: tuple[tuple[str, int], ...]
+    candidate_tokens: int
+    candidate_details: tuple[tuple[str, int], ...]
+    thoughts_tokens: int
+    expected_prompt_cost: float
+    expected_completion_cost: float
+    use_explicit_image_batch_rate_override: bool = False
+
+
+_SCENARIOS: Final = (
+    _BatchScenario(
+        name="batch_input_text_and_image",
+        prompt_tokens=1680,
+        prompt_details=(("TEXT", 560), ("IMAGE", 1120)),
+        candidate_tokens=100,
+        candidate_details=(("TEXT", 100),),
+        thoughts_tokens=0,
+        expected_prompt_cost=0.00126,
+        expected_completion_cost=0.000375,
+    ),
+    _BatchScenario(
+        name="batch_text_and_thinking_output",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=400,
+        candidate_details=(("TEXT", 400),),
+        thoughts_tokens=600,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.00375,
+    ),
+    _BatchScenario(
+        name="batch_image_output_1k",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=1120,
+        candidate_details=(("IMAGE", 1120),),
+        thoughts_tokens=0,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.0168,
+    ),
+    _BatchScenario(
+        name="batch_image_output_2k",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=1680,
+        candidate_details=(("IMAGE", 1680),),
+        thoughts_tokens=0,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.0252,
+    ),
+    _BatchScenario(
+        name="batch_image_output_4k",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=3780,
+        candidate_details=(("IMAGE", 3780),),
+        thoughts_tokens=0,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.0567,
+    ),
+    _BatchScenario(
+        name="batch_mixed_text_image_output",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=1220,
+        candidate_details=(("TEXT", 100), ("IMAGE", 1120)),
+        thoughts_tokens=300,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.0183,
+    ),
+    _BatchScenario(
+        name="batch_image_rate_uses_explicit_override",
+        prompt_tokens=100,
+        prompt_details=(("TEXT", 100),),
+        candidate_tokens=1120,
+        candidate_details=(("IMAGE", 1120),),
+        thoughts_tokens=0,
+        expected_prompt_cost=0.000075,
+        expected_completion_cost=0.0224,
+        use_explicit_image_batch_rate_override=True,
+    ),
+)
+
+_SDK_SCRIPT: Final = textwrap.dedent(
+    """
+    import asyncio, json, os
+    from typing import Final
+
+    import litellm
+    from litellm.integrations.custom_logger import CustomLogger
+    from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+    from litellm.types.utils import ModelInfo
+
+    model: Final = "gemini-nano-banana-2.1"
+
+    class CaptureLogger(CustomLogger):
+        async def async_log_success_event(
+            self,
+            kwargs: dict[str, object],
+            response_obj: object,
+            start_time: object,
+            end_time: object,
+        ) -> None:
+            standard: Final = kwargs.get("standard_logging_object")
+            hidden: Final = getattr(response_obj, "_hidden_params", None)
+            if not isinstance(standard, dict):
+                raise RuntimeError("success callback omitted standard_logging_object")
+            if standard.get("call_type") != "aretrieve_batch":
+                return
+            print(
+                "CALLBACK:"
+                + json.dumps(
+                    {
+                        "response_cost": standard.get("response_cost"),
+                        "cost_breakdown": standard.get("cost_breakdown"),
+                        "batch_response_cost": hidden.get("response_cost")
+                        if isinstance(hidden, dict)
+                        else None,
+                    }
+                ),
+                flush=True,
+            )
+
+    async def main() -> None:
+        has_explicit_image_batch_rate_override: Final = (
+            os.environ["USE_EXPLICIT_IMAGE_BATCH_RATE_OVERRIDE"] == "1"
+        )
+        if has_explicit_image_batch_rate_override:
+            pricing: Final[ModelInfo] = {
+                **litellm.model_cost["vertex_ai/gemini-nano-banana-2.1"],
+                "output_cost_per_image_token_batches": 2e-5,
+            }
+            litellm.register_model({f"vertex_ai/{model}": pricing}, persist_across_reloads=False)
+        litellm.user_url_validation = False
+        litellm.disable_vertex_batch_output_transformation = os.environ["TRANSFORM_OUTPUT"] != "1"
+        info: Final = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai")
+        assert info["key"] == f"vertex_ai/{model}", info
+        assert info["litellm_provider"] == "vertex_ai-language-models", info
+        print(
+            "REGISTRY:"
+            + json.dumps({"key": info["key"], "provider": info["litellm_provider"]}),
+            flush=True,
+        )
+        logger: Final = CaptureLogger()
+        litellm.logging_callback_manager.add_litellm_async_success_callback(logger)
+        await litellm.aretrieve_batch(
+            "scripted-batch",
+            custom_llm_provider="vertex_ai",
+            model=model,
+            api_base=os.environ["VERTEX_API_BASE"],
+            vertex_project=os.environ["VERTEX_PROJECT"],
+            vertex_location="us-central1",
+            vertex_credentials=os.environ["VERTEX_CREDENTIALS"],
+            gcs_bucket_name="scripted-bucket",
+            num_retries=0,
+        )
+        await asyncio.sleep(0)
+        await GLOBAL_LOGGING_WORKER.flush()
+
+    asyncio.run(main())
+    """
+)
+
+
+def _prediction_jsonl(scenario: _BatchScenario) -> bytes:
+    prompt_details: Final = tuple(
+        {"modality": modality, "tokenCount": count} for modality, count in scenario.prompt_details
+    )
+    candidate_details: Final = tuple(
+        {"modality": modality, "tokenCount": count} for modality, count in scenario.candidate_details
+    )
+    candidate_parts: Final = tuple(
+        {"text": "scripted batch output"} if modality == "TEXT" else _IMAGE_PART
+        for modality, _ in scenario.candidate_details
+    )
+    usage_metadata: Final[dict[str, JsonValue]] = {
+        "promptTokenCount": scenario.prompt_tokens,
+        "candidatesTokenCount": scenario.candidate_tokens,
+        "totalTokenCount": scenario.prompt_tokens + scenario.candidate_tokens + scenario.thoughts_tokens,
+        "promptTokensDetails": prompt_details,
+        "candidatesTokensDetails": candidate_details,
+        **({"thoughtsTokenCount": scenario.thoughts_tokens} if scenario.thoughts_tokens else {}),
+    }
+    row: Final[dict[str, JsonValue]] = {
+        "request": {
+            "contents": [
+                {
+                    "role": "user",
+                    "parts": [{"text": "Scripted Vertex Gemini batch request"}],
+                }
+            ]
+        },
+        "status": "",
+        "response": {
+            "candidates": [
+                {
+                    "content": {"parts": candidate_parts, "role": "model"},
+                    "finishReason": "STOP",
+                    "index": 0,
+                }
+            ],
+            "usageMetadata": usage_metadata,
+            "modelVersion": _MODEL,
+        },
+    }
+    return json.dumps(row, separators=(",", ":")).encode("utf-8") + b"\n"
+
+
+def _api_reply(request: Request) -> Reply:
+    if request.method == "POST" and request.target == "/_oauth/token":
+        return Reply(body=b'{"access_token":"scripted-token","expires_in":3600,"token_type":"Bearer"}')
+    if request.method == "GET" and request.target.endswith(f"/batchPredictionJobs/{_BATCH_ID}"):
+        return Reply(
+            body=json.dumps(
+                {
+                    "name": (f"projects/{_PROJECT}/locations/us-central1/batchPredictionJobs/{_BATCH_ID}"),
+                    "state": "JOB_STATE_SUCCEEDED",
+                    "outputInfo": {
+                        "gcsOutputDirectory": (
+                            "gs://scripted-bucket/litellm-vertex-files/"
+                            "publishers/google/models/gemini-nano-banana-2.1/scripted-prefix"
+                        )
+                    },
+                    "createTime": "2026-10-06T00:00:00.000Z",
+                }
+            ).encode("utf-8")
+        )
+    return Reply(status=404, body=b"{}")
+
+
+def _gcs_reply(content: bytes):
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target.endswith("predictions.jsonl?alt=media"):
+            return Reply(body=content, content_type="application/jsonl")
+        return Reply(status=404, body=b"{}")
+
+    return respond
+
+
+def _subprocess_environment(
+    api: Wire,
+    certificate: Path,
+    credentials: str,
+    proxy_url: str,
+    scenario: _BatchScenario,
+    transform_output: bool,
+) -> dict[str, str]:
+    repo_root: Final = Path(__file__).resolve().parents[3]
+    python_path: Final = os.pathsep.join(path for path in (str(repo_root), os.environ.get("PYTHONPATH")) if path)
+    return {
+        **os.environ,
+        "HTTPS_PROXY": proxy_url,
+        "https_proxy": proxy_url,
+        "HTTP_PROXY": "",
+        "http_proxy": "",
+        "SSL_CERT_FILE": str(certificate),
+        "NO_PROXY": "127.0.0.1,localhost",
+        "no_proxy": "127.0.0.1,localhost",
+        "VERTEX_API_BASE": api.url,
+        "VERTEX_CREDENTIALS": credentials,
+        "VERTEX_PROJECT": _PROJECT,
+        "USE_EXPLICIT_IMAGE_BATCH_RATE_OVERRIDE": (
+            "1" if scenario.use_explicit_image_batch_rate_override else "0"
+        ),
+        "TRANSFORM_OUTPUT": "1" if transform_output else "0",
+        "LITELLM_LOCAL_MODEL_COST_MAP": "True",
+        "PYTHONPATH": python_path,
+    }
+
+
+@pytest.mark.parametrize("transform_output", (False, True), ids=("untransformed", "transformed"))
+@pytest.mark.parametrize("scenario", _SCENARIOS, ids=tuple(case.name for case in _SCENARIOS))
+def test_aretrieve_batch_costs_native_gemini_image_tokens(
+    scenario: _BatchScenario, transform_output: bool, tmp_path: Path
+) -> None:
+    certificate, key = write_self_signed_cert(tmp_path, names=("storage.googleapis.com",))
+    tls: Final = server_context(certificate, key)
+    row: Final = _prediction_jsonl(scenario)
+
+    with wire_server(_api_reply) as api:
+        credentials: Final = service_account_json(_PROJECT, api.url)
+        with wire_server(_gcs_reply(row), tls=tls) as gcs:
+            gcs_port: Final = urlsplit(gcs.url).port
+            assert gcs_port is not None
+            with _redirect_https_host("storage.googleapis.com", 443, to_port=gcs_port) as proxy_url:
+                environment: Final = _subprocess_environment(
+                    api, certificate, credentials, proxy_url, scenario, transform_output
+                )
+                outcome: Final = subprocess.run(
+                    [sys.executable, "-P", "-c", _SDK_SCRIPT],
+                    env=environment,
+                    capture_output=True,
+                    text=True,
+                    timeout=60,
+                    check=False,
+                )
+
+                assert outcome.returncode == 0, (outcome.stdout, outcome.stderr)
+                registry_events: Final = tuple(
+                    _JSON_OBJECT.validate_json(line[len(_REGISTRY_PREFIX) :])
+                    for line in outcome.stdout.splitlines()
+                    if line.startswith(_REGISTRY_PREFIX)
+                )
+                callback_events: Final = tuple(
+                    _JSON_OBJECT.validate_json(line[len(_CALLBACK_PREFIX) :])
+                    for line in outcome.stdout.splitlines()
+                    if line.startswith(_CALLBACK_PREFIX)
+                )
+                expected_registry_key: Final = f"vertex_ai/{_MODEL}"
+                assert registry_events == (
+                    {"key": expected_registry_key, "provider": "vertex_ai-language-models"},
+                ), outcome.stdout
+                assert len(callback_events) == 1, (outcome.stdout, outcome.stderr)
+                event: Final = callback_events[0]
+                assert event["response_cost"] == pytest.approx(
+                    scenario.expected_prompt_cost + scenario.expected_completion_cost
+                ), event
+                assert event["batch_response_cost"] == pytest.approx(
+                    scenario.expected_prompt_cost + scenario.expected_completion_cost
+                ), event
+                breakdown: Final = event["cost_breakdown"]
+                assert isinstance(breakdown, dict), event
+                assert breakdown["input_cost"] == pytest.approx(scenario.expected_prompt_cost), event
+                assert breakdown["output_cost"] == pytest.approx(scenario.expected_completion_cost), event
+                assert breakdown["total_cost"] == pytest.approx(
+                    scenario.expected_prompt_cost + scenario.expected_completion_cost
+                ), event
+
+                gcs_requests: Final = gcs.drain()
+                assert any(
+                    request.method == "GET" and request.target.endswith("predictions.jsonl?alt=media")
+                    for request in gcs_requests
+                ), tuple((request.method, request.target) for request in gcs_requests)
+                api_requests: Final = api.drain()
+                assert any(
+                    request.method == "GET"
+                    and request.target.endswith(f"/batchPredictionJobs/{_BATCH_ID}")
+                    and request.headers.get("authorization") == "Bearer scripted-token"
+                    for request in api_requests
+                ), tuple((request.method, request.target) for request in api_requests)

tests/integration/spend/test_responses_bridge_spend_once.py

@@ -0,0 +1,111 @@
+import json
+from base64 import b64decode
+from collections.abc import Callable
+from hashlib import sha256
+from typing import Final
+from uuid import uuid4
+
+import pytest
+from integration._support.client import Gateway, eventually, object_value
+from integration._support.database import read_rows
+from integration._support.wire import Reply, Request, wire_server
+from pydantic import JsonValue
+
+PROMPT_TOKENS: Final = 30
+COMPLETION_TOKENS: Final = 5
+INPUT_RATE: Final = 0.001
+OUTPUT_RATE: Final = 0.002
+EXPECTED_SPEND: Final = PROMPT_TOKENS * INPUT_RATE + COMPLETION_TOKENS * OUTPUT_RATE
+
+
+def _responses_reply(prompt: str, response_id: str) -> Callable[[Request], Reply]:
+    def respond(request: Request) -> Reply:
+        if request.method == "GET" and request.target == "/v1/models":
+            return Reply(
+                body=json.dumps({"object": "list", "data": [{"id": "gpt-6.1-sol", "object": "model"}]}).encode()
+            )
+        assert request.method == "POST", request.method
+        assert request.target == "/v1/responses", request.target
+        body: Final = object_value(json.loads(request.body))
+        encoded_input: Final = json.dumps(body["input"])
+        assert prompt in encoded_input, encoded_input
+        assert body.get("stream") is not True, body
+        response: Final = {
+            "id": response_id,
+            "object": "response",
+            "created_at": 1700000000,
+            "status": "completed",
+            "model": body["model"],
+            "output": [
+                {
+                    "type": "message",
+                    "id": f"msg_{uuid4().hex}",
+                    "status": "completed",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": "ok", "annotations": []}],
+                }
+            ],
+            "usage": {
+                "input_tokens": PROMPT_TOKENS,
+                "output_tokens": COMPLETION_TOKENS,
+                "total_tokens": PROMPT_TOKENS + COMPLETION_TOKENS,
+            },
+        }
+        return Reply(body=json.dumps(response).encode())
+
+    return respond
+
+
+def _rows_for_key(key: str) -> list[dict[str, JsonValue]]:
+    return read_rows(
+        'SELECT litellm_call_id, request_id, spend FROM "LiteLLM_SpendLogs" WHERE api_key=%s',
+        (sha256(key.encode()).hexdigest(),),
+    )
+
+
+@pytest.mark.timeout(120)
+def test_bridged_chat_completion_above_log_offload_threshold_logs_and_charges_once(gateway: Gateway) -> None:
+    prompt_prefix: Final = f"spend once {uuid4().hex[:8]} "
+    long_prompt: Final = (prompt_prefix + "lorem ipsum " * 30_000)[:300_000]
+    upstream_id: Final = f"resp_{uuid4().hex}"
+    with (
+        wire_server(_responses_reply(long_prompt, upstream_id)) as wire,
+        gateway.scenario() as scenario,
+    ):
+        model: Final = scenario.model(
+            model="openai/responses/gpt-6.1-sol",
+            api_base=f"{wire.url}/v1",
+            input_cost_per_token=INPUT_RATE,
+            output_cost_per_token=OUTPUT_RATE,
+        )
+        key: Final = scenario.key(models=[model])
+        digest: Final = sha256(key.encode()).hexdigest()
+
+        response: Final = gateway.request(
+            "POST",
+            "/v1/chat/completions",
+            {"model": model, "messages": [{"role": "user", "content": long_prompt}], "stream": False},
+            key=key,
+        )
+        assert response.status_code == 200, response.text
+        assert "ok" in response.text, response.text
+        assert response.json()["id"] == upstream_id, response.text
+        call_id: Final = response.headers["x-litellm-call-id"]
+
+        rows: Final = eventually(
+            lambda: _rows_for_key(key),
+            lambda values: any(row["litellm_call_id"] == call_id for row in values),
+            seconds=70,
+        )
+        call_rows: Final = [row for row in rows if row["litellm_call_id"] == call_id]
+        assert len(call_rows) == 1, rows
+        assert float(str(call_rows[0]["spend"])) == pytest.approx(EXPECTED_SPEND), rows
+        logged_id: Final = b64decode(str(call_rows[0]["request_id"]).removeprefix("resp_")).decode()
+        assert logged_id.startswith("litellm:") and f";response_id:{upstream_id}" in logged_id, rows
+
+        key_spend: Final = eventually(
+            lambda: read_rows('SELECT spend FROM "LiteLLM_VerificationToken" WHERE token=%s', (digest,)),
+            lambda values: len(values) == 1 and float(str(values[0]["spend"])) >= EXPECTED_SPEND,
+            seconds=70,
+        )
+        assert float(str(key_spend[0]["spend"])) == pytest.approx(EXPECTED_SPEND), key_spend

tests/unit/caching/test_caching_handler.py

@@ -23,17 +23,23 @@ from litellm.caching.caching_handler import (
     _is_chat_completion_cached_dict,
     _should_defer_streaming_cache_hit_callbacks,
 )
+from litellm.caching import DualCache, InMemoryCache
 from litellm.caching.caching import LiteLLMCacheType
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY
 from litellm.types.utils import CallTypes
 from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
+    Delta,
     ModelResponse,
+    ModelResponseStream,
+    StreamingChoices,
     EmbeddingResponse,
     TextCompletionResponse,
     TranscriptionResponse,
     Embedding,
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
+from collections.abc import Awaitable, Callable
 from datetime import timedelta, datetime
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -2313,3 +2319,355 @@ async def test_response_cache_lookup_and_write_declare_the_llm_response_target(m
 
     assert seen == {"get": "llm_response", "set": "llm_response"}
     assert current_service_target() is None
+
+
+def _completion_logging_obj(call_type: str) -> LiteLLMLogging:
+    return LiteLLMLogging(
+        litellm_call_id=str(uuid.uuid4()),
+        call_type=call_type,
+        model="gpt-3.5-turbo",
+        messages=[],
+        function_id=str(uuid.uuid4()),
+        stream=False,
+        start_time=_FIXED_START,
+    )
+
+
+_FIXED_START = datetime(2026, 1, 1)
+
+
+def _unique_messages() -> list[dict[str, str]]:
+    return [{"role": "user", "content": f"no choices {uuid.uuid4()}"}]
+
+
+async def aanthropic_messages(**kwargs: object) -> None:
+    return None
+
+
+def _responses_api_response_without_output() -> ResponsesAPIResponse:
+    return ResponsesAPIResponse(
+        id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response", output=[]
+    )
+
+
+def _anthropic_message_without_content() -> dict[str, object]:
+    return {
+        "id": f"msg_{uuid.uuid4()}",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-sonnet-5",
+        "content": [],
+    }
+
+
+def _responses_api_response_missing_output() -> ResponsesAPIResponse:
+    return ResponsesAPIResponse.model_construct(
+        id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response"
+    )
+
+
+def _anthropic_message_with_null_content() -> dict[str, object]:
+    return {
+        "id": f"msg_{uuid.uuid4()}",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-sonnet-5",
+        "content": None,
+    }
+
+
+def _anthropic_message_missing_content() -> dict[str, object]:
+    return {"id": f"msg_{uuid.uuid4()}", "type": "message", "role": "assistant", "model": "claude-sonnet-5"}
+
+
+def _anthropic_stream_without_content_blocks() -> dict[str, object]:
+    return {
+        CACHED_STREAM_EVENTS_KEY: [
+            'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_1", "content": []}}\n\n',
+            'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}}\n\n',
+            'event: message_stop\ndata: {"type": "message_stop"}\n\n',
+        ]
+    }
+
+
+EmptyResult = ModelResponse | TextCompletionResponse | ResponsesAPIResponse | dict[str, object]
+EmptyResponseCase = tuple[EmptyResult, Callable[..., Awaitable[object]], str, dict[str, object]]
+
+
+def _empty_response_cases() -> list[EmptyResponseCase]:
+    return [
+        (litellm.ModelResponse(choices=[]), litellm.acompletion, CallTypes.acompletion.value, {"messages": _unique_messages()}),
+        (
+            litellm.TextCompletionResponse(choices=[]),
+            litellm.atext_completion,
+            CallTypes.atext_completion.value,
+            {"prompt": str(uuid.uuid4())},
+        ),
+        (
+            _responses_api_response_without_output(),
+            aresponses,
+            CallTypes.aresponses.value,
+            {"input": str(uuid.uuid4())},
+        ),
+        (
+            _responses_api_response_missing_output(),
+            aresponses,
+            CallTypes.aresponses.value,
+            {"input": str(uuid.uuid4())},
+        ),
+        (
+            _anthropic_message_without_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_message_with_null_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_message_missing_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_stream_without_content_blocks(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16, "stream": True},
+        ),
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases())
+async def test_async_set_cache_skips_response_without_output(
+    empty_result: EmptyResult,
+    original_function: Callable[..., Awaitable[object]],
+    call_type: str,
+    kwargs: dict[str, object],
+):
+    setup_cache()
+    handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START)
+
+    await handler.async_set_cache(result=empty_result, original_function=original_function, kwargs=kwargs)
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert await litellm.cache.async_get_cache(**kwargs) is None
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=original_function,
+        logging_obj=_completion_logging_obj(call_type),
+        start_time=_FIXED_START,
+        call_type=call_type,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases())
+async def test_async_get_cache_treats_stored_response_without_output_as_miss(
+    empty_result: EmptyResult,
+    original_function: Callable[..., Awaitable[object]],
+    call_type: str,
+    kwargs: dict[str, object],
+):
+    setup_cache()
+    stored = empty_result.model_dump_json() if hasattr(empty_result, "model_dump_json") else empty_result
+    await litellm.cache.async_add_cache(stored, **kwargs)
+    assert await litellm.cache.async_get_cache(**kwargs) is not None
+
+    handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START)
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=original_function,
+        logging_obj=_completion_logging_obj(call_type),
+        start_time=_FIXED_START,
+        call_type=call_type,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+@pytest.mark.asyncio
+async def test_async_get_cache_heals_stored_completion_without_choices():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+    poisoned = litellm.ModelResponse(
+        choices=[], usage=litellm.Usage(prompt_tokens=7, completion_tokens=0, total_tokens=7)
+    )
+    await litellm.cache.async_add_cache(poisoned.model_dump_json(), **kwargs)
+    assert await litellm.cache.async_get_cache(**kwargs) is not None
+
+    async def lookup():
+        return await handler._async_get_cache(
+            model="gpt-3.5-turbo",
+            original_function=litellm.acompletion,
+            logging_obj=_completion_logging_obj(CallTypes.acompletion.value),
+            start_time=_FIXED_START,
+            call_type=CallTypes.acompletion.value,
+            kwargs=kwargs,
+        )
+
+    assert (await lookup()).cached_result is None
+
+    await handler.async_set_cache(result=chat_completion_response, original_function=litellm.acompletion, kwargs=kwargs)
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    healed = (await lookup()).cached_result
+    assert healed is not None
+    assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content
+
+
+def test_sync_set_cache_skips_response_without_output():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+
+    handler.sync_set_cache(result=litellm.ModelResponse(choices=[]), kwargs=kwargs)
+
+    assert litellm.cache.get_cache(**kwargs) is None
+    lookup = handler._sync_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=completion,
+        logging_obj=_completion_logging_obj(CallTypes.completion.value),
+        start_time=_FIXED_START,
+        call_type=CallTypes.completion.value,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+def test_sync_get_cache_heals_stored_completion_without_choices():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+    litellm.cache.add_cache(litellm.ModelResponse(choices=[]).model_dump_json(), **kwargs)
+    assert litellm.cache.get_cache(**kwargs) is not None
+
+    def lookup():
+        return handler._sync_get_cache(
+            model="gpt-3.5-turbo",
+            original_function=completion,
+            logging_obj=_completion_logging_obj(CallTypes.completion.value),
+            start_time=_FIXED_START,
+            call_type=CallTypes.completion.value,
+            kwargs=kwargs,
+        )
+
+    assert lookup().cached_result is None
+
+    handler.sync_set_cache(result=chat_completion_response, kwargs=kwargs)
+    healed = lookup().cached_result
+    assert healed is not None
+    assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content
+
+
+@pytest.mark.asyncio
+async def test_acompletion_after_a_response_without_choices_calls_the_provider_again():
+    setup_cache()
+    messages = _unique_messages()
+
+    first = await litellm.acompletion(
+        model="gpt-4o", messages=messages, mock_response=litellm.ModelResponse(choices=[]), caching=True
+    )
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    assert first.choices == []
+
+    second = await litellm.acompletion(model="gpt-4o", messages=messages, mock_response="hi", caching=True)
+    assert second.choices[0].message.content == "hi"
+    assert second._hidden_params.get("cache_hit") is not True
+
+
+def _stream_chunk(choices: list[StreamingChoices]) -> ModelResponseStream:
+    return ModelResponseStream(
+        id="chatcmpl-stream", created=0, model="gpt-4o", object="chat.completion.chunk", choices=choices
+    )
+
+
+def _closing_chunk_without_output() -> ModelResponseStream:
+    return _stream_chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())])
+
+
+def _cached_content(cached: object) -> object:
+    assert cached is not None
+    stored = cached if isinstance(cached, dict) else json.loads(cached)
+    return stored["choices"][0]["message"]["content"]
+
+
+def _content_chunk(content: str) -> ModelResponseStream:
+    return _stream_chunk([StreamingChoices(index=0, delta=Delta(content=content, role="assistant"))])
+
+
+@pytest.mark.asyncio
+async def test_async_streamed_answer_without_output_is_never_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    await handler._add_streaming_response_to_cache(_closing_chunk_without_output())
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert litellm.cache.get_cache(**kwargs) is None
+
+
+@pytest.mark.asyncio
+async def test_async_streamed_answer_with_content_is_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    await handler._add_streaming_response_to_cache(_content_chunk("hi"))
+    await handler._add_streaming_response_to_cache(_closing_chunk_without_output())
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi"
+
+
+def test_sync_streamed_answer_without_output_is_never_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output())
+
+    assert litellm.cache.get_cache(**kwargs) is None
+
+
+def test_sync_streamed_answer_with_content_is_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    handler._sync_add_streaming_response_to_cache(_content_chunk("hi"))
+    handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output())
+
+    assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi"
+
+
+@pytest.mark.asyncio
+async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_without_output():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START)
+    handler.dual_cache = DualCache(in_memory_cache=InMemoryCache())
+    kwargs = {"messages": _unique_messages()}
+    key = litellm.cache.get_cache_key(**kwargs)
+    poisoned = litellm.ModelResponse(choices=[]).model_dump_json()
+    await handler.dual_cache.async_set_cache(key, {"timestamp": _FIXED_START.timestamp(), "response": poisoned})
+    assert await handler.dual_cache.async_get_cache(key) is not None
+
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=litellm.acompletion,
+        logging_obj=_completion_logging_obj(CallTypes.acompletion.value),
+        start_time=_FIXED_START,
+        call_type=CallTypes.acompletion.value,
+        kwargs=kwargs,
+    )
+
+    assert lookup.cached_result is None
+    assert await handler.dual_cache.async_get_cache(key) is None

tests/unit/enterprise/proxy/management_endpoints/test_project_endpoints_prisma.py

@@ -1,5 +1,9 @@
 import os
 import traceback
+from collections.abc import Mapping
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from typing import Final
 from litellm._uuid import uuid
 from unittest import mock
 
@@ -37,8 +41,14 @@ from litellm.proxy._types import (
     DeleteProjectRequest,
     NewTeamRequest,
     UserAPIKeyAuth,
+    LiteLLM_OrganizationMembershipTable,
+    LiteLLM_TeamTable,
+    LiteLLM_UserTable,
+    Member,
     ProxyException,
 )
+from litellm.proxy.management.teams.authz import TeamAccess
+from litellm.proxy.management.users.service import PrismaOrgRoles
 from tests._master_key import MASTER_KEY
 
 proxy_logging_obj = ProxyLogging(user_api_key_cache=DualCache())
@@ -1381,3 +1391,208 @@ async def test_new_project_flag_on_access_group_model_returns_400(monkeypatch):
 
     assert "prod-models" in str(exc_info.value)
     assert "expand to multiple models at request time" in str(exc_info.value)
+
+
+_ACCESS_NOW: Final = datetime.now(timezone.utc)
+
+
+@dataclass(frozen=True, slots=True)
+class _ProjectRow:
+    project_id: str
+    team_id: str | None
+
+
+def _membership(user_id: str, org_id: str, role: LitellmUserRoles) -> LiteLLM_OrganizationMembershipTable:
+    return LiteLLM_OrganizationMembershipTable(
+        user_id=user_id, organization_id=org_id, user_role=role.value, created_at=_ACCESS_NOW, updated_at=_ACCESS_NOW
+    )
+
+
+def _user(user_id: str, teams: tuple[str, ...] = (), admin_of: tuple[str, ...] = ()) -> LiteLLM_UserTable:
+    return LiteLLM_UserTable(
+        user_id=user_id,
+        teams=list(teams),
+        organization_memberships=[_membership(user_id, "org-a", LitellmUserRoles.INTERNAL_USER)]
+        + [_membership(user_id, org_id, LitellmUserRoles.ORG_ADMIN) for org_id in admin_of],
+    )
+
+
+_TEAMS: Final = {
+    team.team_id: team
+    for team in (
+        LiteLLM_TeamTable(
+            team_id="team-a1",
+            organization_id="org-a",
+            members_with_roles=[Member(user_id="member", role="user"), Member(user_id="team-admin", role="admin")],
+        ),
+        LiteLLM_TeamTable(team_id="team-a2", organization_id="org-a"),
+        LiteLLM_TeamTable(
+            team_id="team-b1",
+            organization_id="org-b",
+            members_with_roles=[Member(user_id="org-admin-a-member-b1", role="user")],
+        ),
+        LiteLLM_TeamTable(team_id="team-orgless"),
+    )
+}
+_PROJECTS: Final = (
+    _ProjectRow("p-a1", "team-a1"),
+    _ProjectRow("p-a2", "team-a2"),
+    _ProjectRow("p-b1", "team-b1"),
+    _ProjectRow("p-orgless", "team-orgless"),
+    _ProjectRow("p-teamless", None),
+    _ProjectRow("p-deleted-team", "team-deleted"),
+)
+_USERS: Final = {
+    user.user_id: user
+    for user in (
+        _user("member", teams=("team-a1",)),
+        _user("team-admin", teams=("team-a1",)),
+        _user("org-admin-a", admin_of=("org-a",)),
+        _user("org-admin-b", admin_of=("org-b",)),
+        _user("org-admin-a-member-b1", teams=("team-b1",), admin_of=("org-a",)),
+    )
+}
+
+
+def _row_matches(row: object, where: Mapping[str, object]) -> bool:
+    return all(_condition_holds(row, key, condition) for key, condition in where.items())
+
+
+def _condition_holds(row: object, key: str, condition) -> bool:
+    if key == "OR":
+        return any(_row_matches(row, branch) for branch in condition)
+    if key == "litellm_team_table":
+        team = _TEAMS.get(getattr(row, "team_id"))
+        return team is not None and _row_matches(team, condition["is"])
+    value = getattr(row, key)
+    return value in condition["in"] if isinstance(condition, dict) else value == condition
+
+
+class _FakeTeamTable:
+    async def find_unique(self, where: Mapping[str, str], include: object = None) -> LiteLLM_TeamTable | None:
+        return _TEAMS.get(where["team_id"])
+
+
+class _FakeProjectTable:
+    async def find_unique(self, where: Mapping[str, str], include: object = None) -> _ProjectRow | None:
+        return next((p for p in _PROJECTS if p.project_id == where["project_id"]), None)
+
+    async def find_many(self, where: Mapping[str, object] | None = None, include: object = None) -> list[_ProjectRow]:
+        return [p for p in _PROJECTS if where is None or _row_matches(p, where)]
+
+
+class _FakeUserTable:
+    async def find_unique(
+        self, where: Mapping[str, str], include: Mapping[str, bool] | None = None
+    ) -> LiteLLM_UserTable | None:
+        user = _USERS.get(where["user_id"])
+        if user is None or (include or {}).get("organization_memberships"):
+            return user
+        return user.model_copy(update={"organization_memberships": None})
+
+
+@pytest.fixture
+def project_access_db(monkeypatch):
+    db = mock.MagicMock(
+        litellm_teamtable=_FakeTeamTable(), litellm_projecttable=_FakeProjectTable(), litellm_usertable=_FakeUserTable()
+    )
+    monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", mock.MagicMock(db=db))
+
+
+async def _team_access_over_cached_users() -> TeamAccess:
+    cache = UserApiKeyCache()
+    for user in _USERS.values():
+        await cache.async_set_cache(key=user.user_id, value=user)
+    return TeamAccess(org_roles=PrismaOrgRoles(None, cache, proxy_logging_obj))
+
+
+def _caller(user_id: str, role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER) -> UserAPIKeyAuth:
+    return UserAPIKeyAuth(user_role=role, api_key="sk-caller", user_id=user_id)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_id", "expected"),
+    [
+        ("member", {"p-a1"}),
+        ("team-admin", {"p-a1"}),
+        ("org-admin-a", {"p-a1", "p-a2"}),
+        ("org-admin-b", {"p-b1"}),
+        ("org-admin-a-member-b1", {"p-a1", "p-a2", "p-b1"}),
+        ("unknown-user", set()),
+    ],
+)
+async def test_list_projects_scopes_to_teams_the_caller_can_view(project_access_db, user_id, expected):
+    from litellm_enterprise.proxy.management_endpoints.project_endpoints import list_projects
+
+    projects = await list_projects(user_api_key_dict=_caller(user_id))
+
+    assert {p.project_id for p in projects} == expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("role", [LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY])
+async def test_list_projects_admin_view_sees_every_project(project_access_db, role):
+    from litellm_enterprise.proxy.management_endpoints.project_endpoints import list_projects
+
+    projects = await list_projects(user_api_key_dict=_caller("someone", role))
+
+    assert {p.project_id for p in projects} == {p.project_id for p in _PROJECTS}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_id", "project_id"),
+    [
+        ("member", "p-a1"),
+        ("team-admin", "p-a1"),
+        ("org-admin-a", "p-a1"),
+        ("org-admin-a", "p-a2"),
+        ("org-admin-a-member-b1", "p-b1"),
+    ],
+)
+async def test_project_info_allows_team_members_and_org_admins_of_the_team_org(project_access_db, user_id, project_id):
+    project = await project_info(
+        project_id=project_id,
+        user_api_key_dict=_caller(user_id),
+        team_access=await _team_access_over_cached_users(),
+    )
+
+    assert project.project_id == project_id
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_id", "project_id"),
+    [
+        ("member", "p-a2"),
+        ("team-admin", "p-b1"),
+        ("org-admin-b", "p-a2"),
+        ("org-admin-a", "p-b1"),
+        ("org-admin-a", "p-orgless"),
+        ("org-admin-a", "p-teamless"),
+        ("member", "p-deleted-team"),
+    ],
+)
+async def test_project_info_denies_callers_who_cannot_view_the_team(project_access_db, user_id, project_id):
+    with pytest.raises(ProxyException) as exc_info:
+        await project_info(
+            project_id=project_id,
+            user_api_key_dict=_caller(user_id),
+            team_access=await _team_access_over_cached_users(),
+        )
+
+    assert str(exc_info.value.code) == "403"
+    assert "You don't have access to this project" in exc_info.value.message
+
+
+@pytest.mark.asyncio
+async def test_project_info_missing_project_is_404_even_for_org_admins(project_access_db):
+    with pytest.raises(ProxyException) as exc_info:
+        await project_info(
+            project_id="p-missing",
+            user_api_key_dict=_caller("org-admin-a"),
+            team_access=await _team_access_over_cached_users(),
+        )
+
+    assert str(exc_info.value.code) == "404"

tests/unit/integrations/otel/test_otel_v2_logger.py

@@ -182,10 +182,63 @@ def test_async_log_success_event_emits_llm_call_span():
     assert span.attributes[GenAI.OPERATION_NAME] == "chat"
     assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o"
     assert span.attributes[LiteLLM.CALL_ID] == "call_1"
+    assert not any(key.startswith("litellm.routing.") for key in span.attributes)
     # Success leaves status UNSET (semconv default), not forced OK.
     assert span.status.status_code is StatusCode.UNSET
 
 
+@pytest.mark.parametrize("fail", [False, True])
+@pytest.mark.parametrize("legacy_compat", [False, True])
+def test_routing_diagnostics_reach_llm_spans_without_promoting_prompt_text(fail: bool, legacy_compat: bool) -> None:
+    logger, exporter = _logger(legacy_compat=legacy_compat)
+    decision: Final = {
+        "router_model_name": "auto-router",
+        "router_type": "complexity",
+        "router_config_id": "definition-1",
+        "router_config_updated_at": "2026-10-06T00:00:00Z",
+        "router_config_fingerprint": "definition-hash",
+        "routed_model": "answer",
+        "cause": "heuristic",
+        "score": 0.25,
+        "classifier_cost": 0.001,
+        "classifier_failure_reason": "timeout",
+        "classifier_error_type": "TimeoutError",
+        "escalated": False,
+    }
+    payload: Final = _payload(
+        metadata={
+            "routing_decision": {
+                **decision,
+                "signals": ["private prompt"],
+                "matched_keyword": "private prompt",
+                "escalation_keyword": "private prompt",
+                "classifier_crux": "private prompt",
+                "tier_litellm_params": {"api_key": "private key"},
+                "classifier_probabilities": {"easy": 0.25},
+                "unknown_field": "private config",
+                "tier_label": {"invalid": "scalar field"},
+            }
+        },
+        status="failure" if fail else "success",
+    )
+    _emit_llm(logger, _kwargs(payload), fail=fail)
+    (span,) = exporter.get_finished_spans()
+    actual: Final = {key: value for key, value in span.attributes.items() if key.startswith("litellm.routing.")}
+    assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()}
+    assert isinstance(actual["litellm.routing.score"], float)
+    assert actual["litellm.routing.escalated"] is False
+
+
+@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]])
+def test_malformed_routing_metadata_preserves_the_completed_llm_span(decision: object) -> None:
+    logger, exporter = _logger()
+    _emit_llm(logger, _kwargs(_payload(metadata={"routing_decision": decision})))
+    (span,) = exporter.get_finished_spans()
+    assert span.attributes[GenAI.REQUEST_MODEL] == "gpt-4o"
+    assert span.attributes[LiteLLM.CALL_ID] == "call_1"
+    assert not any(key.startswith("litellm.routing.") for key in span.attributes)
+
+
 def test_llm_call_span_carries_the_callers_conversation_id():
     logger, exporter = _logger()
     kwargs = {**_kwargs(), "litellm_params": {"litellm_session_id": "conv-42", "metadata": {}}}
@@ -1103,6 +1156,29 @@ def test_phase_event_lands_on_root_span_even_inside_active_phase_span():
     assert by_name["auth /chat/completions"].events == ()
 
 
+def test_routing_attributes_stay_on_the_active_phase_while_retry_events_stay_on_root() -> None:
+    logger, exporter = _logger()
+    root: Final = logger.tracer.start_span("request", kind=SpanKind.SERVER)
+    set_request_root_span(root)
+    with trace.use_span(root, end_on_exit=True):
+        logger.set_phase_attributes({"litellm.routing.cause": "outside phase"})
+        with logger.start_phase_span("route auto-router"):
+            logger.set_phase_attributes({"litellm.routing.router_config_id": "definition-1"})
+            with logger.tracer.start_as_current_span("classifier transport"):
+                logger.set_phase_attributes({"litellm.routing.cause": "heuristic"})
+                logger.add_phase_event("litellm.request.retry", {"error.type": "TimeoutError"})
+        logger.set_phase_attributes({"litellm.routing.cause": "after phase"})
+    by_name: Final = {span.name: span for span in exporter.get_finished_spans()}
+    route: Final = by_name["route auto-router"]
+    assert route.attributes["litellm.routing.router_config_id"] == "definition-1"
+    assert route.attributes["litellm.routing.cause"] == "heuristic"
+    assert route.parent.span_id == root.get_span_context().span_id
+    assert route.events == ()
+    assert "litellm.routing.cause" not in by_name["request"].attributes
+    assert "litellm.routing.cause" not in by_name["classifier transport"].attributes
+    assert by_name["request"].events[0].attributes["error.type"] == "TimeoutError"
+
+
 def test_live_llm_span_anchors_to_root_with_no_active_span():
     """Bug 2 (pass-through), live path: even with no span active at ``pre_call``,
     the anchor is a recordable parent, so the span opens live under the server root

tests/unit/integrations/otel/test_runtime.py

@@ -10,6 +10,11 @@ import lock. These tests pin the import to a single resolution.
 import builtins
 import importlib.abc
 import sys
+from collections.abc import Sequence
+from types import ModuleType
+from typing import Final
+
+import pytest
 
 import litellm.integrations.otel.runtime as runtime
 
@@ -92,4 +97,28 @@ def test_phase_span_does_not_import_the_proxy_in_an_sdk_process(monkeypatch):
     with runtime.phase_span("route gpt-5-mini") as span:
         assert span is None
 
+    assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None
     assert proxy_imports == []
+
+
+def test_phase_attributes_no_op_when_sdk_import_is_unavailable(monkeypatch: pytest.MonkeyPatch) -> None:
+    runtime._otel_runtime.cache_clear()
+    real_import: Final = builtins.__import__
+
+    def without_sdk(
+        name: str,
+        globals: dict[str, object] | None = None,
+        locals: dict[str, object] | None = None,
+        fromlist: Sequence[str] = (),
+        level: int = 0,
+    ) -> ModuleType:
+        if name == "litellm.integrations.otel" and "logger" in fromlist:
+            raise ImportError("OpenTelemetry SDK is not installed")
+        return real_import(name, globals, locals, fromlist, level)
+
+    monkeypatch.setattr(builtins, "__import__", without_sdk)
+    try:
+        assert runtime.phase_attributes({"litellm.routing.score": 0.25}) is None
+        assert runtime._otel_runtime() is None
+    finally:
+        runtime._otel_runtime.cache_clear()

tests/unit/integrations/test_opentelemetry.py

@@ -310,6 +310,65 @@ class TestOpenTelemetryTeamAttributesOnChildSpans(unittest.TestCase):
         )
 
 
+def test_v1_routing_diagnostics_add_scalar_attributes_and_preserve_legacy_metadata() -> None:
+    exporter: Final = InMemorySpanExporter()
+    provider: Final = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    otel: Final = OpenTelemetry(tracer_provider=provider)
+    decision: Final = {
+        "router_model_name": "auto-router",
+        "routed_model": "answer",
+        "router_config_id": "definition-1",
+        "cause": "heuristic",
+        "classifier_failure_reason": "timeout",
+        "classifier_error_type": "TimeoutError",
+        "classifier_cost": 0.001,
+    }
+    with provider.get_tracer(__name__).start_as_current_span("inference") as span:
+        otel.set_attributes(
+            span,
+            {
+                "model": "answer",
+                "standard_logging_object": {
+                    "id": "routing-call",
+                    "call_type": "acompletion",
+                    "metadata": {"routing_decision": decision},
+                },
+            },
+            {"model": "answer", "choices": [], "usage": {}},
+        )
+    (finished,) = exporter.get_finished_spans()
+    actual: Final = {
+        key: value for key, value in finished.attributes.items() if key.startswith("litellm.routing.")
+    }
+    assert actual == {f"litellm.routing.{key}": value for key, value in decision.items()}
+    assert finished.attributes["metadata.routing_decision"] == str(decision)
+
+
+@pytest.mark.parametrize("decision", ["custom", 42, ["custom"]])
+def test_v1_malformed_routing_metadata_preserves_inference_attributes(decision: object) -> None:
+    exporter: Final = InMemorySpanExporter()
+    provider: Final = TracerProvider()
+    provider.add_span_processor(SimpleSpanProcessor(exporter))
+    otel: Final = OpenTelemetry(tracer_provider=provider)
+    with provider.get_tracer(__name__).start_as_current_span("inference") as span:
+        otel.set_attributes(
+            span,
+            {
+                "model": "answer",
+                "standard_logging_object": {
+                    "id": "routing-call",
+                    "call_type": "acompletion",
+                    "metadata": {"routing_decision": decision},
+                },
+            },
+            {"model": "answer", "choices": [], "usage": {}},
+        )
+    (finished,) = exporter.get_finished_spans()
+    assert finished.attributes["gen_ai.request.model"] == "answer"
+    assert not any(key.startswith("litellm.routing.") for key in finished.attributes)
+
+
 class TestOpenTelemetryCostBreakdown(unittest.TestCase):
     def test_cost_breakdown_emitted_to_otel_span(self):
         """

tests/unit/litellm_core_utils/test_litellm_logging.py

@@ -6014,7 +6014,7 @@ def _interactions_logging_obj(stream: bool, call_type: str = "acreate"):
         messages=[],
         stream=stream,
         call_type=call_type,
-        start_time=time.time(),
+        start_time=datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc).timestamp(),
         litellm_call_id="interactions-call-id",
         function_id="interactions-fn-id",
     )
@@ -9748,3 +9748,61 @@ def test_get_custom_logger_compatible_class_does_not_match_generic_api_logger(
         assert logging_module.get_custom_logger_compatible_class(integration) is None
     finally:
         logging_module._in_memory_loggers.clear()
+
+
+@pytest.mark.asyncio
+async def test_background_interaction_completion_logs_while_in_progress_handler_is_parked(monkeypatch):
+    """
+    The create's in_progress success handler can still be parked on awaited work when
+    the settlement arrives. The completed result is the only event carrying usage and
+    cost, so it has to reach the success callbacks whatever the one-success-log-per-request
+    dedupe does, or the settlement is marked billed with no spend row behind it.
+    """
+    from litellm.litellm_core_utils import litellm_logging
+    from litellm.types.interactions import InteractionsAPIResponse
+
+    class CountingLogger(CustomLogger):
+        def __init__(self):
+            super().__init__()
+            self.logged_results: list[object] = []
+
+        async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
+            self.logged_results.append(response_obj)
+
+    counting_logger: Final = CountingLogger()
+    monkeypatch.setattr(litellm, "_async_success_callback", [counting_logger])
+
+    in_progress_parked: Final = asyncio.Event()
+    release_in_progress: Final = asyncio.Event()
+    real_truncate: Final = litellm_logging.truncate_base64_in_messages_async
+    calls: list[int] = []
+
+    async def parked_then_real_truncate(messages):
+        calls.append(1)
+        if len(calls) == 1:
+            in_progress_parked.set()
+            await release_in_progress.wait()
+        return await real_truncate(messages)
+
+    monkeypatch.setattr(litellm_logging, "truncate_base64_in_messages_async", parked_then_real_truncate)
+
+    logging_obj: Final = _interactions_logging_obj(stream=False)
+    in_progress: Final = InteractionsAPIResponse(id="interactions/abc", model="gemini-2.5-flash", status="in_progress")
+    completed: Final = InteractionsAPIResponse(
+        id="interactions/abc",
+        model="gemini-2.5-flash",
+        status="completed",
+        steps=[],
+        usage=dict(INTERACTIONS_USAGE_BLOCK),
+    )
+
+    first: Final = asyncio.create_task(logging_obj.async_success_handler(result=in_progress))
+    await in_progress_parked.wait()
+    completion: Final = asyncio.create_task(logging_obj.async_log_background_interaction_completion(result=completed))
+    await asyncio.sleep(0)
+    await asyncio.sleep(0)
+    release_in_progress.set()
+    await first
+    await completion
+
+    assert counting_logger.logged_results == [completed, in_progress], counting_logger.logged_results

tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py

@@ -160,6 +160,26 @@ async def test_streaming_cache_is_not_shared_with_non_streaming(local_cache, req
     assert non_streaming["content"][0]["text"] == "ALPHA"
 
 
+@pytest.mark.asyncio
+async def test_stream_without_content_blocks_is_not_cached(local_cache, request_kwargs, monkeypatch):
+    empty_events = [STREAM_EVENTS[0], STREAM_EVENTS[4], STREAM_EVENTS[5]]
+    fake_handler = _CountingHandler(
+        [_byte_stream(empty_events), _byte_stream(STREAM_EVENTS), _byte_stream([b"event: never_used\n\n"])]
+    )
+    monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler)
+
+    empty = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+    await asyncio.sleep(0)
+    refilled = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+    await asyncio.sleep(0)
+    replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+
+    assert empty == empty_events
+    assert refilled == STREAM_EVENTS
+    assert replayed == STREAM_EVENTS
+    assert len(fake_handler.calls) == 2
+
+
 @pytest.mark.asyncio
 async def test_failed_stream_is_not_cached(local_cache, request_kwargs, monkeypatch):
     error_events = STREAM_EVENTS[:3] + [

tests/unit/proxy/db/test_daily_spend_bulk_upsert.py

@@ -13,6 +13,7 @@ from litellm.proxy.db.daily_spend_bulk_upsert import (
     merge_by_conflict_key,
 )
 from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
 
 TAG_TABLE = DAILY_SPEND_TABLES["tag"]
 USER_TABLE = DAILY_SPEND_TABLES["user"]
@@ -146,8 +147,12 @@ def test_non_tag_tables_carry_no_request_id_column():
 class _RecordingDb:
     def __init__(self) -> None:
         self.statements: list[tuple[str, tuple[object, ...]]] = []
+        self.session_settings: list[tuple[str, tuple[object, ...]]] = []
 
     async def execute_raw(self, query: str, *args: object) -> int:
+        if query == ROLLUP_LOCK_TIMEOUT_SQL:
+            self.session_settings.append((query, args))
+            return 0
         self.statements.append((query, args))
         return len(args)
 

tests/unit/proxy/db/test_db_spend_update_writer.py

@@ -19,6 +19,7 @@ from redis.exceptions import DataError
 import litellm
 from litellm._logging import verbose_proxy_logger
 from litellm._service_logger import ServiceTypes
+from litellm.constants import SPEND_ROLLUP_LOCK_TIMEOUT_MS
 from litellm.proxy._types import DailyTagSpendTransaction, Litellm_EntityType, SpendUpdateQueueItem
 from litellm.proxy.db.db_spend_update_writer import (
     _TEAM_ADVISORY_LOCK_SQL,
@@ -33,6 +34,7 @@ from litellm.proxy.db.db_transaction_queue.spend_update_queue import SpendUpdate
 from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
     build_window_spend_transaction,
 )
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
 from tests.unit.proxy.db.fake_prisma_engine import engine_call
 
 
@@ -103,11 +105,21 @@ async def test_update_database_attributes_router_rejected_failure_to_model_group
     )
 
     with (
-        patch("litellm.proxy.proxy_server.disable_spend_logs", True),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
-        patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
-        patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
-        patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
-        patch("litellm.proxy.proxy_server.llm_router", llm_router),  # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.disable_spend_logs", True
+        ),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.prisma_client", MagicMock()
+        ),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.user_api_key_cache", MagicMock()
+        ),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"
+        ),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.llm_router", llm_router
+        ),  # test-quality-ok: get_llm_router reads this proxy_server module global at call time; no injection seam
     ):
         await db_writer.update_database(
             token="test-token",
@@ -320,7 +332,9 @@ async def test_a_routed_request_reaches_the_auto_router_rollup_whether_or_not_sp
     }
 
     with (
-        patch("litellm.proxy.proxy_server.disable_spend_logs", disable_spend_logs),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
+        patch(
+            "litellm.proxy.proxy_server.disable_spend_logs", disable_spend_logs
+        ),  # test-quality-ok: update_database reads this proxy_server module global at call time; no injection seam
         patch("litellm.proxy.proxy_server.prisma_client", prisma),
         patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
         patch(
@@ -356,9 +370,13 @@ class _RecordingDb:
 
     def __init__(self, execute_raw: Callable[[], int] | None = None) -> None:
         self.statements: list[Statement] = []
+        self.session_settings: list[Statement] = []
         self._execute_raw = execute_raw
 
     async def execute_raw(self, query: str, *args: object) -> int:
+        if query == ROLLUP_LOCK_TIMEOUT_SQL:
+            self.session_settings.append((query, args))
+            return 0
         self.statements.append((query, args))
         if self._execute_raw is not None:
             return self._execute_raw()
@@ -752,7 +770,7 @@ async def test_update_tag_db_with_valid_tags():
     """
     Test that _update_tag_db correctly processes valid tags and adds them to the spend update queue.
     """
-    from litellm.proxy._types import Litellm_EntityType, SpendUpdateQueueItem
+    from litellm.proxy._types import Litellm_EntityType
 
     writer = DBSpendUpdateWriter()
     mock_prisma = MagicMock()
@@ -1048,7 +1066,7 @@ async def test_commit_spend_updates_to_db_writes_team_member_spend_in_one_roster
         ),
     )
 
-    lock_call, spend_call = mock_transaction.execute_raw.await_args_list
+    _lock_timeout_call, lock_call, spend_call = mock_transaction.execute_raw.await_args_list
     lock_statement, locked_team_id = lock_call.args
     assert lock_statement is _TEAM_ADVISORY_LOCK_SQL
     assert locked_team_id == team_id
@@ -1092,7 +1110,7 @@ async def test_commit_spend_updates_to_db_orders_team_member_rows_by_team_then_u
         ),
     )
 
-    *lock_calls, spend_call = mock_transaction.execute_raw.await_args_list
+    _lock_timeout_call, *lock_calls, spend_call = mock_transaction.execute_raw.await_args_list
     _statement, members = spend_call.args
     assert [lock_call.args for lock_call in lock_calls] == [
         (_TEAM_ADVISORY_LOCK_SQL, "eng"),
@@ -1730,7 +1748,7 @@ async def test_endpoint_field_is_correctly_mapped_from_call_type():
 
     for key, transaction in update_dict.items():
         # Verify endpoint is included in the key
-        assert key == f"test-user_2024-01-01_test-key_gpt-4_openai_/chat/completions"
+        assert key == "test-user_2024-01-01_test-key_gpt-4_openai_/chat/completions"
 
         # Verify endpoint is set in the transaction
         assert transaction["endpoint"] == "/chat/completions"
@@ -2912,17 +2930,22 @@ async def test_daily_transaction_carries_compression_saved_tokens():
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("estimate, recorded_savings, expected", [
-    pytest.param(None, None, -0.005, id="plain-classifier-cost"),
-    pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"),
-    pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"),
-    pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"),
-    pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"),
-    pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"),
-    pytest.param(None, -0.003, -0.003, id="legacy"),
-])
+@pytest.mark.parametrize(
+    "estimate, recorded_savings, expected",
+    [
+        pytest.param(None, None, -0.005, id="plain-classifier-cost"),
+        pytest.param({"version": 1, "status": "unknown"}, None, 0.0, id="unknown"),
+        pytest.param({"version": 2, "status": "unknown"}, None, 0.0, id="unknown-v2"),
+        pytest.param({"version": 1, "status": "unknown"}, -0.003, 0.0, id="unknown-stale-value"),
+        pytest.param({"version": 0, "status": "estimated"}, -0.003, 0.0, id="unsupported-version"),
+        pytest.param({"version": 1, "status": "estimated"}, -0.003, -0.003, id="estimated"),
+        pytest.param(None, -0.003, -0.003, id="legacy"),
+    ],
+)
 async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
-    estimate: dict[str, object] | None, recorded_savings: float | None, expected: float,
+    estimate: dict[str, object] | None,
+    recorded_savings: float | None,
+    expected: float,
 ) -> None:
     """Requests without any compression metadata produce a zero count."""
     writer = DBSpendUpdateWriter()
@@ -2941,12 +2964,14 @@ async def test_daily_transaction_compression_saved_tokens_zero_when_absent(
         "prompt_tokens": 100,
         "completion_tokens": 10,
         "spend": 0.01,
-        "metadata": json.dumps({
-            "usage_object": {"prompt_tokens": 100, "completion_tokens": 10},
-            "routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005},
-            "autorouter_savings": recorded_savings,
-            "autorouter_savings_estimate": estimate,
-        }),
+        "metadata": json.dumps(
+            {
+                "usage_object": {"prompt_tokens": 100, "completion_tokens": 10},
+                "routing_decision": {"savings_baseline_model": "anthropic/claude-sonnet-5", "classifier_cost": 0.005},
+                "autorouter_savings": recorded_savings,
+                "autorouter_savings_estimate": estimate,
+            }
+        ),
     }
 
     transaction = await writer._common_add_spend_log_transaction_to_daily_transaction(
@@ -3417,6 +3442,9 @@ async def test_failed_per_entity_increment_from_redis_restores_only_what_may_sti
         def batch_(self):
             return _BatchContext()
 
+        async def execute_raw(self, query: str, *args: object) -> int:
+            return 0
+
         async def __aenter__(self):
             return self
 
@@ -3439,9 +3467,7 @@ async def test_failed_per_entity_increment_from_redis_restores_only_what_may_sti
         )
 
     mock_redis_update_buffer.restore_transactions_to_redis.assert_awaited_once()
-    restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs[
-        "db_spend_update_transactions"
-    ]
+    restored = mock_redis_update_buffer.restore_transactions_to_redis.call_args.kwargs["db_spend_update_transactions"]
     assert restored["user_list_transactions"] is None
     assert restored["team_list_transactions"] == {"team-1": 1.5}
     assert restored["key_list_transactions"] == ({"key-1": 1.5} if safe_to_resend else None)
@@ -3985,6 +4011,77 @@ async def test_commit_spend_updates_retries_deadlock_on_every_entity_path(monkey
     proxy_logging.failure_handler.assert_not_called()
 
 
+_SPEND_UPDATE_PATHS: Final = [
+    ("key_list_transactions", "sk-abc"),
+    ("user_list_transactions", "user-1"),
+    ("team_list_transactions", "team-1"),
+    ("team_member_list_transactions", "team_id::team-1::user_id::user-1"),
+    ("org_list_transactions", "org-1"),
+    ("org_member_list_transactions", "organization_id::org-1::user_id::user-1"),
+    ("tag_list_transactions", "tag-1"),
+    ("agent_list_transactions", "agent-1"),
+]
+
+
+@pytest.mark.parametrize("transactions_key, sample_key", _SPEND_UPDATE_PATHS)
+@pytest.mark.asyncio
+async def test_commit_spend_updates_retries_a_rollup_lock_timeout_then_commits(
+    monkeypatch, transactions_key, sample_key
+):
+    """A spend UPDATE cancelled under ``lock_timeout`` (55P03) never took its row lock, so the
+    transaction rolled back with nothing applied. Every entity path retries it like a deadlock
+    and lands the increment exactly once, instead of raising and dropping the drained batch."""
+    slept = []
+    monkeypatch.setattr(
+        "litellm.proxy.db.db_spend_update_writer.asyncio.sleep",
+        AsyncMock(side_effect=lambda s: slept.append(s)),
+    )
+
+    mock_batcher = MagicMock()
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.tx = MagicMock(side_effect=[_failing_tx(_lock_timeout_error()), _good_tx(mock_batcher)])
+
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+    proxy_logging.call_details = {}
+
+    await DBSpendUpdateWriter()._commit_spend_updates_to_db(
+        prisma_client=mock_prisma_client,
+        n_retry_times=3,
+        proxy_logging_obj=proxy_logging,
+        db_spend_update_transactions=_empty_spend_transactions(**{transactions_key: {sample_key: 0.5}}),
+    )
+
+    assert {
+        "transactions_opened": mock_prisma_client.db.tx.call_count,
+        "backoff_sleeps": len(slept),
+        "failure_handler_calls": proxy_logging.failure_handler.await_count,
+    } == {"transactions_opened": 2, "backoff_sleeps": 1, "failure_handler_calls": 0}
+
+
+@pytest.mark.asyncio
+async def test_commit_spend_updates_raises_after_exhausting_lock_timeout_retries(monkeypatch):
+    """A row that stays locked past every retry still surfaces as a failure rather than
+    looping forever or being swallowed; the budget is the same one deadlocks get."""
+    monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", AsyncMock(return_value=None))
+
+    mock_prisma_client = MagicMock()
+    mock_prisma_client.db.tx = MagicMock(side_effect=lambda *a, **k: _failing_tx(_lock_timeout_error()))
+
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+
+    with pytest.raises(PrismaDataError, match="lock timeout"):
+        await DBSpendUpdateWriter()._commit_spend_updates_to_db(
+            prisma_client=mock_prisma_client,
+            n_retry_times=2,
+            proxy_logging_obj=proxy_logging,
+            db_spend_update_transactions=_empty_spend_transactions(key_list_transactions={"sk-abc": 0.5}),
+        )
+
+    assert mock_prisma_client.db.tx.call_count == 3
+
+
 @pytest.mark.asyncio
 @pytest.mark.parametrize(
     "call_type, expects_flush",
@@ -4881,3 +4978,92 @@ async def test_shutdown_drain_that_lands_before_the_interrupted_tag_commit_resol
     assert redis_buffer.restored == [drained], "a tag batch whose COMMIT came back failed must be restored to Redis"
     (upsert,) = _daily_upserts(final_db, "LiteLLM_DailyTagSpend")
     assert _row_values(upsert, "api_requests") == [1]
+
+
+def _lock_timeout_error() -> PrismaDataError:
+    return PrismaDataError(
+        data={
+            "user_facing_error": {
+                "is_panic": False,
+                "message": "Error querying the database: canceling statement due to lock timeout",
+                "meta": {"code": "55P03", "message": "canceling statement due to lock timeout"},
+            }
+        }
+    )
+
+
+@pytest.mark.asyncio
+async def test_daily_spend_upsert_runs_under_the_rollup_lock_timeout():
+    """The bulk daily upsert opens its transaction by bounding how long it may wait on
+    rows another pod holds; the setting is transaction-local and bound as a parameter."""
+    prisma_client = _RecordingPrisma()
+    daily_spend_transactions = {
+        "key": {
+            "user_id": "user-1",
+            "date": "2024-01-01",
+            "api_key": "test-api-key",
+            "model": "gpt-4",
+            "custom_llm_provider": "openai",
+            "prompt_tokens": 10,
+            "completion_tokens": 20,
+            "spend": 0.1,
+            "api_requests": 1,
+            "successful_requests": 1,
+            "failed_requests": 0,
+        }
+    }
+
+    await DBSpendUpdateWriter._update_daily_spend(
+        n_retry_times=0,
+        prisma_client=prisma_client,
+        proxy_logging_obj=MagicMock(),
+        daily_spend_transactions=daily_spend_transactions,
+        entity_type="user",
+        entity_id_field="user_id",
+    )
+
+    assert prisma_client.db.session_settings == [(ROLLUP_LOCK_TIMEOUT_SQL, (f"{SPEND_ROLLUP_LOCK_TIMEOUT_MS}ms",))]
+    assert len(prisma_client.db.statements) == 1
+    assert daily_spend_transactions == {}
+
+
+@pytest.mark.asyncio
+async def test_daily_spend_rows_survive_a_lock_timeout_for_the_next_flush():
+    """A statement cancelled under ``lock_timeout`` (55P03) never applied, so its rows
+    are not data Postgres refused: they stay queued for the next flush instead of being
+    dropped the way a constraint violation is."""
+
+    def cancelled_waiting_for_the_row() -> int:
+        raise _lock_timeout_error()
+
+    prisma_client = _RecordingPrisma(execute_raw=cancelled_waiting_for_the_row)
+    proxy_logging = MagicMock()
+    proxy_logging.failure_handler = AsyncMock()
+    daily_spend_transactions = {
+        "key": {
+            "user_id": "user-1",
+            "date": "2024-01-01",
+            "api_key": "test-api-key",
+            "model": "gpt-4",
+            "custom_llm_provider": "openai",
+            "prompt_tokens": 10,
+            "completion_tokens": 20,
+            "spend": 0.1,
+            "api_requests": 1,
+            "successful_requests": 1,
+            "failed_requests": 0,
+        }
+    }
+
+    with pytest.raises(PrismaDataError, match="lock timeout"):
+        await DBSpendUpdateWriter._update_daily_spend(
+            n_retry_times=0,
+            prisma_client=prisma_client,
+            proxy_logging_obj=proxy_logging,
+            daily_spend_transactions=daily_spend_transactions,
+            entity_type="user",
+            entity_id_field="user_id",
+        )
+
+    assert len(prisma_client.db.statements) == 1
+    assert list(daily_spend_transactions) == ["key"]

tests/unit/proxy/db/test_exception_handler.py

@@ -1,12 +1,10 @@
 import asyncio
-import json
 import sys
 from typing import Final
 from unittest.mock import MagicMock, patch
 
 import httpx
 import pytest
-from fastapi import HTTPException, Request
 from prisma import errors as prisma_errors
 from prisma.engine.errors import BinaryNotFoundError, EngineConnectionError, EngineRequestError
 from prisma.errors import (
@@ -22,9 +20,7 @@ from prisma.errors import (
     UniqueViolationError,
 )
 
-
 import litellm
-from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ProxyErrorTypes, ProxyException
 from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
 
@@ -55,22 +51,12 @@ def test_is_database_infrastructure_error_prisma_connection_errors(prisma_error)
         PrismaError(),
         PrismaError("validation failed on query"),
         DataError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
-        UniqueViolationError(
-            data={"user_facing_error": {"meta": {"table": "test_table"}}}
-        ),
-        ForeignKeyViolationError(
-            data={"user_facing_error": {"meta": {"table": "test_table"}}}
-        ),
-        MissingRequiredValueError(
-            data={"user_facing_error": {"meta": {"table": "test_table"}}}
-        ),
+        UniqueViolationError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
+        ForeignKeyViolationError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
+        MissingRequiredValueError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
         RawQueryError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
-        TableNotFoundError(
-            data={"user_facing_error": {"meta": {"table": "test_table"}}}
-        ),
-        RecordNotFoundError(
-            data={"user_facing_error": {"meta": {"table": "test_table"}}}
-        ),
+        TableNotFoundError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
+        RecordNotFoundError(data={"user_facing_error": {"meta": {"table": "test_table"}}}),
     ],
 )
 def test_is_database_transport_error_non_connection_prisma_errors(prisma_error):
@@ -82,12 +68,7 @@ def test_is_database_connection_generic_errors():
     """
     Test non-Prisma error cases for database connection checking
     """
-    assert (
-        PrismaDBExceptionHandler.is_database_connection_error(
-            Exception("Regular error")
-        )
-        == False
-    )
+    assert PrismaDBExceptionHandler.is_database_connection_error(Exception("Regular error")) == False
 
     # Test with ProxyException (DB connection)
     db_proxy_exception = ProxyException(
@@ -95,17 +76,11 @@ def test_is_database_connection_generic_errors():
         type=ProxyErrorTypes.no_db_connection,
         param="test-param",
     )
-    assert (
-        PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception)
-        == True
-    )
+    assert PrismaDBExceptionHandler.is_database_connection_error(db_proxy_exception) == True
 
     # Test with non-DB error
     regular_exception = Exception("Regular error")
-    assert (
-        PrismaDBExceptionHandler.is_database_connection_error(regular_exception)
-        == False
-    )
+    assert PrismaDBExceptionHandler.is_database_connection_error(regular_exception) == False
 
 
 @pytest.mark.parametrize(
@@ -143,12 +118,7 @@ def test_is_database_service_unavailable_error_prisma_p1001_masquerades_as_datae
             }
         }
     )
-    assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(
-            p1001_as_dataerror
-        )
-        is True
-    )
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(p1001_as_dataerror) is True
 
 
 def test_is_prisma_data_error_only_true_for_dataerror():
@@ -196,12 +166,7 @@ def test_is_database_service_unavailable_error_cached_plan_escapes_as_503():
             }
         }
     )
-    assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(
-            cached_plan_error
-        )
-        is True
-    )
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(cached_plan_error) is True
 
 
 def test_is_database_service_unavailable_error_prisma_engine_malformed_payload():
@@ -229,10 +194,7 @@ def test_is_database_service_unavailable_error_prisma_engine_malformed_payload()
         prisma_engine_utils.handle_response_errors(None, malformed_payload)
 
     assert "no attribute 'get'" in str(exc_info.value)
-    assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value)
-        is True
-    )
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) is True
 
 
 def test_is_prisma_engine_internal_error_excludes_application_attributeerror():
@@ -247,23 +209,15 @@ def test_is_prisma_engine_internal_error_excludes_application_attributeerror():
     with pytest.raises(AttributeError) as exc_info:
         application_bug()
 
-    assert (
-        PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value)
-        is False
-    )
-    assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value)
-        is False
-    )
+    assert PrismaDBExceptionHandler.is_prisma_engine_internal_error(exc_info.value) is False
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(exc_info.value) is False
 
 
 def test_is_prisma_engine_internal_error_excludes_data_layer_prisma_error():
     """A data-layer ``PrismaError`` (the DB IS reachable and rejected the data)
     must stay 401. These are always raised from prisma internals, so the check
     excludes any ``PrismaError`` by type before inspecting the traceback."""
-    data_layer_error = UniqueViolationError(
-        data={"user_facing_error": {"meta": {"table": "t"}}}
-    )
+    data_layer_error = UniqueViolationError(data={"user_facing_error": {"meta": {"table": "t"}}})
     with pytest.raises(UniqueViolationError) as exc_info:
         raise data_layer_error
     e = exc_info.value
@@ -284,9 +238,7 @@ def test_is_database_service_unavailable_error_excludes_non_infra(error):
     """Data-layer errors (the DB IS reachable and answered) and generic
     non-DB errors must NOT be classified as service-unavailable, otherwise a
     genuine 401 would be masked as a transient 503."""
-    assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False
-    )
+    assert PrismaDBExceptionHandler.is_database_service_unavailable_error(error) is False
 
 
 def _wrapped_like_get_user_object(original):
@@ -393,22 +345,14 @@ def test_is_database_service_unavailable_error_asyncpg(monkeypatch):
     monkeypatch.setitem(sys.modules, "asyncpg.exceptions", fake_exceptions)
 
     assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(
-            PostgresConnectionError("connection reset")
-        )
+        PrismaDBExceptionHandler.is_database_service_unavailable_error(PostgresConnectionError("connection reset"))
         is True
     )
     assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(
-            InterfaceError("connection was closed")
-        )
-        is True
+        PrismaDBExceptionHandler.is_database_service_unavailable_error(InterfaceError("connection was closed")) is True
     )
     assert (
-        PrismaDBExceptionHandler.is_database_service_unavailable_error(
-            UniqueViolationError("duplicate key")
-        )
-        is False
+        PrismaDBExceptionHandler.is_database_service_unavailable_error(UniqueViolationError("duplicate key")) is False
     )
 
 
@@ -675,7 +619,10 @@ def test_is_deadlock_error_excludes_non_deadlocks(error):
             "22021",
         ),
         (RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"message": "m"}}}), None),
-        (RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"code": 42, "message": "m"}}}), None),
+        (
+            RawQueryError(data={"user_facing_error": {"error_code": "P2010", "meta": {"code": 42, "message": "m"}}}),
+            None,
+        ),
         (prisma_errors.DataError(data={"user_facing_error": {"meta": None}}), None),
         (
             prisma_errors.DataError(
@@ -695,7 +642,9 @@ def test_is_deadlock_error_excludes_non_deadlocks(error):
         (httpx.ReadTimeout("no reply"), None),
     ],
 )
-def test_postgres_sqlstate_reads_the_code_prisma_attached_to_the_failed_statement(error: Exception, sqlstate: str | None):
+def test_postgres_sqlstate_reads_the_code_prisma_attached_to_the_failed_statement(
+    error: Exception, sqlstate: str | None
+):
     """Only a prisma data error carrying Postgres's own error code yields a SQLSTATE, whether in ``meta``
     or, for a batched statement, only in the message; a codeless or malformed payload, an engine-level
     error, and a transport error yield None."""
@@ -905,7 +854,7 @@ def _pool_timeout_error() -> DataError:
         RawQueryError(
             data={
                 "user_facing_error": {
-                    "message": 'Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`',
+                    "message": "Raw query failed. Code: `53300`. Message: `db error: FATAL: sorry, too many clients already`",
                     "meta": {"code": "53300", "message": "FATAL: sorry, too many clients already"},
                     "error_code": "P2010",
                 }
@@ -945,3 +894,60 @@ def test_capacity_error_is_seen_through_a_wrapping_exception() -> None:
     wrapped: Final = RuntimeError("spend flush failed")
     wrapped.__cause__ = _capacity_error("sorry, too many clients already")
     assert PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(wrapped) is True
+
+
+def _lock_timeout_error(
+    message: str = "canceling statement due to lock timeout", code: str | None = "55P03"
+) -> DataError:
+    """The shape prisma raises for a statement Postgres cancelled under ``lock_timeout``."""
+    user_facing: Final[dict[str, object]] = {"is_panic": False, "message": f"Error querying the database: {message}"}
+    if code is not None:
+        user_facing["meta"] = {"code": code, "message": message}
+    return DataError(data={"user_facing_error": user_facing})
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        _lock_timeout_error(),
+        _lock_timeout_error(message="Abbruch der Anweisung wegen Zeitüberschreitung beim Warten auf eine Sperre"),
+        _lock_timeout_error(code=None),
+        DataError(
+            data={
+                "user_facing_error": {
+                    "message": 'Error occurred during query execution: ConnectorError(ConnectorError { user_facing_error: None, kind: QueryError(PostgresError { code: "55P03", message: "canceling statement due to lock timeout", severity: "ERROR" }) })'
+                }
+            }
+        ),
+    ],
+    ids=["meta_55P03", "meta_55P03_localised_message", "message_only", "batched_statement"],
+)
+def test_is_lock_timeout_error_recognises_a_statement_cancelled_under_lock_timeout(error: DataError) -> None:
+    """SQLSTATE 55P03 means the statement never took its lock, so it never applied
+    and its rows are safe to re-send. It is a wait budget, not a full server, so
+    it stays out of the connection-capacity classification."""
+    assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is True
+    assert PrismaDBExceptionHandler.is_database_capacity_error(error) is False
+
+
+@pytest.mark.parametrize(
+    "error",
+    [
+        _lock_timeout_error(message="canceling statement due to statement timeout", code="57014"),
+        _lock_timeout_error(message="deadlock detected", code="40P01"),
+        _capacity_error("sorry, too many clients already"),
+        _pool_timeout_error(),
+        httpx.ConnectError("connection refused"),
+        RuntimeError("canceling statement due to lock timeout"),
+    ],
+    ids=[
+        "statement_timeout_57014",
+        "deadlock_40P01",
+        "capacity_53300",
+        "pool_timeout_P2024",
+        "transport",
+        "not_prisma",
+    ],
+)
+def test_is_lock_timeout_error_excludes_other_failures(error: Exception) -> None:
+    assert PrismaDBExceptionHandler.is_lock_timeout_error(error) is False

tests/unit/proxy/db/test_model_access_group_spend.py

@@ -64,6 +64,9 @@ class _FakeTransaction:
     def batch_(self) -> _FakeBatchManager:
         return _FakeBatchManager(self._batcher)
 
+    async def execute_raw(self, query: str, *args: object) -> int:
+        return 0
+
     async def __aenter__(self) -> "_FakeTransaction":
         return self
 

tests/unit/proxy/db/test_model_usage_rollup.py

@@ -6,6 +6,7 @@ from unittest.mock import MagicMock
 import httpx
 import pytest
 
+from litellm.proxy.db import rollup_lock_timeout
 from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
 from litellm.proxy.db.model_usage_rollup import (
     ModelUsageKey,
@@ -14,11 +15,13 @@ from litellm.proxy.db.model_usage_rollup import (
     flush_model_usage_transactions,
     model_usage_task_type,
 )
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
 
 
 class _FakeBatcher:
     def __init__(self) -> None:
         self.litellm_dailymodelusage = MagicMock()
+        self.execute_raw = MagicMock()
 
     async def __aenter__(self) -> "_FakeBatcher":
         return self
@@ -201,3 +204,26 @@ async def test_request_time_path_queues_usage_instead_of_writing_to_the_db() ->
 
     assert [transaction.key.model for transaction in prisma.model_usage_transactions] == ["openai/gpt-5.4-mini"]
     prisma.db.litellm_dailymodelusage.upsert.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_rollup_transaction_sets_the_lock_timeout_before_its_first_upsert(monkeypatch) -> None:
+    """The DailyModelUsage rollup shares the DailyToolSpend write shape, so it runs
+    under the same per-transaction lock budget, issued before any upsert queues."""
+    monkeypatch.setattr(rollup_lock_timeout, "SPEND_ROLLUP_LOCK_TIMEOUT_MS", 250)
+    batcher = _FakeBatcher()
+    prisma = _prisma(MagicMock(return_value=batcher))
+    order: list[str] = []
+    batcher.execute_raw.side_effect = lambda *_: order.append("lock_timeout")
+    batcher.litellm_dailymodelusage.upsert.side_effect = lambda **_: order.append("upsert")
+
+    await flush_model_usage_transactions(
+        prisma_client=prisma,
+        transactions=[
+            build_model_usage_transaction(_payload()),
+            build_model_usage_transaction(_payload(model="other")),
+        ],
+    )
+
+    assert batcher.execute_raw.call_args_list == [((ROLLUP_LOCK_TIMEOUT_SQL, "250ms"),)]
+    assert order == ["lock_timeout", "upsert", "upsert"]

tests/unit/proxy/db/test_prisma_query_span.py

@@ -106,6 +106,7 @@ def test_a_payload_the_parser_does_not_know_stays_the_legacy_function_named_span
         ),
         ('INSERT INTO "LiteLLM_DailyUserSpend" (id) VALUES ($1)', ("insert", "LiteLLM_DailyUserSpend")),
         ("SET LOCAL lock_timeout = 1000", ("set", "lock_timeout")),
+        ("SELECT set_config('lock_timeout', $1::text, true)", ("set", "lock_timeout")),
         ("SELECT COUNT(*) FROM pg_stat_activity", ("select", "pg_catalog")),
         ("SELECT 1", ("ping", None)),
         ("SELECT current_setting('transaction_read_only') AS transaction_read_only", ("select", "pg_catalog")),

tests/unit/proxy/db/test_spend_log_tool_index.py

@@ -4,22 +4,23 @@ only) and the flush that writes LiteLLM_SpendLogToolIndex in bounded statements
 plus the LiteLLM_DailyToolSpend rollup in one transaction.
 """
 
-from types import SimpleNamespace
 from collections.abc import Awaitable, Callable
+from types import SimpleNamespace
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock
 
 import httpx
 import pytest
 
-from litellm.proxy.db import spend_log_tool_index
+from litellm.proxy.db import rollup_lock_timeout, spend_log_tool_index
+from litellm.proxy.db.log_db_metrics import record_db_io
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
 from litellm.proxy.db.spend_log_tool_index import (
     ToolUsageTransaction,
     build_tool_usage_transaction,
     flush_tool_usage_transactions,
     response_tool_call_names,
 )
-from litellm.proxy.db.log_db_metrics import record_db_io
 from tests.unit.proxy.db.fake_prisma_engine import engine_call
 
 
@@ -32,6 +33,7 @@ class _FakeBatcher:
     def __init__(self) -> None:
         self.litellm_spendlogtoolindex = MagicMock()
         self.litellm_dailytoolspend = MagicMock()
+        self.execute_raw = MagicMock()
 
     async def __aenter__(self) -> "_FakeBatcher":
         return self
@@ -84,7 +86,9 @@ class TestBuildToolUsageTransaction:
                 mcp_namespaced_tool_name=None,
                 spend=0.5,
                 total_tokens=100,
-                completion_response=SimpleNamespace(choices=[SimpleNamespace(message=SimpleNamespace(tool_calls=None))]),
+                completion_response=SimpleNamespace(
+                    choices=[SimpleNamespace(message=SimpleNamespace(tool_calls=None))]
+                ),
             )
             is None
         )
@@ -395,3 +399,25 @@ async def test_a_tool_usage_flush_renders_one_postgres_span_per_table_written(
         "postgres.insert LiteLLM_SpendLogToolIndex",
         "postgres.upsert LiteLLM_DailyToolSpend",
     )
+
+
+@pytest.mark.asyncio
+async def test_rollup_transaction_sets_the_lock_timeout_before_its_first_upsert(monkeypatch) -> None:
+    """Every DailyToolSpend rollup transaction opens by bounding how long its upserts
+    may wait on a row another pod holds, so a waiter costs one cancelled statement
+    rather than a pooled connection pinned until the holder commits. The budget is the
+    SPEND_ROLLUP_LOCK_TIMEOUT_MS knob, bound as a parameter, not baked into the SQL."""
+    monkeypatch.setattr(rollup_lock_timeout, "SPEND_ROLLUP_LOCK_TIMEOUT_MS", 250)
+    prisma, batcher = _prisma_with_batcher()
+    order: list[str] = []
+    batcher.execute_raw.side_effect = lambda *_: order.append("lock_timeout")
+    batcher.litellm_dailytoolspend.upsert.side_effect = lambda **_: order.append("upsert")
+
+    await flush_tool_usage_transactions(
+        prisma_client=prisma,
+        transactions=[_transaction("r1", tool_names=("tool_a", "tool_b"))],
+    )
+
+    assert batcher.execute_raw.call_args_list == [((ROLLUP_LOCK_TIMEOUT_SQL, "250ms"),)]
+    assert order == ["lock_timeout", "upsert", "upsert"]
+    assert "set_config('lock_timeout'" in ROLLUP_LOCK_TIMEOUT_SQL and ", true)" in ROLLUP_LOCK_TIMEOUT_SQL

tests/unit/proxy/db/test_update_daily_tag_spend.py

@@ -1,12 +1,12 @@
-from typing import Dict
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 
-from litellm.proxy.utils import update_daily_tag_spend
 from litellm.proxy._types import DailyTagSpendTransaction
-import httpx
 from litellm.proxy.db.db_spend_update_writer import DBSpendUpdateWriter
+from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
+from litellm.proxy.utils import update_daily_tag_spend
 
 
 @pytest.mark.asyncio
@@ -18,9 +18,7 @@ async def test_update_daily_tag_spend_delegates_to_tag_commit_writer():
     proxy_logging_obj.db_spend_update_writer = MagicMock()
     proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer
     proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock()
-    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = (
-        AsyncMock()
-    )
+    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock()
 
     await update_daily_tag_spend(
         prisma_client,
@@ -43,12 +41,8 @@ async def test_update_daily_tag_spend_logs_error_and_does_not_raise():
     redis_update_buffer._should_commit_spend_updates_to_redis.return_value = False
     proxy_logging_obj.db_spend_update_writer = MagicMock()
     proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer
-    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock(
-        side_effect=ValueError("boom")
-    )
-    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = (
-        AsyncMock()
-    )
+    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock(side_effect=ValueError("boom"))
+    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock()
 
     with patch("litellm.proxy.utils.verbose_proxy_logger.error") as error_logger:
         await update_daily_tag_spend(
@@ -69,9 +63,7 @@ async def test_update_daily_tag_spend_uses_redis_writer_when_enabled():
     proxy_logging_obj.db_spend_update_writer = MagicMock()
     proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock()
     proxy_logging_obj.db_spend_update_writer.redis_update_buffer = redis_update_buffer
-    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = (
-        AsyncMock()
-    )
+    proxy_logging_obj.db_spend_update_writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock()
 
     await update_daily_tag_spend(
         prisma_client,
@@ -92,17 +84,20 @@ async def test_daily_tag_spend_retries_then_succeeds():
     proxy_logging_obj = MagicMock()
 
     # Fail the upsert 3 times with retryable DB errors, then succeed.
-    prisma_client.db.execute_raw = AsyncMock(
-        side_effect=[
-            httpx.ConnectError("x"),
-            httpx.ConnectError("x"),
-            httpx.ConnectError("x"),
-            1,
-        ]
-    )
+    upsert_outcomes = iter([httpx.ConnectError("x"), httpx.ConnectError("x"), httpx.ConnectError("x"), 1])
+
+    async def execute_raw(query: str, *args: object) -> int:
+        if query == ROLLUP_LOCK_TIMEOUT_SQL:
+            return 0
+        outcome = next(upsert_outcomes)
+        if isinstance(outcome, Exception):
+            raise outcome
+        return outcome
+
+    prisma_client.db.execute_raw = AsyncMock(side_effect=execute_raw)
     prisma_client.db.tx.return_value.__aenter__.return_value.execute_raw = prisma_client.db.execute_raw
 
-    daily_spend_transactions: Dict[str, DailyTagSpendTransaction] = {
+    daily_spend_transactions: dict[str, DailyTagSpendTransaction] = {
         "k": {
             "tag": "prod-tag",
             "date": "2026-04-03",
@@ -135,7 +130,8 @@ async def test_daily_tag_spend_retries_then_succeeds():
             daily_spend_transactions=daily_spend_transactions,
         )
 
-    assert prisma_client.db.execute_raw.await_count == 4
+    upsert_attempts = [c for c in prisma_client.db.execute_raw.await_args_list if c.args[0] != ROLLUP_LOCK_TIMEOUT_SQL]
+    assert len(upsert_attempts) == 4
     assert sleep_mock.await_count == 3
     # The batch is one statement, so the successful attempt is a single call carrying
     # the row rather than one call per key.

tests/unit/proxy/lens/test_inference.py

@@ -621,6 +621,101 @@ async def test_renewal_preserves_the_original_failure_while_request_cleanup_is_p
     assert db.stored.reservations == (hold,)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("stalled", ("budget", "model", "budget_wait"))
+async def test_request_deadline_expiry_returns_gateway_timeout(stalled: str, monkeypatch: pytest.MonkeyPatch) -> None:
+    import asyncio
+
+    from litellm.proxy.lens import inference
+    from litellm.proxy.lens.inference import BUDGET_LEASE, reserve_amount, reserved_budget
+    from litellm.proxy.lens.models import BudgetReservation
+    from litellm.proxy.lens.repository import LensRepository
+    from tests.unit.proxy.lens.test_endpoints import ResultDatabase
+    from tests.unit.proxy.lens.test_state import NOW, lens
+
+    budget_deadline: Final = inference.BUDGET_WAIT_TIMEOUT
+    request_deadline: Final = budget_deadline * 2 if stalled == "budget_wait" else budget_deadline / 2
+    monkeypatch.setattr(litellm, "request_timeout", request_deadline)
+    loop: Final = asyncio.get_running_loop()
+    monkeypatch.setattr(loop, "time", lambda: 0.0)
+    loop.call_soon(monkeypatch.setattr, loop, "time", lambda: min(request_deadline, budget_deadline) + 1)
+    db: Final = ResultDatabase(lens())
+    hold: Final = BudgetReservation(
+        id="active", job_id="run", amount=90, month=db.stored.budget_month, expires_at=NOW + BUDGET_LEASE
+    )
+    admitted: Final = asyncio.Event()
+
+    with pytest.raises(HTTPException) as error:
+        async with reserved_budget(
+            LensRepository(db),
+            "lens",
+            hold.id,
+            (lambda e: reserve_amount(e, hold, NOW)) if stalled == "model" else (lambda e: e),
+            admitted,
+        ):
+            await asyncio.Event().wait()
+    assert error.value.status_code == 504
+    assert error.value.detail == (
+        "Analysis request timed out waiting for budget"
+        if stalled == "budget_wait"
+        else "Analysis request timed out waiting for budget or model output"
+    )
+    assert isinstance(error.value.__cause__, asyncio.TimeoutError)
+    assert admitted.is_set() is (stalled == "model")
+
+
+@pytest.mark.asyncio
+async def test_renewal_deadline_cancels_stalled_database_and_model(monkeypatch: pytest.MonkeyPatch) -> None:
+    import asyncio
+    from collections.abc import AsyncGenerator
+    from contextlib import asynccontextmanager
+
+    from litellm.proxy.lens import inference
+    from litellm.proxy.lens.repository import Database, LensRepository, Row
+
+    interval: Final = inference.BUDGET_RENEW_INTERVAL
+    loop: Final = asyncio.get_running_loop()
+    monkeypatch.setattr(loop, "time", lambda: 0.0)
+    admitted: Final = asyncio.Event()
+    database_cancelled: Final = asyncio.Event()
+    model_cancelled: Final = asyncio.Event()
+
+    class StalledDatabase:
+        async def query_raw(self, query: str, *args: object) -> tuple[Row, ...]:
+            loop.call_soon(monkeypatch.setattr, loop, "time", lambda: 2 * (interval + 1))
+            try:
+                await asyncio.Event().wait()
+            finally:
+                database_cancelled.set()
+            return ()
+
+        async def execute_raw(self, query: str, *args: object) -> int:
+            pytest.fail("A stalled read cannot write")
+
+        @asynccontextmanager
+        async def transaction(self) -> AsyncGenerator[Database]:
+            yield self
+
+    async def model() -> tuple[ModelResponse, float | None]:
+        admitted.set()
+        loop.call_soon(monkeypatch.setattr, loop, "time", lambda: interval + 1)
+        try:
+            await asyncio.Event().wait()
+        finally:
+            model_cancelled.set()
+        pytest.fail("The stalled model must be cancelled when renewal times out")
+
+    with pytest.raises(HTTPException) as error:
+        await inference.model_with_renewal(
+            model(), inference.renew_budget_reservation(LensRepository(StalledDatabase()), "lens", "active", admitted)
+        )
+    assert error.value.status_code == 503
+    assert error.value.detail == "Analysis budget reservation renewal timed out"
+    assert isinstance(error.value.__cause__, asyncio.TimeoutError)
+    assert database_cancelled.is_set()
+    assert model_cancelled.is_set()
+
+
 @pytest.mark.asyncio
 async def test_completed_paid_response_survives_simultaneous_renewal_failure() -> None:
     from litellm.proxy.lens.inference import model_with_renewal

tests/unit/proxy/management_endpoints/test_auto_router_endpoints.py

@@ -1222,15 +1222,14 @@ class TestAutoRouterBenchmarks:
         assert [group.router_name for group in response.groups] == ["tagged"]
 
     def test_the_listed_kinds_match_the_router_types_traffic_can_record(self):
-        """The one reason semantic is excluded, pinned against both declarations: a kind the
-        rollup can record must be listable, and a kind it cannot must not be."""
         from typing import get_args, get_type_hints
 
         from litellm.router_utils.auto_router_model_naming import StrategyRouterKind
         from litellm.types.utils import StandardLoggingRoutingDecision
 
-        recorded = set(get_args(get_type_hints(StandardLoggingRoutingDecision)["router_type"]))
-        assert set(get_args(StrategyRouterKind)) - {"semantic"} == recorded
+        readonly_router_type: Final = get_type_hints(StandardLoggingRoutingDecision, include_extras=True)["router_type"]
+        recorded: Final = set(get_args(get_args(readonly_router_type)[0]))
+        assert set(get_args(StrategyRouterKind)) == recorded
 
 
 # ---------------------------------------------------------------------------

tests/unit/proxy/management_endpoints/test_model_insights_endpoints.py

@@ -225,6 +225,9 @@ class _InMemoryBatcher:
     def __init__(self, table: _InMemoryUsageTable) -> None:
         self.litellm_dailymodelusage = table
 
+    def execute_raw(self, query: str, *args: object) -> None:
+        return None
+
     async def __aenter__(self) -> "_InMemoryBatcher":
         return self
 

tests/unit/proxy/management_endpoints/test_model_management_endpoints.py

@@ -6097,9 +6097,9 @@ class TestStrategyRouterWriteValidation:
             AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,
             _auto_router_capability_slot,
         )
-        from litellm.router_utils.auto_router_model_naming import gated_capability_of
+        from litellm.router_utils.auto_router_model_naming import gated_capabilities_of
 
-        capability = gated_capability_of(effective_params)
+        capabilities: Final = gated_capabilities_of(effective_params)
 
         fake = self._FakeDb(db_models)
         live_router = self._live_router_holding_one_capability(limit, config_config) if config_config is not None else None
@@ -6116,9 +6116,9 @@ class TestStrategyRouterWriteValidation:
                     async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id):
                         pass
                 assert exc_info.value.status_code == 403
-                assert capability is not None
+                assert len(capabilities) == 1
                 assert "At most 1 auto-router" in str(exc_info.value.detail)
-                assert capability.subject in str(exc_info.value.detail)
+                assert capabilities[0].subject in str(exc_info.value.detail)
                 assert "'auto_router' feature lifts the limit" in str(exc_info.value.detail)
                 return
             async with _auto_router_capability_slot(fake, effective_params=effective_params, model_id=model_id) as tables:
@@ -6135,8 +6135,59 @@ class TestStrategyRouterWriteValidation:
         assert lock_params == (AUTO_ROUTER_CAPABILITY_SLOT_LOCK_KEY,)
         assert count_params == (model_id or "",)
         assert "AS model" in count_sql
-        assert capability is not None
-        assert capability.sql_config_predicate.split("{config}")[-1].strip() in count_sql
+        assert len(capabilities) == 1
+        assert capabilities[0].sql_config_predicate.split("{config}")[-1].strip() in count_sql
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
+    @pytest.mark.parametrize(
+        ("other_config", "denied_subject"),
+        (
+            (None, None),
+            (_V2, "heuristic_v2"),
+            (_CUSTOM_PROMPT, "operator-written classifier prompt"),
+        ),
+    )
+    async def test_v2_chain_reserves_both_capabilities_under_one_lock(
+        self, classifier: str, other_config: Mapping[str, object] | None, denied_subject: str | None
+    ) -> None:
+        from fastapi import HTTPException
+
+        from litellm.proxy.management_endpoints.model_management_endpoints import _auto_router_capability_slot
+
+        candidate: Final = {
+            "model": "auto_router/complexity_router",
+            "complexity_router_config": {
+                **self._CUSTOM_PROMPT,
+                "classifier_type": classifier,
+                "local_heuristic": "heuristic_v2",
+                **(
+                    {"heuristic_first_max_tier": "MEDIUM"}
+                    if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
+                ),
+            },
+        }
+        fake: Final = self._FakeDb([])
+        router: Final = self._live_router_holding_one_capability(1, other_config) if other_config is not None else None
+        with (
+            patch("litellm.proxy.proxy_server._license_check.auto_router_capability_limit", return_value=1),  # test-quality-ok: [TQ008] the guard reads the proxy license singleton
+            patch("litellm.proxy.proxy_server.llm_router", router),  # test-quality-ok: [TQ008] the guard reads the proxy router singleton
+            patch("litellm.proxy.proxy_server.heuristic_v1_tuning_baselines", {}),  # test-quality-ok: [TQ008] the guard reads the proxy baseline singleton
+            patch("litellm.proxy.management_endpoints.model_management_endpoints.publish_config_change", new=AsyncMock()),  # test-quality-ok: [TQ008] Redis publication is outside admission under test
+        ):
+            if denied_subject is not None:
+                with pytest.raises(HTTPException) as failure:
+                    async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id"):
+                        pytest.fail("Exhausted capability allowed a write")
+                assert failure.value.status_code == 403
+                assert denied_subject in str(failure.value.detail)
+            else:
+                async with _auto_router_capability_slot(fake, effective_params=candidate, model_id="held-id") as table:
+                    assert table is fake.tx_obj.litellm_proxymodeltable
+        assert sum("pg_advisory_xact_lock" in sql for sql, _ in fake.tx_obj.raw_calls) == 1
+        counts: Final = tuple((sql, args) for sql, args in fake.tx_obj.raw_calls if "AS model" in sql)
+        assert len(counts) == (1 if denied_subject == "heuristic_v2" else 2)
+        assert all(args == ("held-id",) for _, args in counts)
 
     _TUNED_A = {"classifier_type": "heuristic", "tiers": {"SIMPLE": "gpt-4o-mini", "MEDIUM": "gpt-4o"}}
     _TUNED_A_EDITED = {**_TUNED_A, "dimension_weights": {"codePresence": 0.9}}

tests/unit/proxy/management_helpers/test_auto_router_availability.py

@@ -61,6 +61,33 @@ def test_edit_does_not_exempt_another_classifier_allowance() -> None:
     assert next(slot for slot in result.allowances if slot.key == "llm_v2").remaining == 0
 
 
+@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
+@pytest.mark.parametrize(
+    ("other_classifier", "expected_label"),
+    (("heuristic_v2", "Heuristic v2"), ("llm", "Custom tiers or classifier instructions")),
+)
+def test_v2_chain_checks_both_allowances_and_keeps_both_owned_slots(
+    classifier: str, other_classifier: str, expected_label: str
+) -> None:
+    prompt: Final = {"classification_prompt": "Grade by difficulty"}
+    candidate: Final = deployment("chain", classifier, config={"local_heuristic": "heuristic_v2", **prompt})
+    blocked: Final = auto_router_availability(
+        others=(deployment("other", other_classifier, config=prompt),),
+        existing=None,
+        candidate=candidate,
+        baselines={},
+        limit=1,
+    )
+    assert blocked.error is not None and expected_label in blocked.error
+    edited: Final = auto_router_availability(
+        others=(), existing=candidate, candidate=candidate, baselines={}, limit=1
+    )
+    assert edited.error is None
+    assert tuple(slot.key for slot in edited.allowances if slot.used_by_this_router) == (
+        "heuristic_v2", "tier_or_classifier_prompt"
+    )
+
+
 def test_model_selection_does_not_claim_occupied_scoring_allowance() -> None:
     original: Final = deployment("legacy", "heuristic")
     changed: Final = deployment("other", "heuristic", tuned=True)

tests/unit/proxy/test_tracing_endpoints.py

@@ -86,6 +86,7 @@ TRACE_RESPONSE: Final = {
         "agent_count": 0,
         "agent_invocations": 0,
         "llm_calls": 0,
+        "priced_calls": 0,
         "tool_calls": 0,
         "error_count": 0,
         "input_tokens": 0,

tests/unit/proxy/utils/prisma_and_spend/test_spend_functions.py

@@ -15,7 +15,7 @@ import json
 from collections.abc import Callable
 from contextlib import suppress
 from datetime import datetime, timezone
-from typing import Any, Dict, Final, List
+from typing import Any, Final
 from unittest.mock import AsyncMock, MagicMock
 
 import pytest
@@ -111,15 +111,11 @@ async def test_update_daily_tag_spend_redis_path_when_buffered(
     writer = MagicMock()
     proxy_logging.db_spend_update_writer = writer
     writer.redis_update_buffer = MagicMock()
-    writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(
-        return_value=True
-    )
+    writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(return_value=True)
     writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock()
     writer._commit_daily_tag_spend_to_db = AsyncMock()
 
-    await update_daily_tag_spend(
-        prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging
-    )
+    await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
     redis_kwargs = writer._commit_daily_tag_spend_to_db_with_redis.await_args.kwargs
     pinned = {
         "redis_calls": writer._commit_daily_tag_spend_to_db_with_redis.await_count,
@@ -130,9 +126,7 @@ async def test_update_daily_tag_spend_redis_path_when_buffered(
     assert pinned == {
         "redis_calls": 1,
         "direct_calls": 0,
-        "redis_kwargs_keys": sorted(
-            ["prisma_client", "n_retry_times", "proxy_logging_obj"]
-        ),
+        "redis_kwargs_keys": sorted(["prisma_client", "n_retry_times", "proxy_logging_obj"]),
         "redis_n_retries": 3,
     }
 
@@ -145,15 +139,11 @@ async def test_update_daily_tag_spend_direct_path_when_no_redis(
     writer = MagicMock()
     proxy_logging.db_spend_update_writer = writer
     writer.redis_update_buffer = MagicMock()
-    writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(
-        return_value=False
-    )
+    writer.redis_update_buffer._should_commit_spend_updates_to_redis = MagicMock(return_value=False)
     writer._commit_daily_tag_spend_to_db_with_redis = AsyncMock()
     writer._commit_daily_tag_spend_to_db = AsyncMock()
 
-    await update_daily_tag_spend(
-        prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging
-    )
+    await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
     assert writer._commit_daily_tag_spend_to_db.await_count == 1
     assert writer._commit_daily_tag_spend_to_db_with_redis.await_count == 0
 
@@ -175,9 +165,7 @@ async def test_update_daily_tag_spend_logs_and_swallows_errors(
     proxy_logging.db_spend_update_writer._commit_daily_tag_spend_to_db = AsyncMock(
         side_effect=RuntimeError("commit boom")
     )
-    await update_daily_tag_spend(
-        prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging
-    )
+    await update_daily_tag_spend(prisma_client=mock_prisma_client, proxy_logging_obj=proxy_logging)
 
 
 @pytest.mark.asyncio
@@ -265,15 +253,11 @@ async def test_update_spend_logs_job_processes_and_clears_queue(
     mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock()
 
     # Stub auxiliary imports so the test focuses on the spend-logs write path.
-    import litellm.proxy.guardrails.usage_tracking as guard_mod
     import litellm.proxy.db.spend_log_tool_index as tool_mod
+    import litellm.proxy.guardrails.usage_tracking as guard_mod
 
-    monkeypatch.setattr(
-        guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
-    )
-    monkeypatch.setattr(
-        tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False
-    )
+    monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
+    monkeypatch.setattr(tool_mod, "flush_tool_usage_transactions", AsyncMock(), raising=False)
 
     await update_spend_logs_job(
         prisma_client=mock_prisma_client,
@@ -283,12 +267,10 @@ async def test_update_spend_logs_job_processes_and_clears_queue(
     pinned = {
         "create_many_calls": mock_prisma_client.db.litellm_spendlogs.create_many.await_count,
         "queue_after": mock_prisma_client.spend_log_transactions,
-        "first_data_request_id": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs[
-            "data"
-        ][0]["request_id"],
-        "skip_duplicates_set": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs[
-            "skip_duplicates"
+        "first_data_request_id": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["data"][0][
+            "request_id"
         ],
+        "skip_duplicates_set": mock_prisma_client.db.litellm_spendlogs.create_many.await_args.kwargs["skip_duplicates"],
     }
     assert pinned == {
         "create_many_calls": 1,
@@ -315,9 +297,7 @@ async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled(
         mock_prisma_client.spend_log_transactions.append(row_arriving_mid_flush)
         raise asyncio.CancelledError()
 
-    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
-        side_effect=_cancel_mid_write
-    )
+    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_cancel_mid_write)
 
     with pytest.raises(asyncio.CancelledError):
         await update_spend_logs_job(
@@ -326,9 +306,7 @@ async def test_update_spend_logs_job_requeues_popped_rows_when_write_cancelled(
             proxy_logging_obj=proxy_logging,
         )
 
-    assert [
-        row["request_id"] for row in mock_prisma_client.spend_log_transactions
-    ] == ["r1", "r2", "r3"]
+    assert [row["request_id"] for row in mock_prisma_client.spend_log_transactions] == ["r1", "r2", "r3"]
 
 
 @pytest.mark.asyncio
@@ -369,12 +347,8 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
     import litellm.proxy.db.spend_log_tool_index as tool_mod
     import litellm.proxy.guardrails.usage_tracking as guard_mod
 
-    monkeypatch.setattr(
-        guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
-    )
-    monkeypatch.setattr(
-        tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
-    )
+    monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
+    monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
 
     proxy_logging = MagicMock()
     proxy_logging.failure_handler = AsyncMock()
@@ -385,9 +359,7 @@ async def test_drain_spend_logs_queue_flushes_rows_queued_while_draining(
     async def _write(*args: Any, **kwargs: Any) -> None:
         written.extend(row["request_id"] for row in kwargs["data"])
         if len(written) == 1:
-            mock_prisma_client.spend_log_transactions.append(
-                make_spend_log_row(request_id="r2")
-            )
+            mock_prisma_client.spend_log_transactions.append(make_spend_log_row(request_id="r2"))
 
     mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write)
 
@@ -408,12 +380,8 @@ async def test_drain_spend_logs_queue_stops_monitor_and_keeps_its_popped_rows(
     import litellm.proxy.db.spend_log_tool_index as tool_mod
     import litellm.proxy.guardrails.usage_tracking as guard_mod
 
-    monkeypatch.setattr(
-        guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
-    )
-    monkeypatch.setattr(
-        tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
-    )
+    monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
+    monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
 
     proxy_logging = MagicMock()
     proxy_logging.failure_handler = AsyncMock()
@@ -460,12 +428,8 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes(
     import litellm.proxy.db.spend_log_tool_index as tool_mod
     import litellm.proxy.guardrails.usage_tracking as guard_mod
 
-    monkeypatch.setattr(
-        guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False
-    )
-    monkeypatch.setattr(
-        tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False
-    )
+    monkeypatch.setattr(guard_mod, "process_spend_logs_guardrail_usage", AsyncMock(), raising=False)
+    monkeypatch.setattr(tool_mod, "process_spend_logs_tool_usage", AsyncMock(), raising=False)
 
     proxy_logging = MagicMock()
     proxy_logging.failure_handler = AsyncMock()
@@ -474,9 +438,7 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes(
     async def _write_and_refill(*args: Any, **kwargs: Any) -> None:
         mock_prisma_client.spend_log_transactions.append(make_spend_log_row())
 
-    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(
-        side_effect=_write_and_refill
-    )
+    mock_prisma_client.db.litellm_spendlogs.create_many = AsyncMock(side_effect=_write_and_refill)
 
     await drain_spend_logs_queue(
         prisma_client=mock_prisma_client,
@@ -484,10 +446,7 @@ async def test_drain_spend_logs_queue_gives_up_after_max_passes(
         proxy_logging_obj=proxy_logging,
     )
 
-    assert (
-        mock_prisma_client.db.litellm_spendlogs.create_many.await_count
-        == MAX_SPEND_LOG_DRAIN_ITERATIONS
-    )
+    assert mock_prisma_client.db.litellm_spendlogs.create_many.await_count == MAX_SPEND_LOG_DRAIN_ITERATIONS
 
 
 @pytest.mark.asyncio
@@ -496,8 +455,8 @@ async def test_monitor_spend_logs_queue_invokes_job_when_queue_nonempty(
     make_spend_log_row: Any,
     monkeypatch: pytest.MonkeyPatch,
 ) -> None:
-    import litellm.proxy.utils as utils_mod
     import litellm.constants as constants_mod
+    import litellm.proxy.utils as utils_mod
 
     monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False)
     monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_SIZE_THRESHOLD", 1, raising=False)
@@ -530,8 +489,8 @@ async def test_monitor_spend_logs_queue_swallows_errors_and_backs_off(
     """An exception inside the loop is logged with backoff and the loop
     continues running rather than crashing the monitor task.
     """
-    import litellm.proxy.utils as utils_mod
     import litellm.constants as constants_mod
+    import litellm.proxy.utils as utils_mod
 
     monkeypatch.setattr(constants_mod, "SPEND_LOG_QUEUE_POLL_INTERVAL", 0.0, raising=False)
 
@@ -721,8 +680,7 @@ def test_raise_failed_update_spend_exception_emits_failure_handler() -> None:
             else None
         ),
         "non_blocking_in_traceback": (
-            "Non-Blocking"
-            in proxy_logging.failure_handler.call_args.kwargs["traceback_str"]
+            "Non-Blocking" in proxy_logging.failure_handler.call_args.kwargs["traceback_str"]
             if proxy_logging.failure_handler.call_args
             else False
         ),
@@ -1086,3 +1044,152 @@ async def test_tool_usage_flush_still_drops_ambiguous_failures(
     )
 
     assert mock_prisma_client.tool_usage_transactions == []
+
+
+def _postgres_lock_timeout() -> Exception:
+    """prisma's shape for Postgres SQLSTATE 55P03: the rollup upsert waited past
+    ``lock_timeout`` for a row another pod held and was cancelled before it took the lock."""
+    from prisma.errors import DataError
+
+    return DataError(
+        data={
+            "user_facing_error": {
+                "is_panic": False,
+                "message": "Error querying the database: canceling statement due to lock timeout",
+                "meta": {"code": "55P03", "message": "canceling statement due to lock timeout"},
+            }
+        }
+    )
+
+
+@pytest.mark.asyncio
+async def test_tool_usage_flush_requeues_when_the_rollup_row_lock_times_out(
+    mock_prisma_client: MagicMock,
+) -> None:
+    """A rollup transaction cancelled by ``lock_timeout`` never applied: the waiter
+    gives its connection back instead of pinning it behind the holder, and its
+    increments go back to the queue head for the next flush rather than being dropped.
+    Unlike a full server, one contended row says nothing about the other writers, so the
+    job carries on: the model usage rollup still opens its own transaction and the
+    auto-router drain still runs."""
+    mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock()
+    rollup_batch = MagicMock()
+    rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock())
+    rollup_batch.__aexit__ = AsyncMock(side_effect=_postgres_lock_timeout())
+    mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch)
+    mock_prisma_client.spend_log_transactions = []
+    first, second = _tool_usage_transaction("r1"), _tool_usage_transaction("r2")
+    mock_prisma_client.tool_usage_transactions = [first, second]
+    mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")]
+    mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
+
+    await update_spend_logs_job(
+        prisma_client=mock_prisma_client,
+        db_writer_client=None,
+        proxy_logging_obj=MagicMock(),
+    )
+
+    assert {
+        "rollup_transactions": mock_prisma_client.db.batch_.call_count,
+        "queue_after": mock_prisma_client.tool_usage_transactions,
+        "autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions,
+    } == {"rollup_transactions": 2, "queue_after": [first, second], "autorouter_queue_after": []}
+
+
+def _model_usage_transaction(model: str) -> object:
+    from litellm.proxy.db.model_usage_rollup import ModelUsageKey, ModelUsageTransaction
+
+    return ModelUsageTransaction(
+        key=ModelUsageKey(
+            date="2026-10-03", model_group="gpt", model=model, custom_llm_provider="openai", task_type="chat"
+        ),
+        spend=0.001,
+        prompt_tokens=1,
+        completion_tokens=1,
+        successful=True,
+    )
+
+
+def _model_usage_batch_failing_with(error: Exception, mock_prisma_client: MagicMock) -> MagicMock:
+    """Point the model usage rollup's ``batch_()`` at a transaction that fails on commit, with
+    no tool usage queued so the tool rollup never opens one."""
+    mock_prisma_client.db.litellm_spendlogtoolindex.create_many = AsyncMock()
+    mock_prisma_client.tool_usage_transactions = []
+    mock_prisma_client.spend_log_transactions = []
+    rollup_batch = MagicMock()
+    rollup_batch.__aenter__ = AsyncMock(return_value=MagicMock())
+    rollup_batch.__aexit__ = AsyncMock(side_effect=error)
+    mock_prisma_client.db.batch_ = MagicMock(return_value=rollup_batch)
+    return rollup_batch
+
+
+@pytest.mark.asyncio
+async def test_model_usage_flush_requeues_and_carries_on_when_its_row_lock_times_out(
+    mock_prisma_client: MagicMock,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """The DailyModelUsage rollup shares its rows across every pod exactly like the tool rollup.
+    A batch cancelled under ``lock_timeout`` never applied, so its transactions go back to the
+    queue head for the next flush instead of being dropped, and because only that rollup's row
+    was contended the job still drains the auto-router queue behind it."""
+    monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None))
+    _model_usage_batch_failing_with(_postgres_lock_timeout(), mock_prisma_client)
+    first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini")
+    mock_prisma_client.model_usage_transactions = [first, second]
+    mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
+
+    await update_spend_logs_job(
+        prisma_client=mock_prisma_client,
+        db_writer_client=None,
+        proxy_logging_obj=MagicMock(),
+    )
+
+    assert {
+        "queue_after": mock_prisma_client.model_usage_transactions,
+        "autorouter_queue_after": mock_prisma_client.autorouter_turn_transactions,
+    } == {"queue_after": [first, second], "autorouter_queue_after": []}
+
+
+@pytest.mark.asyncio
+async def test_model_usage_flush_requeues_and_stops_the_job_when_postgres_is_out_of_connections(
+    mock_prisma_client: MagicMock,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """A batch refused for want of a connection (53300) was never sent either, so it is kept, but
+    a full server will refuse the writers behind it too: the job stops there like the tool
+    rollup does, so the drain loop backs off instead of re-hitting the server."""
+    monkeypatch.setattr("litellm.proxy.db.model_usage_rollup.asyncio.sleep", AsyncMock(return_value=None))
+    _model_usage_batch_failing_with(_postgres_out_of_connections(), mock_prisma_client)
+    first, second = _model_usage_transaction("gpt-4o"), _model_usage_transaction("gpt-4o-mini")
+    mock_prisma_client.model_usage_transactions = [first, second]
+    mock_prisma_client.autorouter_turn_transactions = [MagicMock(name="autorouter_turn")]
+
+    with pytest.raises(Exception, match="too many clients already"):
+        await update_spend_logs_job(
+            prisma_client=mock_prisma_client,
+            db_writer_client=None,
+            proxy_logging_obj=MagicMock(),
+        )
+
+    assert {
+        "queue_after": mock_prisma_client.model_usage_transactions,
+        "autorouter_queue_after": len(mock_prisma_client.autorouter_turn_transactions),
+    } == {"queue_after": [first, second], "autorouter_queue_after": 1}
+
+
+@pytest.mark.asyncio
+async def test_model_usage_flush_still_drops_a_batch_that_may_have_applied(
+    mock_prisma_client: MagicMock,
+) -> None:
+    """Any other failure is ambiguous about whether the increments landed, so replaying it could
+    double-count: the batch is logged and dropped, and the job carries on."""
+    _model_usage_batch_failing_with(RuntimeError("engine lost mid-commit"), mock_prisma_client)
+    mock_prisma_client.model_usage_transactions = [_model_usage_transaction("gpt-4o")]
+
+    await update_spend_logs_job(
+        prisma_client=mock_prisma_client,
+        db_writer_client=None,
+        proxy_logging_obj=MagicMock(),
+    )
+
+    assert mock_prisma_client.model_usage_transactions == []

tests/unit/responses/test_streaming_iterator.py

@@ -646,7 +646,15 @@ def test_stream_cache_write_completes_when_asyncio_run_closes_the_loop(monkeypat
             status="completed",
             model="test-model",
             object="response",
-            output=[],
+            output=[
+                {
+                    "type": "message",
+                    "id": "msg_lit6184",
+                    "status": "completed",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": "cached", "annotations": []}],
+                }
+            ],
         ),
     )
     monkeypatch.setattr(litellm, "cache", _SlowWriteCache())
@@ -1261,6 +1269,25 @@ def test_billed_terminal_response_copies_when_estimating_and_leaves_the_original
     assert response.usage is None
 
 
+def test_persist_completed_response_to_cache_skips_a_response_without_output(monkeypatch):
+    logging_obj: Final = _logging_obj_stub()
+    caching_handler: Final = Mock()
+    caching_handler.request_kwargs = {"stream": True}
+    logging_obj._llm_caching_handler = caching_handler
+    iterator: Final = _make_iterator(sse_events=[], logging_obj=logging_obj)
+    iterator.completed_response = ResponseCompletedEvent.model_construct(
+        type="response.completed",
+        response=ResponsesAPIResponse.model_construct(id="resp_empty", output=[], usage=None),
+    )
+    cache: Final = Mock()
+    monkeypatch.setattr(litellm, "cache", cache)
+
+    iterator._persist_completed_response_to_cache(is_async=False)
+
+    cache.add_cache.assert_not_called()
+    caching_handler._should_store_result_in_cache.assert_not_called()
+
+
 def test_persist_completed_response_to_cache_survives_an_unserializable_response(monkeypatch):
     bad_response: Final = ResponsesAPIResponse.model_construct(id="r", output=[object()], usage=None)
     with pytest.raises(PydanticSerializationError):

tests/unit/router_strategy/test_auto_router.py

@@ -6,6 +6,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import pytest
 
 
+from litellm.exceptions import Timeout as LiteLLMTimeout
 from litellm.router_strategy.auto_router.auto_router import AutoRouter
 
 pytestmark_skip_beta = pytest.mark.skip(
@@ -342,6 +343,8 @@ class FixedRouteLayer:
         self.route_choice = route_choice
 
     async def acall(self, vector: Any) -> Any:
+        if isinstance(self.route_choice, BaseException):
+            raise self.route_choice
         return self.route_choice
 
 
@@ -372,11 +375,15 @@ class StubEmbeddingRouter:
 class FailingEmbeddingRouter(StubEmbeddingRouter):
     """Router whose query embedding fails, as it does when the prompt exceeds the encoder's window."""
 
-    async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any:
-        raise ValueError(
+    def __init__(self, error: Exception | None = None) -> None:
+        super().__init__()
+        self.error = error or ValueError(
             "litellm.InternalServerError: input is too large to process. increase the physical batch size"
         )
 
+    async def aembedding(self, input: List[str], model: str, **kwargs: Any) -> Any:
+        raise self.error
+
 
 def _auto_router(routelayer: Any, litellm_router_instance: Any = None, **kwargs: Any) -> AutoRouter:
     auto_router: Final = AutoRouter(
@@ -395,24 +402,35 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
     """The hook returns the alias's default model instead of failing or leaking the alias downstream."""
 
     @pytest.mark.asyncio
-    async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self):
+    @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
+    async def test_should_fall_back_to_default_model_when_the_embedding_call_fails(self, request_model: str) -> None:
         auto_router: Final = _auto_router(FixedRouteLayer(None), litellm_router_instance=FailingEmbeddingRouter())
 
         result: Final = await auto_router.async_pre_routing_hook(
-            model="my-auto-router",
+            model=request_model,
             request_kwargs={},
             messages=[{"role": "user", "content": "a" * 100_000}],
         )
 
         assert result is not None
         assert result.model == "fallback-model"
+        assert result.routing_decision == {
+            "router_model_name": "my-auto-router",
+            "router_type": "semantic",
+            "routed_model": "fallback-model",
+            "cause": "semantic_error",
+            "classifier_model": auto_router.embedding_model,
+            "classifier_failure_reason": "classifier_error",
+            "classifier_error_type": "ValueError",
+        }
 
     @pytest.mark.asyncio
-    async def test_should_fall_back_to_default_model_when_no_route_matches(self):
+    @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
+    async def test_should_fall_back_to_default_model_when_no_route_matches(self, request_model: str) -> None:
         auto_router: Final = _auto_router(FixedRouteLayer(None))
 
         result: Final = await auto_router.async_pre_routing_hook(
-            model="my-auto-router",
+            model=request_model,
             request_kwargs={},
             messages=[{"role": "user", "content": "nothing like any route"}],
         )
@@ -420,6 +438,13 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
         assert result is not None
         # Leaving "my-auto-router" here fails downstream with "Unmapped LLM provider".
         assert result.model == "fallback-model"
+        assert result.routing_decision == {
+            "router_model_name": "my-auto-router",
+            "router_type": "semantic",
+            "routed_model": "fallback-model",
+            "cause": "semantic_no_match",
+            "classifier_model": auto_router.embedding_model,
+        }
 
     @pytest.mark.asyncio
     async def test_should_fall_back_to_default_model_when_the_route_layer_returns_an_empty_list(self):
@@ -452,14 +477,15 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
         assert result.model == "code-model"
 
     @pytest.mark.asyncio
-    async def test_should_still_route_to_the_matched_route_when_one_matches(self):
+    @pytest.mark.parametrize("request_model", ["my-auto-router", "public-router-alias"])
+    async def test_should_still_route_to_the_matched_route_when_one_matches(self, request_model: str) -> None:
         from semantic_router.schema import RouteChoice
 
         router: Final = StubEmbeddingRouter()
         auto_router: Final = _auto_router(FixedRouteLayer(RouteChoice(name="code-model")), litellm_router_instance=router)
 
         result: Final = await auto_router.async_pre_routing_hook(
-            model="my-auto-router",
+            model=request_model,
             request_kwargs={},
             messages=[{"role": "user", "content": "fix this stack trace"}],
         )
@@ -467,6 +493,43 @@ class TestAutoRouterAlwaysResolvesARoutableModel:
         assert result is not None
         assert result.model == "code-model"
         assert router.seen_text == "fix this stack trace"
+        assert result.routing_decision == {
+            "router_model_name": "my-auto-router",
+            "router_type": "semantic",
+            "routed_model": "code-model",
+            "cause": "semantic_match",
+            "classifier_model": auto_router.embedding_model,
+        }
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("error", (
+        TimeoutError("private request content"),
+        asyncio.TimeoutError("private request content"),
+        LiteLLMTimeout(message="private request content", model="embedding", llm_provider="openai"),
+    ))
+    @pytest.mark.parametrize("wrapped", [False, True])
+    async def test_should_record_timeout_without_exception_message(self, error: Exception, wrapped: bool) -> None:
+        auto_router: Final = _auto_router(
+            FixedRouteLayer(None if wrapped else error),
+            litellm_router_instance=FailingEmbeddingRouter(error) if wrapped else StubEmbeddingRouter(),
+        )
+        result: Final = await auto_router.async_pre_routing_hook(
+            model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}]
+        )
+        assert result is not None and result.model == "fallback-model"
+        assert result.routing_decision is not None
+        assert result.routing_decision["cause"] == "semantic_error"
+        assert result.routing_decision["classifier_failure_reason"] == "timeout"
+        assert result.routing_decision["classifier_error_type"] == type(error).__name__
+        assert "private request content" not in str(result.routing_decision)
+
+    @pytest.mark.asyncio
+    async def test_should_propagate_cancellation_during_matching(self) -> None:
+        auto_router: Final = _auto_router(FixedRouteLayer(asyncio.CancelledError()))
+        with pytest.raises(asyncio.CancelledError):
+            await auto_router.async_pre_routing_hook(
+                model="my-auto-router", request_kwargs={}, messages=[{"role": "user", "content": "route this"}]
+            )
 
 
 class TestAutoRouterEmbeddingInputCap:

tests/unit/router_strategy/test_complexity_router.py

@@ -20,11 +20,12 @@ from unittest.mock import AsyncMock, MagicMock, patch
 import httpx
 import pytest
 import respx
-from pydantic import ValidationError
+from pydantic import TypeAdapter, ValidationError
 
 import litellm
 from litellm import Router
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.types.utils import StandardLoggingRoutingDecision
 from litellm.router_utils.auto_router_model_naming import (
     CUSTOMIZATION_CAPABILITY,
     GATED_AUTO_ROUTER_CAPABILITIES,
@@ -484,6 +485,10 @@ class TestComplexityRouterInit:
         assert first.cause != "jev_classifier"
         assert second.cause != "jev_classifier"
         assert client.calls == 1
+        assert first.classifier_failure_reason == "timeout"
+        assert first.classifier_error_type == "TimeoutError"
+        assert second.classifier_failure_reason == "circuit_open"
+        assert second.classifier_error_type is None
         assert _CLASSIFIER_CIRCUIT_OPEN_SIGNAL in second.signals
 
     @pytest.mark.asyncio
@@ -518,6 +523,8 @@ class TestComplexityRouterInit:
         outcome = await router.aclassify("Explain this")
 
         assert outcome.cause != "jev_classifier"
+        assert outcome.classifier_failure_reason == "classifier_error"
+        assert outcome.classifier_error_type == ("RuntimeError" if isinstance(response, RuntimeError) else "ValueError")
 
 
 class TestTokenScoring:
@@ -2120,6 +2127,37 @@ class TestRouterComplexityDeploymentMethods:
             "model_info": {"id": model_id},
         }
 
+    @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
+    @pytest.mark.parametrize("held_capability", ("heuristic_v2", "custom_prompt"))
+    def test_v2_chain_registration_enforces_each_claimed_allowance(
+        self, classifier: str, held_capability: str
+    ) -> None:
+        held: Final = (
+            self._router_row("existing", "existing-id", "heuristic_v2")
+            if held_capability == "heuristic_v2"
+            else self._custom_prompt_row("existing", "existing-id")
+        )
+        chain: Final = {
+            "model_name": "chain",
+            "model_info": {"id": "chain-id"},
+            "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {
+                    "classifier_type": classifier,
+                    "local_heuristic": "heuristic_v2",
+                    **(
+                        {"heuristic_first_max_tier": "MEDIUM"}
+                        if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
+                    ),
+                    "classifier_llm_config": {"model": "gpt-4o-mini", "system_prompt": "Grade by difficulty"},
+                    "tiers": {"SIMPLE": "gpt-4o-mini"},
+                },
+            },
+        }
+        subject: Final = "heuristic_v2" if held_capability == "heuristic_v2" else "operator-written classifier prompt"
+        with pytest.raises(ValueError, match=subject):
+            Router(model_list=[self._POOL, held, chain], auto_router_capability_limit=lambda: 1)
+
     @pytest.mark.parametrize("field", ["classification_prompt", "classification_examples"])
     def test_operator_written_prompt_sections_claim_the_customization_slot(self, field: str) -> None:
         """The dashboard prompt editor writes opening instructions and calibration examples as their own
@@ -3311,6 +3349,8 @@ class TestCapabilityClassifier:
         assert outcome.tier == ComplexityTier.REASONING
         assert outcome.cause == "capability_classifier_fallback"
         assert outcome.signals == ("capability-classifier-fallback",)
+        assert outcome.classifier_failure_reason == "invalid_response"
+        assert outcome.classifier_error_type == "ValidationError"
 
     @pytest.mark.asyncio
     async def test_classifier_call_failure_fails_closed_to_capable_model(self, mock_router_instance):
@@ -3322,6 +3362,8 @@ class TestCapabilityClassifier:
         )
         assert response.model == "capable-model"
         assert response.routing_decision["cause"] == "capability_classifier_fallback"
+        assert response.routing_decision["classifier_failure_reason"] == "timeout"
+        assert response.routing_decision["classifier_error_type"] == "TimeoutError"
         assert "classifier_p_solve" not in response.routing_decision
         assert "classifier_threshold" not in response.routing_decision
 
@@ -4195,6 +4237,9 @@ class TestLLMClassifier:
         assert first.cause == "heuristic_scorer"
         assert second.cause == "heuristic_scorer"
         assert "classifier-circuit-open" in second.signals
+        assert first.classifier_failure_reason == "timeout"
+        assert second.classifier_failure_reason == "circuit_open"
+        assert second.classifier_error_type is None
         mock_router_instance.acompletion.assert_awaited_once()
 
     def test_classifier_circuit_allows_one_probe_and_closes_on_success(self):
@@ -8158,18 +8203,24 @@ class TestClassifierPlugin:
         router = _plugin_router(mock_router_instance, _FixedTierClassifier(None))
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "declined"
+        assert outcome.classifier_error_type is None
 
     @pytest.mark.asyncio
     async def test_plugin_error_falls_back_to_heuristic(self, mock_router_instance):
         router = _plugin_router(mock_router_instance, _RaisingClassifier())
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "classifier_error"
+        assert outcome.classifier_error_type == "RuntimeError"
 
     @pytest.mark.asyncio
     async def test_plugin_timeout_falls_back_to_heuristic(self, mock_router_instance):
         router = _plugin_router(mock_router_instance, _SlowClassifier(), classifier_plugin_timeout_ms=20)
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "timeout"
+        assert outcome.classifier_error_type == "TimeoutError"
 
     @pytest.mark.asyncio
     async def test_plugin_non_string_verdict_falls_back_to_heuristic(self, mock_router_instance):
@@ -8177,12 +8228,16 @@ class TestClassifierPlugin:
         router = _plugin_router(mock_router_instance, _FixedTierClassifier(42))
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "invalid_response"
+        assert outcome.classifier_error_type is None
 
     @pytest.mark.asyncio
     async def test_plugin_unknown_tier_falls_back_to_heuristic(self, mock_router_instance):
         router = _plugin_router(mock_router_instance, _FixedTierClassifier("galactic"))
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "invalid_response"
+        assert outcome.classifier_error_type is None
 
     @pytest.mark.asyncio
     async def test_plugin_tier_without_pool_falls_back(self, mock_router_instance):
@@ -8198,6 +8253,8 @@ class TestClassifierPlugin:
         )
         outcome = await router.aclassify("what is 2+2?")
         assert outcome.cause == "heuristic_scorer"
+        assert outcome.classifier_failure_reason == "invalid_response"
+        assert outcome.classifier_error_type is None
 
     @pytest.mark.asyncio
     async def test_plugin_failure_with_default_model_fallback(self, mock_router_instance):
@@ -9533,6 +9590,101 @@ class TestRedactedLoggingDropsPromptText:
         assert decision["cause"] == "literal_keyword_match"
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "overrides, expected_cause",
+    [
+        ({"classifier_type": "llm"}, "heuristic_scorer"),
+        ({"classifier_type": "llm", "classifier_fallback": "default_model"}, "default_model_fallback"),
+        (
+            {
+                "classifier_type": "capability",
+                "capability_classifier_config": {
+                    "efficient_tier": "SIMPLE", "capable_tier": "REASONING",
+                    "base_threshold": 0.5, "threshold_step": 0.1,
+                },
+            },
+            "capability_classifier_fallback",
+        ),
+        (
+            {
+                "classifier_type": "llm_v2",
+                "llm_v2_config": {
+                    "efficient_profile": "Routine tasks", "capable_profile": "Complex tasks",
+                    "harness": "Read files and run checks", "max_quality_gap": 0.05,
+                },
+            },
+            "llm_v2_fallback",
+        ),
+    ],
+)
+async def test_classifier_failure_survives_real_router_decision_and_redaction(
+    overrides: Mapping[str, object], expected_cause: str, monkeypatch: pytest.MonkeyPatch
+) -> None:
+    monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+    router: Final = Router(model_list=[
+        {
+            "model_name": "auto",
+            "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {
+                    "tiers": {"SIMPLE": "cheap", "REASONING": "capable"},
+                    "default_model": "cheap",
+                    "classifier_llm_config": {"model": "classifier", "timeout_ms": 2000},
+                    **overrides,
+                },
+            },
+        },
+        {
+            "model_name": "classifier",
+            "litellm_params": {
+                "model": "openai/test-classifier", "api_key": "test-key",
+                "api_base": "https://classifier.test/v1",
+            },
+        },
+        {"model_name": "cheap", "litellm_params": {"model": "openai/test-cheap", "api_key": "test-key"}},
+        {"model_name": "capable", "litellm_params": {"model": "openai/test-capable", "api_key": "test-key"}},
+    ])
+    with respx.mock:
+        upstream: Final = respx.post("https://classifier.test/v1/chat/completions").respond(
+            200,
+            json={
+                "id": "classifier-result", "object": "chat.completion", "model": "test-classifier",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": "PRIVATE_INVALID_REPLY"},
+                             "finish_reason": "stop"}],
+            },
+        )
+        metadata: Final[dict[str, object]] = {"headers": {"x-litellm-enable-message-redaction": True}}
+        request: Final[dict[str, object]] = {"metadata": metadata, "turn_off_message_logging": True}
+        response: Final = await router.async_pre_routing_hook(
+            model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}]
+        )
+        assert upstream.called
+        assert response is not None and response.routing_decision is not None
+        decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python(metadata["routing_decision"])
+        assert decision["cause"] == expected_cause
+        assert decision["classifier_failure_reason"] == "invalid_response"
+        assert decision["classifier_error_type"] == "ValidationError"
+        assert "PRIVATE_INVALID_REPLY" not in json.dumps(decision)
+        assert "signals" not in decision
+        if overrides["classifier_type"] == "llm":
+            upstream.respond(200, json={
+                "id": "classifier-recovered", "object": "chat.completion", "model": "test-classifier",
+                "choices": [{"index": 0, "message": {"role": "assistant", "content": '{"tier":"SIMPLE"}'},
+                             "finish_reason": "stop"}],
+            })
+            recovered: Final = await router.async_pre_routing_hook(
+                model="auto", request_kwargs=request, messages=[{"role": "user", "content": "hi"}]
+            )
+            assert recovered is not None and recovered.routing_decision is not None
+            assert recovered.routing_decision["cause"] == "llm_classifier"
+            recovered_decision: Final = TypeAdapter(StandardLoggingRoutingDecision).validate_python(
+                metadata["routing_decision"]
+            )
+            assert "classifier_failure_reason" not in recovered_decision
+            assert "classifier_error_type" not in recovered_decision
+
+
 def test_every_routing_decision_field_is_classified():
     """Redaction is derived from a declaration, not a list at the call site, so every
     field has to be classified as quoting the prompt or aggregating it. A field added
@@ -13998,6 +14150,131 @@ class TestHybrid:
         assert outcome.cause == "heuristic_scorer"
 
 
+@pytest.mark.parametrize("classifier_type", ("heuristic", "heuristic_v2", "llm"))
+def test_local_heuristic_is_rejected_outside_chaining(classifier_type: str) -> None:
+    with pytest.raises(ValidationError, match="local_heuristic requires"):
+        ComplexityRouterConfig.model_validate(
+            {
+                "classifier_type": classifier_type,
+                "local_heuristic": "heuristic_v2",
+                "classifier_llm_config": {"model": "judge"},
+            }
+        )
+
+
+@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid"))
+def test_v2_chain_rejects_v1_custom_dimensions(classifier_type: str) -> None:
+    with pytest.raises(ValidationError, match="custom_dimensions requires"):
+        ComplexityRouterConfig.model_validate(
+            {
+                "classifier_type": classifier_type,
+                "local_heuristic": "heuristic_v2",
+                "classifier_llm_config": {"model": "judge"},
+                "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
+                "hybrid_boundary_margin": 0.03 if classifier_type == "hybrid" else None,
+                "tiers": dict(HEURISTIC_FIRST_TIERS),
+                "custom_dimensions": [{"name": "custom", "weight": 0.5, "patterns": ["custom"]}],
+            }
+        )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "classifier_type, threshold, margin, expected_tier, judge_called",
+    [
+        ("heuristic_first", 0.1, 0.03, "SIMPLE", False),
+        ("heuristic_first", 0.8, 0.03, "REASONING", True),
+        ("hybrid", 0.8, 0.03, "COMPLEX", False),
+        ("hybrid", 0.89, 0.03, "REASONING", True),
+        ("hybrid", 0.22, 0.02, "REASONING", True),
+        ("hybrid", 21 / 102, 0.0, "REASONING", True),
+        ("hybrid", 1.0, 0.0, "REASONING", True),
+    ],
+)
+async def test_selected_v2_chain_routes_using_success_probabilities(
+    mock_router_instance: MagicMock,
+    classifier_type: str,
+    threshold: float,
+    margin: float,
+    expected_tier: str,
+    judge_called: bool,
+) -> None:
+    mock_router_instance.acompletion = AsyncMock(return_value=_llm_response('{"tier": "REASONING"}'))
+    router: Final = ComplexityRouter(
+        model_name="v2-chain",
+        litellm_router_instance=mock_router_instance,
+        complexity_router_config={
+            "classifier_type": classifier_type,
+            "local_heuristic": "heuristic_v2",
+            "heuristic_v2_artifact": _heuristic_v2_artifact(),
+            "heuristic_v2_success_threshold": threshold,
+            "classifier_llm_config": {"model": "judge"},
+            "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
+            "hybrid_boundary_margin": margin if classifier_type == "hybrid" else None,
+            "tiers": dict(HEURISTIC_FIRST_TIERS),
+        },
+    )
+    response: Final = await router.async_pre_routing_hook(
+        model="v2-chain",
+        request_kwargs={},
+        messages=[{"role": "user", "content": "write a python function to reverse a string"}],
+    )
+    assert response is not None and response.routing_decision is not None
+    assert response.routing_decision["tier"] == expected_tier
+    assert mock_router_instance.acompletion.await_count == int(judge_called)
+    assert response.routing_decision["cause"] == (
+        "llm_classifier" if judge_called else f"{classifier_type}_short_circuit"
+    )
+    assert response.routing_decision.get("heuristic_v2_forecast") == (
+        None
+        if judge_called
+        else {
+            "probabilities": dict(
+                zip((tier.value for tier in TIER_SEVERITY_ORDER), (11 / 102, 21 / 102, 91 / 102, 100 / 102))
+            ),
+            "threshold": threshold,
+            "predicted_tier": expected_tier,
+            "request_type": "code_generation",
+        }
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("classifier_type", ("heuristic_first", "hybrid"))
+@pytest.mark.parametrize("encrypted", (False, True))
+@pytest.mark.parametrize("fallback", ("heuristic", "default_model"))
+async def test_v2_chain_judge_failure_uses_selected_fallback(
+    classifier_type: str, encrypted: bool, fallback: str
+) -> None:
+    _, dependency = _native_classifier_router(failure=RuntimeError("judge failed"))
+    dependency.acompletion = AsyncMock(side_effect=RuntimeError("judge failed"))
+    router: Final = ComplexityRouter(
+        model_name="v2-chain",
+        litellm_router_instance=dependency,
+        complexity_router_config={
+            "classifier_type": classifier_type,
+            "local_heuristic": "heuristic_v2",
+            "heuristic_v2_artifact": _heuristic_v2_artifact(),
+            "classifier_llm_config": {"model": "classifier"},
+            "heuristic_first_max_tier": "SIMPLE" if classifier_type == "heuristic_first" else None,
+            "hybrid_boundary_margin": 1.0 if classifier_type == "hybrid" else None,
+            "classifier_fallback": fallback,
+            "default_model": "gpt-4o",
+            "tiers": dict(HEURISTIC_FIRST_TIERS),
+        },
+    )
+    outcome: Final = await router.aclassify(
+        "hello", request_kwargs={"input": [_encrypted_agent_task()]} if encrypted else None
+    )
+    assert outcome.cause == ("heuristic_v2" if fallback == "heuristic" else "default_model_fallback")
+    assert outcome.tier == (ComplexityTier.COMPLEX if fallback == "heuristic" else ComplexityTier.MEDIUM)
+    assert (outcome.heuristic_v2_forecast is not None) == (fallback == "heuristic")
+    assert outcome.classifier_failure_reason == "classifier_error"
+    assert outcome.classifier_error_type == "RuntimeError"
+    assert dependency.aresponses.await_count == int(encrypted)
+    assert dependency.acompletion.await_count == int(not encrypted)
+
+
 def _windowed_router(*deployments: tuple) -> Router:
     """Real Router; each deployment is (group, provider_model, declared window or None).
     None means no declared override on a model the cost map does not know: unresolvable."""
@@ -16407,17 +16684,31 @@ class TestClassifierVision:
     @pytest.mark.parametrize(
         "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
     )
+    @pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2"))
     async def test_local_scorer_cannot_short_circuit_a_turn_it_cannot_see(
-        self, mock_router_instance, classifier_type, extra, short_circuit_cause
-    ):
+        self,
+        mock_router_instance: MagicMock,
+        classifier_type: str,
+        extra: dict[str, str | float],
+        short_circuit_cause: str,
+        local_heuristic: str,
+    ) -> None:
         """The scorer reads text alone, so its confidence is not a verdict on an image turn.
 
         Both arms are tuned so the scorer WOULD short-circuit on this exact text, which is what
         makes the image the only variable; a margin loose enough to leave the score undecided
         would pass whether or not the guard exists.
         """
-        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
-        response = await router.async_pre_routing_hook(
+        router: Final = self._router(
+            mock_router_instance,
+            vision={"enabled": True},
+            classifier_type=classifier_type,
+            local_heuristic=local_heuristic,
+            heuristic_v2_artifact=_heuristic_v2_artifact(),
+            heuristic_v2_success_threshold=0.0,
+            **extra,
+        )
+        response: Final = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=self._turn({"type": "text", "text": "what is this"}, IMG_PART)
         )
         assert response.routing_decision["cause"] == "llm_classifier"
@@ -16426,12 +16717,26 @@ class TestClassifierVision:
     @pytest.mark.parametrize(
         "classifier_type, extra, short_circuit_cause", SHORT_CIRCUIT_ARMS, ids=["heuristic_first", "hybrid"]
     )
+    @pytest.mark.parametrize("local_heuristic", ("heuristic", "heuristic_v2"))
     async def test_local_scorer_still_short_circuits_without_images(
-        self, mock_router_instance, classifier_type, extra, short_circuit_cause
-    ):
+        self,
+        mock_router_instance: MagicMock,
+        classifier_type: str,
+        extra: dict[str, str | float],
+        short_circuit_cause: str,
+        local_heuristic: str,
+    ) -> None:
         """The negative class: same router, same text, no image, and the scorer still decides."""
-        router = self._router(mock_router_instance, vision={"enabled": True}, classifier_type=classifier_type, **extra)
-        response = await router.async_pre_routing_hook(
+        router: Final = self._router(
+            mock_router_instance,
+            vision={"enabled": True},
+            classifier_type=classifier_type,
+            local_heuristic=local_heuristic,
+            heuristic_v2_artifact=_heuristic_v2_artifact(),
+            heuristic_v2_success_threshold=0.0,
+            **extra,
+        )
+        response: Final = await router.async_pre_routing_hook(
             model="m", request_kwargs={}, messages=[{"role": "user", "content": "what is this"}]
         )
         assert response.routing_decision["cause"] == short_circuit_cause

tests/unit/router_utils/test_auto_router_model_naming.py

@@ -9,10 +9,10 @@ from litellm.router_utils.auto_router_model_naming import (
     GATED_AUTO_ROUTER_CAPABILITIES,
     capability_limit_violation,
     carries_complexity_router_settings,
-    claimed_capability,
+    claimed_capabilities,
     classify_strategy_router_model,
     count_capability_routers,
-    gated_capability_of,
+    gated_capabilities_of,
     strategy_router_dependencies,
     validate_complexity_router_config_placement,
     validate_complexity_router_config_write,
@@ -74,11 +74,11 @@ def test_open_source_classifier_enumerates_its_accounting_model(
 def test_only_non_default_open_source_instructions_claim_the_shared_customization_slot(
     instructions: str | None, classifier_type: str, config_key: str
 ) -> None:
-    capability: Final = claimed_capability(
+    capabilities: Final = claimed_capabilities(
         {"classifier_type": classifier_type, config_key: {"instructions": instructions}}
     )
-    assert (capability.key if capability else None) == (
-        "tier_or_classifier_prompt" if instructions == "Route conservatively" else None
+    assert tuple(capability.key for capability in capabilities) == (
+        ("tier_or_classifier_prompt",) if instructions == "Route conservatively" else ()
     )
 
 
@@ -273,8 +273,8 @@ def test_fuse_write_accepts_presets_and_custom_text_with_the_same_entitlement()
     )
     assert validate_complexity_router_config_write(presets) is None
     assert validate_complexity_router_config_write(custom) is None
-    assert claimed_capability(presets) is claimed_capability(custom)
-    assert claimed_capability(presets) is not None
+    assert claimed_capabilities(presets) == claimed_capabilities(custom)
+    assert claimed_capabilities(presets)
 
 
 @pytest.mark.parametrize("field", ("efficient_profile", "capable_profile", "harness"))
@@ -579,8 +579,8 @@ def test_custom_classifier_prompt_capability(config: Mapping[str, object], expec
     A shipped rubric preset stays free, and the heuristic scorers never read system_prompt, so a
     value sitting on one is inert and claims nothing (heuristic_v2 still claims its own capability).
     """
-    claimed = claimed_capability(config)
-    assert (None if claimed is None else claimed.key) == expected_key
+    claimed: Final = claimed_capabilities(config)
+    assert tuple(capability.key for capability in claimed) == (() if expected_key is None else (expected_key,))
 
 
 @pytest.mark.parametrize(
@@ -640,13 +640,13 @@ def test_is_complexity_router_model(model: str | None, expected: bool) -> None:
         ({}, None),
     ],
 )
-def test_gated_capability_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None:
+def test_gated_capabilities_of(litellm_params: Mapping[str, object], expected_key: str | None) -> None:
     """Only a complexity router claiming a licensed capability counts toward that capability's limit.
 
     Renaming the built-in tiers through tier_labels is not a custom tier set, so it stays ungated.
     """
-    capability = gated_capability_of(litellm_params)
-    assert (None if capability is None else capability.key) == expected_key
+    capabilities: Final = gated_capabilities_of(litellm_params)
+    assert tuple(capability.key for capability in capabilities) == (() if expected_key is None else (expected_key,))
 
 
 @pytest.mark.parametrize("capability", GATED_AUTO_ROUTER_CAPABILITIES, ids=lambda c: c.key)
@@ -729,11 +729,29 @@ def test_every_gated_capability_has_a_distinct_predicate_and_sql_spelling() -> N
         },
     ],
 )
-def test_capabilities_are_mutually_exclusive_on_one_config(config: Mapping[str, object]) -> None:
-    """No config claims two capabilities, which is what lets one lock and one count serve them all.
-
-    The config validator is what makes this true and is pinned separately in test_complexity_router:
-    tier_definitions rejects every heuristic classifier_type and rejects the classifier system_prompt,
-    and system_prompt only counts for the classifier types heuristic_v2 is not one of.
-    """
+def test_standalone_classifiers_claim_at_most_one_capability(config: Mapping[str, object]) -> None:
     assert sum(1 for capability in GATED_AUTO_ROUTER_CAPABILITIES if capability.uses(config)) <= 1
+
+
+@pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
+@pytest.mark.parametrize("local_heuristic", (None, "heuristic", "heuristic_v2"))
+@pytest.mark.parametrize("custom_prompt", (False, True))
+def test_chained_heuristic_and_judge_prompt_claim_each_active_capability(
+    classifier: str, local_heuristic: str | None, custom_prompt: bool
+) -> None:
+    config: Final = {
+        "classifier_type": classifier,
+        **({"local_heuristic": local_heuristic} if local_heuristic is not None else {}),
+        **({"classification_prompt": "Grade by difficulty"} if custom_prompt else {}),
+    }
+    params: Final = {"model": "auto_router/complexity_router", "complexity_router_config": config}
+    expected: Final = (
+        *(("heuristic_v2",) if local_heuristic == "heuristic_v2" else ()),
+        *(("tier_or_classifier_prompt",) if custom_prompt else ()),
+    )
+    assert tuple(capability.key for capability in claimed_capabilities(config)) == expected
+    assert tuple(capability.key for capability in gated_capabilities_of(params)) == expected
+    for capability in GATED_AUTO_ROUTER_CAPABILITIES:
+        assert count_capability_routers(({"litellm_params": params},), capability=capability) == (
+            1 if capability.key in expected else 0
+        )

tests/unit/router_utils/test_auto_router_tuning_baseline.py

@@ -183,6 +183,28 @@ class TestRouterIdentity:
 
 
 class TestHeuristicV1Scope:
+    @pytest.mark.parametrize("classifier", ("heuristic_first", "hybrid"))
+    def test_v2_chains_ignore_retained_v1_tuning_until_v1_is_selected(self, classifier: str) -> None:
+        config: Final = {
+            "classifier_type": classifier,
+            "classifier_llm_config": {"model": "judge"},
+            **(
+                {"heuristic_first_max_tier": "MEDIUM"}
+                if classifier == "heuristic_first" else {"hybrid_boundary_margin": 0.1}
+            ),
+            "tiers": _TIERS,
+            "code_keywords": ["internal-api"],
+        }
+        v1: Final = _router("chain", config)
+        v2: Final = _router("chain", {**config, "local_heuristic": "heuristic_v2"})
+        other: Final = _router("other", {"code_keywords": ["internal-api"]})
+        baselines: Final = snapshot_tuning_baselines((_router("chain", {}),))
+        assert heuristic_v1_router_fingerprint(v2) is None
+        assert not snapshot_tuning_baselines((v2,))
+        assert not mutable_tuned_identities((v2,), baselines)
+        assert tuning_quota_violation(candidate=v2, others=(other,), baselines=baselines, limit=1) is None
+        assert tuning_quota_violation(candidate=v1, others=(other,), baselines=baselines, limit=1) is not None
+
     @pytest.mark.parametrize(
         "config,in_scope",
         [

tests/unit/test_cost_calculator.py

@@ -13,6 +13,7 @@ from litellm.cost_calculator import (
     BaseTokenUsageProcessor,
     RealtimeAPITokenUsageProcessor,
     ResponsesWebSocketTokenUsageProcessor,
+    batch_cost_calculator,
     completion_cost,
     cost_per_token,
     handle_realtime_stream_cost_calculation,
@@ -27,6 +28,7 @@ from litellm.types.utils import (
     CacheCreationTokenDetails,
     CallTypes,
     Choices,
+    CompletionTokensDetailsWrapper,
     EmbeddingResponse,
     ImageObject,
     ImageResponse,
@@ -3448,8 +3450,6 @@ def _batch_cache_usage() -> Usage:
 
 
 def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates():
-    from litellm.cost_calculator import batch_cost_calculator
-
     model_info: ModelInfo = {
         "input_cost_per_token_batches": 1e-7,
         "input_cost_per_audio_token_batches": 3.25e-6,
@@ -3478,8 +3478,6 @@ def test_batch_cost_calculator_prices_multimodal_tokens_at_modality_rates():
 
 
 def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities():
-    from litellm.cost_calculator import batch_cost_calculator
-
     model_info: ModelInfo = {"input_cost_per_token_batches": 1e-7}
     usage = Usage(
         prompt_tokens=100,
@@ -3498,6 +3496,87 @@ def test_batch_cost_calculator_falls_back_to_text_batch_rate_for_modalities():
     assert prompt_cost == pytest.approx(100 * 1e-7)
 
 
+@pytest.mark.parametrize(
+    ("image_batch_rate", "image_tokens", "expected_completion_cost"),
+    (
+        (5e-5, 80, 0.00408),
+        (None, 80, 0.00328),
+        (5e-5, 140, 0.006),
+    ),
+)
+def test_batch_cost_calculator_prices_image_completion_tokens_at_image_batch_rate(
+    image_batch_rate: float | None, image_tokens: int, expected_completion_cost: float
+) -> None:
+    model_info: Final[ModelInfo] = (
+        {
+            "output_cost_per_token_batches": 2e-6,
+            "output_cost_per_image_token": 8e-5,
+            "output_cost_per_image_token_batches": image_batch_rate,
+        }
+        if image_batch_rate is not None
+        else {
+            "output_cost_per_token_batches": 2e-6,
+            "output_cost_per_image_token": 8e-5,
+        }
+    )
+    usage: Final = Usage(
+        prompt_tokens=0,
+        completion_tokens=120,
+        total_tokens=120,
+        completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=image_tokens),
+    )
+
+    costs: Final = batch_cost_calculator(
+        usage=usage,
+        model="gemini-nano-banana-2.1",
+        custom_llm_provider="vertex_ai",
+        model_info=model_info,
+    )
+
+    assert costs[1] == pytest.approx(expected_completion_cost)
+
+
+def test_batch_cost_calculator_merges_image_only_deployment_rate_with_global_pricing(
+    _local_model_cost_map: None,
+) -> None:
+    model: Final = "gemini/gemini-3-pro-image"
+    global_model_info: Final = litellm.get_model_info(model=model, custom_llm_provider="gemini")
+    input_batch_rate: Final = global_model_info["input_cost_per_token_batches"]
+    output_batch_rate: Final = global_model_info["output_cost_per_token_batches"]
+    global_image_batch_rate: Final = global_model_info["output_cost_per_image_token_batches"]
+    assert input_batch_rate is not None
+    assert output_batch_rate is not None
+    assert global_image_batch_rate is not None
+
+    deployment_image_batch_rate: Final = 1e-6
+    assert deployment_image_batch_rate != global_image_batch_rate
+
+    text_tokens: Final = 300
+    image_tokens: Final = 200
+    usage: Final = Usage(
+        prompt_tokens=1_000,
+        completion_tokens=text_tokens + image_tokens,
+        total_tokens=1_500,
+        completion_tokens_details=CompletionTokensDetailsWrapper(image_tokens=image_tokens),
+    )
+    model_info: Final = ModelInfo(output_cost_per_image_token_batches=deployment_image_batch_rate)
+
+    prompt_cost, completion_cost = batch_cost_calculator(
+        usage=usage,
+        model=model,
+        custom_llm_provider="gemini",
+        model_info=model_info,
+    )
+
+    assert prompt_cost > 0
+    assert completion_cost > 0
+    assert prompt_cost == pytest.approx(usage.prompt_tokens * input_batch_rate)
+    expected_completion_cost: Final = text_tokens * output_batch_rate + image_tokens * deployment_image_batch_rate
+    global_rate_completion_cost: Final = text_tokens * output_batch_rate + image_tokens * global_image_batch_rate
+    assert completion_cost == pytest.approx(expected_completion_cost)
+    assert completion_cost != pytest.approx(global_rate_completion_cost)
+
+
 def test_batch_cost_calculator_prices_cache_creation_tokens_at_cache_write_rate():
     """
     LIT-4008 regression: anthropic batch usage is dominated by cache tokens.

tests/unit/test_router/test_router.py

@@ -12,7 +12,7 @@ import warnings
 from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable, Mapping
 from datetime import datetime, timedelta, timezone
 from types import SimpleNamespace
-from typing import Final, Literal
+from typing import TYPE_CHECKING, Final, Literal
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import httpx
@@ -31,6 +31,7 @@ from litellm.exceptions import GuardrailRaisedException, MidStreamFallbackError,
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
 from litellm.integrations.custom_guardrail import CustomGuardrail
 from litellm.integrations.custom_logger import CustomLogger
+from litellm.integrations.otel.plumbing.context import set_request_root_span
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.llms.anthropic.pass_through.messages.agentic_streaming_iterator import (
     SERVER_FULFILLED_TOOL_LEAK_ERROR_SSE_BYTES,
@@ -68,8 +69,14 @@ from litellm.types.router import (
     ModelInfo,
     PreRoutingHookResponse,
     RetryPolicy,
+    RoutingContext,
 )
 
+if TYPE_CHECKING:
+    from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+
+    from litellm.integrations.otel.logger import OpenTelemetryV2
+
 
 def test_update_kwargs_does_not_mutate_defaults_and_merges_metadata():
     # initialize a real Router (env‑vars can be empty)
@@ -13335,15 +13342,22 @@ class TestTeamPublicNameReachesPreRoutingStrategies:
 
     @pytest.mark.asyncio
     async def test_team_public_name_shadows_a_global_model_for_that_team_only(self):
-        router = self._router(
-            {self.INTERNAL_NAME: self._RewriteStrategy()},
-            extra_deployments=({"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},),
+        router: Final = litellm.Router(
+            model_list=[
+                self._team_marker(self.INTERNAL_NAME),
+                {"model_name": "gemini-flash", "litellm_params": {"model": "gemini/gemini-3.6-flash"}},
+                {"model_name": self.PUBLIC_NAME, "litellm_params": {"model": "openai/gpt-4o"}},
+            ],
         )
 
         async def routed(request_kwargs: dict) -> str | None:
             response = await router.async_pre_routing_hook(
                 model=self.PUBLIC_NAME, request_kwargs=request_kwargs, messages=self._messages()
             )
+            if response is not None:
+                assert response.routing_decision is not None
+                assert response.routing_decision["router_model_name"] == self.INTERNAL_NAME
+                assert response.routing_decision["router_config_id"] == router.model_list[0]["model_info"]["id"]
             return response.model if response else None
 
         async def selected(request_kwargs: dict) -> str:
@@ -20389,3 +20403,270 @@ async def test_non_chat_surfaces_mark_their_deployment_pick(monkeypatch: pytest.
     router.completion(model="gpt-4o", messages=[{"role": "user", "content": "hi"}])
 
     assert events == [_pick("embed", "initial", 1), _pick("gpt-4o", "initial", 1)]
+
+
+class TestAutoRouterTraceProvenance:
+    @pytest.fixture
+    def _tracing(self, monkeypatch: pytest.MonkeyPatch) -> "tuple[OpenTelemetryV2, InMemorySpanExporter]":
+        from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
+        from litellm.integrations.otel import OpenTelemetryV2Config
+        from litellm.integrations.otel.logger import OpenTelemetryV2
+        from litellm.integrations.otel.plumbing import providers
+        from litellm.proxy import proxy_server
+
+        config: Final = OpenTelemetryV2Config(exporter="in_memory")
+        exporter: Final = InMemorySpanExporter()
+        logger: Final = OpenTelemetryV2(config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter))
+        monkeypatch.setattr(proxy_server, "open_telemetry_logger", logger)
+        return logger, exporter
+
+    @staticmethod
+    def _marker(tag: str, target: str = "answer") -> dict[str, object]:
+        return {
+            "model_name": "traced-router",
+            "model_info": {"id": f"definition-{tag}", "updated_at": "2026-10-01T12:00:00+00:00"},
+            "litellm_params": {
+                "model": "auto_router/complexity_router", "tags": [tag],
+                "complexity_router_config": {
+                    "tiers": dict.fromkeys(("SIMPLE", "MEDIUM", "COMPLEX", "REASONING"), target),
+                },
+            },
+        }
+
+    def test_definition_fingerprint_is_canonical_and_tracks_config_not_credentials(self):
+        original: Final = Deployment.model_validate(self._marker("blue"))
+        reordered: Final = Deployment.model_validate({
+            **self._marker("blue"),
+            "litellm_params": {
+                "api_key": "credential-only-change", "tags": ["blue"], "model": "auto_router/complexity_router",
+                "complexity_router_config": {"tiers": dict.fromkeys(("REASONING", "COMPLEX", "MEDIUM", "SIMPLE"), "answer")},
+            },
+        })
+        fingerprint: Final = Router._routing_definition_fingerprint(original)
+        assert fingerprint is not None
+        assert fingerprint == Router._routing_definition_fingerprint(reordered)
+        assert fingerprint != Router._routing_definition_fingerprint(Deployment.model_validate(self._marker("blue", "other")))
+
+    @pytest.mark.asyncio
+    async def test_direct_classifier_and_plugin_prelude_never_inherit_parent_decisions(
+        self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch
+    ) -> None:
+        logger, exporter = _tracing
+        completed: Final = asyncio.Event()
+
+        class ObservedClassifier:
+            prelude: Mapping[str, object] = {}
+
+            async def run(self, context: RoutingContext) -> RoutingContext:
+                self.prelude = dict(context.metadata)
+                return context
+
+            async def classify(self, context: RoutingContext) -> str:
+                await litellm.acompletion(
+                    model="openai/gpt-5.6-luna", api_key="test", mock_response="SIMPLE",
+                    messages=[{"role": "user", "content": "choose a tier"}],
+                    metadata={**context.metadata, "internal_call_origin": "autorouter_classifier"},
+                )
+                return "SIMPLE"
+
+        class CallbackComplete(CustomLogger):
+            async def async_log_success_event(
+                self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime
+            ) -> None:
+                completed.set()
+
+        monkeypatch.setattr(litellm, "_async_success_callback", [logger, CallbackComplete()])
+        classifier: Final = ObservedClassifier()
+        router: Final = Router(model_list=[
+            {**self._marker("blue"), "litellm_params": {
+                "model": "auto_router/complexity_router",
+                "complexity_router_config": {
+                    "classifier_type": "custom", "classifier_plugin": classifier,
+                    "tiers": {"SIMPLE": "answer"}, "session_affinity": False, "deployment_affinity": False,
+                },
+            }},
+            {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
+        ], plugins=[classifier])
+        stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
+        request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale, "user_api_key_team_id": "team"}}
+        with logger.tracer.start_as_current_span("request") as root:
+            set_request_root_span(root)
+            await router.async_get_available_deployment(
+                model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]
+            )
+            await completed.wait()
+        classifier_span: Final = next(
+            span for span in exporter.get_finished_spans()
+            if (span.attributes or {}).get("litellm.request.purpose") == "autorouter_classifier"
+        )
+        assert not any(key.startswith("litellm.routing.") for key in classifier_span.attributes)
+        assert "routing_decision" not in classifier.prelude
+        assert "_autorouter_baseline_route" not in classifier.prelude
+        assert classifier.prelude["user_api_key_team_id"] == "team"
+        assert "routing_decision" not in request["metadata"]
+        assert "_autorouter_baseline_route" not in request["metadata"]
+        assert request["litellm_metadata"]["routing_decision"]["cause"] == "classifier_plugin"
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("build_fails", [False, True])
+    async def test_semantic_internal_failure_keeps_phase_identity_without_parent_retry_attribution(
+        self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", monkeypatch: pytest.MonkeyPatch,
+        respx_mock: respx.MockRouter, build_fails: bool,
+    ) -> None:
+        logger, exporter = _tracing
+        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+        query_statuses: Final = iter((503, 200))
+
+        def embedding_response(request: httpx.Request) -> httpx.Response:
+            status: Final = 400 if build_fails else (
+                next(query_statuses) if json.loads(request.content)["input"] == ["query-retry"] else 200
+            )
+            return httpx.Response(status, headers={"x-should-retry": "false"}, json={
+                "object": "list", "model": "text-embedding-3-small",
+                "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
+                "usage": {"prompt_tokens": 1, "total_tokens": 1},
+            })
+
+        upstream: Final = respx_mock.post("https://embedding.example/v1/embeddings").mock(side_effect=embedding_response)
+        router: Final = Router(model_list=[
+            {"model_name": "semantic", "model_info": {"id": "semantic-definition"}, "litellm_params": {
+                "model": "auto_router/semantic-router", "auto_router_default_model": "answer",
+                "auto_router_embedding_model": "embed",
+                "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
+            }},
+            *({"model_name": "embed", "model_info": {"id": f"embed-{index}"}, "litellm_params": {
+                "model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1",
+                "api_key": "test", "max_retries": 0,
+            }} for index in range(2)),
+            {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
+        ], num_retries=1, retry_after=0)
+        stale: Final = {"routing_decision": {"router_model_name": "previous"}, "_autorouter_baseline_route": "old"}
+        request: Final = {"metadata": {**stale}, "litellm_metadata": {**stale}}
+        with logger.tracer.start_as_current_span("request") as root:
+            set_request_root_span(root)
+            call: Final = router.async_get_available_deployment(
+                model="semantic", request_kwargs=request, messages=[{"role": "user", "content": "query-retry"}]
+            )
+            if build_fails:
+                with pytest.raises(ValueError, match="API call failed"):
+                    await call
+            else:
+                await call
+        spans: Final = {span.name: span for span in exporter.get_finished_spans()}
+        assert spans["route semantic"].attributes["litellm.routing.router_config_id"] == "semantic-definition"
+        assert not any(event.name == "litellm.routing.retry" for event in spans["request"].events)
+        if build_fails:
+            assert all(
+                "routing_decision" not in bucket and "_autorouter_baseline_route" not in bucket
+                for bucket in (request["metadata"], request["litellm_metadata"])
+            )
+        else:
+            assert sum(json.loads(call.request.content)["input"] == ["query-retry"] for call in upstream.calls) == 2
+            assert request["litellm_metadata"]["routing_decision"]["cause"] == "semantic_match"
+            assert sum(span.name == "route embed" for span in exporter.get_finished_spans()) == 2
+            assert any(event.name == "litellm.request.deployment_selected" for event in spans["request"].events)
+
+    @pytest.mark.asyncio
+    async def test_same_alias_and_tags_across_strategy_families_keep_selected_definition(
+        self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]"
+    ) -> None:
+        logger, exporter = _tracing
+        router: Final = Router(model_list=[
+            self._marker("blue"),
+            {
+                "model_name": "traced-router",
+                "model_info": {"id": "definition-semantic"},
+                "litellm_params": {
+                    "model": "auto_router/semantic-router",
+                    "tags": ["blue"],
+                    "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
+                    "auto_router_default_model": "answer",
+                    "auto_router_embedding_model": "embed",
+                },
+            },
+        ])
+        with logger.start_phase_span("route traced-router"):
+            response: Final = await router.async_pre_routing_hook(
+                model="traced-router", request_kwargs={"metadata": {"tags": ["blue"]}}
+            )
+        assert response is None
+        span: Final = next(span for span in exporter.get_finished_spans() if span.name == "route traced-router")
+        assert span.attributes is not None
+        assert span.attributes["litellm.routing.router_config_id"] == router.model_list[1]["model_info"]["id"]
+        assert span.attributes["litellm.routing.router_type"] == "semantic"
+
+    @pytest.mark.asyncio
+    async def test_semantic_team_alias_keeps_registered_identity_after_routing_and_retry(
+        self,
+        _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]",
+        monkeypatch: pytest.MonkeyPatch,
+        respx_mock: respx.MockRouter,
+    ) -> None:
+        logger, exporter = _tracing
+        monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
+        respx_mock.post("https://embedding.example/v1/embeddings").respond(200, json={
+            "object": "list",
+            "model": "text-embedding-3-small",
+            "data": [{"object": "embedding", "index": 0, "embedding": [1.0, 0.0]}],
+            "usage": {"prompt_tokens": 1, "total_tokens": 1},
+        })
+        router: Final = Router(model_list=[
+            {
+                "model_name": "internal-semantic-router",
+                "model_info": {"id": "semantic-definition", "team_id": "trace-team", "team_public_model_name": "public-router"},
+                "litellm_params": {
+                    "model": "auto_router/semantic-router",
+                    "auto_router_config": json.dumps({"routes": [{"name": "answer", "utterances": ["hello"]}]}),
+                    "auto_router_default_model": "answer",
+                    "auto_router_embedding_model": "embed",
+                },
+            },
+            {"model_name": "embed", "litellm_params": {"model": "openai/text-embedding-3-small", "api_base": "https://embedding.example/v1", "api_key": "test"}},
+            {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test"}},
+        ])
+        request: Final = {"metadata": {"user_api_key_team_id": "trace-team"}}
+        with logger.tracer.start_as_current_span("request"):
+            await router.async_get_available_deployment(
+                model="public-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}]
+            )
+            router.log_retry({**request, "model": "answer"}, ValueError("retry"))
+        decision: Final = request["metadata"]["routing_decision"]
+        assert decision["router_model_name"] == router.model_list[0]["model_name"]
+        assert decision["router_config_id"] == router.model_list[0]["model_info"]["id"]
+        assert decision["cause"] == "semantic_match"
+        spans: Final = {span.name: span for span in exporter.get_finished_spans()}
+        assert spans["route public-router"].attributes["litellm.routing.router_model_name"] == decision["router_model_name"]
+        assert spans["route public-router"].attributes["litellm.routing.router_config_id"] == decision["router_config_id"]
+        retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry")
+        assert retry.attributes["litellm.routing.router_model_name"] == decision["router_model_name"]
+
+    @pytest.mark.asyncio
+    @pytest.mark.parametrize("passthrough", [False, True])
+    async def test_attempt_spans_keep_selected_config_and_plain_fallback_clears_it(
+        self, _tracing: "tuple[OpenTelemetryV2, InMemorySpanExporter]", passthrough: bool
+    ) -> None:
+        logger, exporter = _tracing
+        router: Final = Router(model_list=[
+            self._marker("blue"), self._marker("green"),
+            {"model_name": "answer", "litellm_params": {"model": "openai/gpt-5.6-luna", "api_key": "test", "use_in_pass_through": True, "tags": ["blue", "green"]}},
+        ], enable_tag_filtering=True)
+        request: Final = {"metadata": {"tags": ["green"]}, "litellm_metadata": {"tags": ["green"]}}
+        pick: Final = router.async_get_available_deployment_for_pass_through if passthrough else router.async_get_available_deployment
+        with logger.tracer.start_as_current_span("request"):
+            await pick(model="traced-router", request_kwargs=request, messages=[{"role": "user", "content": "hello"}])
+            decision: Final = request["litellm_metadata"]["routing_decision"]
+            assert decision["router_config_id"] == "definition-green"
+            assert decision["router_config_updated_at"] == "2026-10-01T12:00:00+00:00"
+            assert decision["router_config_fingerprint"] == router.complexity_routers["traced-router"][1].definition_fingerprint
+            assert "routing_decision" not in request["metadata"]
+            router.log_retry({**request, "model": "answer"}, ValueError("private-error-text"))
+            await pick(model="answer", request_kwargs=request, messages=[{"role": "user", "content": "hello"}])
+        assert "routing_decision" not in request["litellm_metadata"]
+        spans: Final = {span.name: span for span in exporter.get_finished_spans()}
+        assert spans["route traced-router"].attributes["litellm.routing.router_config_id"] == "definition-green"
+        assert spans["route traced-router"].attributes["litellm.routing.routed_model"] == "answer"
+        assert not any(key.startswith("litellm.routing.") for key in spans["route answer"].attributes)
+        retry: Final = next(event for event in spans["request"].events if event.name == "litellm.routing.retry")
+        assert retry.attributes["error.type"] == "ValueError"
+        assert retry.attributes["litellm.retry.count"] == 1
+        assert "private-error-text" not in str(retry.attributes)

tests/unit/test_seed_tracing_fixtures.py

@@ -1,3 +1,4 @@
+import base64
 import hashlib
 import json
 import re
@@ -15,13 +16,13 @@ from pydantic import InstanceOf, TypeAdapter
 from litellm.rust_bridge.trace.generated.responses import TraceSQLResponse
 from litellm.rust_bridge.trace.storage import Tenant, span_rows
 from litellm.tracing.types import SpendLogRecord
-from tests._master_key import MASTER_KEY
 from scripts.seed_tracing_fixtures import (
     JSON,
     TRACE_FIXTURES,
     bulk_span_rows,
     fixture_capture,
     fixture_replays,
+    managed_response,
     postgres_row,
     rebase,
     rebase_spend,
@@ -33,6 +34,7 @@ from scripts.seed_tracing_fixtures import (
     spend_fixtures,
     timestamps,
 )
+from tests._master_key import MASTER_KEY
 
 CALL_KEYS: Final = TypeAdapter(tuple[str, ...])
 DATETIMES: Final = TypeAdapter(tuple[datetime, datetime])
@@ -77,9 +79,12 @@ def test_all_fixture_replays_are_recent_and_preserve_spans(path: Path) -> None:
             assert after_span_attributes["lens.original_trace_id"] == seed_id(
                 before_original_trace_id, replay.namespace, 32
             )
-            assert after["TraceId"] == hashlib.sha256(
-                f"litellm.claude.session.v1\0{seed_id(before_session, replay.namespace, 32)}".encode()
-            ).hexdigest()[:32]
+            assert (
+                after["TraceId"]
+                == hashlib.sha256(
+                    f"litellm.claude.session.v1\0{seed_id(before_session, replay.namespace, 32)}".encode()
+                ).hexdigest()[:32]
+            )
             assert after["TraceId"] != before["TraceId"]
         else:
             assert after["TraceId"] == seed_id(trace_id, replay.namespace, 32)
@@ -255,3 +260,27 @@ def test_seed_cli_rejects_invalid_http_timeouts(timeout: str) -> None:
     with pytest.raises(SystemExit) as error:
         seed_arguments(["--timeout-seconds", timeout])
     assert error.value.code == 2
+
+
+def test_replay_rebases_provider_request_evidence_with_the_spend_row() -> None:
+    spend: Final = {**dict(spend_fixtures())["deepagents_swarm"][0], "provider_request_id": "req_replay"}
+    pattern: Final = response_pattern((spend,))
+    replayed: Final = rebase_spend((spend,), 0, "another-run", pattern)[0]
+    export: Final = rebase({"request_id": "req_replay"}, 0, "another-run", pattern)
+
+    assert export == {"request_id": replayed["provider_request_id"]}
+    assert replayed["provider_request_id"] != spend["provider_request_id"]
+    assert replayed["provider_request_id"].startswith("req_")
+
+
+@pytest.mark.parametrize("upstream", ("msg_response", "req_response", "chatcmpl-response"))
+def test_replay_preserves_identity_between_managed_and_plain_response_ids(upstream: str) -> None:
+    payload: Final = f"model:example;response_id:{upstream}"
+    managed: Final = "resp_" + base64.b64encode(payload.encode()).decode()
+    spend: Final = {**dict(spend_fixtures())["deepagents_swarm"][0], "response_id": managed}
+    pattern: Final = response_pattern((spend,))
+    replayed: Final = rebase_spend((spend,), 0, "managed-replay", pattern)[0]
+    plain: Final = rebase(upstream, 0, "managed-replay", pattern)
+
+    assert plain != upstream
+    assert managed_response(replayed["response_id"]) == f"model:example;response_id:{plain}"

tests/unit/test_utils.py

@@ -665,6 +665,7 @@ def validate_model_cost_values(model_data, exceptions=None):
         "input_cost_per_audio_token",
         "output_cost_per_audio_token",
         "output_cost_per_image_token",
+        "output_cost_per_image_token_batches",
         "input_cost_per_video_token",
         "output_cost_per_video_token",
         "input_cost_per_audio_per_second",
@@ -903,6 +904,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                 "output_cost_per_image_2K": {"type": "number"},
                 "output_cost_per_image_4K": {"type": "number"},
                 "output_cost_per_image_token": {"type": "number"},
+                "output_cost_per_image_token_batches": {"type": "number"},
                 "output_cost_per_video_token": {"type": "number"},
                 "output_cost_per_pixel": {"type": "number"},
                 "output_cost_per_second": {"type": "number"},
@@ -6711,3 +6713,67 @@ def test_function_setup_never_logs_the_ocr_data_uri_payload() -> None:
 
     assert logged == [{"role": "user", "content": f"data:application/pdf;base64 ({len(payload)} chars)"}]
     assert payload not in str(logged)
+
+
+@pytest.mark.asyncio
+async def test_nested_wrapper_exits_schedule_one_async_success_log(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Chat over the Responses bridge exits two @client wrappers with one logging object. Issue
+    #44500: both exits enqueued a success handler, and on a large prompt the first one yielded to
+    the worker-thread base64 offload before it marked ``has_logged_async_success``, so the second
+    passed the check too and the request was logged and billed twice. The schedule step claims the
+    log for the object synchronously, so only the inner provider-shaped result is ever logged."""
+    from litellm.litellm_core_utils import litellm_logging
+    from litellm.litellm_core_utils.litellm_logging import Logging
+    from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
+    from litellm.utils import _dispatch_success_logging
+
+    class CountingLogger(CustomLogger):
+        def __init__(self) -> None:
+            super().__init__()
+            self.logged_results: list[object] = []
+
+        async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None:
+            if kwargs["litellm_call_id"] == "bridge-call-id":
+                self.logged_results.append(response_obj)
+
+    counting_logger: Final = CountingLogger()
+    monkeypatch.setattr(litellm, "_async_success_callback", [counting_logger])
+    real_truncate: Final = litellm_logging.truncate_base64_in_messages_async
+
+    async def yielding_truncate(messages):
+        await asyncio.sleep(0)
+        return await real_truncate(messages)
+
+    monkeypatch.setattr(litellm_logging, "truncate_base64_in_messages_async", yielding_truncate)
+
+    messages: Final = [{"role": "user", "content": "hello"}]
+    logging_obj: Final = Logging(
+        model="gpt-5.6-luna",
+        messages=messages,
+        stream=False,
+        call_type="acompletion",
+        start_time=datetime(2026, 1, 1, tzinfo=timezone.utc).timestamp(),
+        litellm_call_id="bridge-call-id",
+        function_id="bridge-fn-id",
+    )
+    logging_obj.update_environment_variables(
+        litellm_params={}, optional_params={}, model="gpt-5.6-luna", custom_llm_provider="openai", input=messages
+    )
+    inner_result: Final = ModelResponse(id="inner")
+    outer_result: Final = ModelResponse(id="outer")
+    now: Final = datetime(2026, 1, 1, tzinfo=timezone.utc)
+
+    for result in (inner_result, outer_result):
+        _dispatch_success_logging(
+            logging_obj=logging_obj,
+            result=result,
+            start_time=now,
+            end_time=now,
+            is_completion_with_fallbacks=False,
+            is_litellm_internal_call=False,
+        )
+    await asyncio.sleep(0)
+    await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=10)
+
+    assert len(counting_logger.logged_results) == 1, counting_logger.logged_results
+    assert counting_logger.logged_results[0] is inner_result

ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/projectAccess.ts

@@ -0,0 +1,46 @@
+import { useUISettings } from "@/app/(dashboard)/hooks/uiSettings/useUISettings";
+import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
+import { parseTeamAdminEditableFields } from "@/components/team/teamAdminEditAccess";
+import { all_admin_roles, internalUserRoles, isAdminRole, isProxyAdminRole } from "@/utils/roles";
+
+const TEAM_ADMIN_PROJECTS_PERMISSION = "projects";
+
+export const projectReaderRoles: readonly string[] = [...all_admin_roles, "Org Admin", ...internalUserRoles];
+
+export const canReadProjects = (userRole: string | null): boolean => projectReaderRoles.includes(userRole ?? "");
+
+export interface ProjectsPageViewer {
+  readonly userRole: string;
+  readonly isOrgAdmin: boolean;
+  readonly isTeamAdmin: boolean;
+}
+
+export const canViewProjectsPage = ({ userRole, isOrgAdmin, isTeamAdmin }: ProjectsPageViewer): boolean =>
+  canReadProjects(userRole) && (isAdminRole(userRole) || isOrgAdmin || isTeamAdmin);
+
+export interface ProjectManager {
+  readonly userRole: string;
+  readonly isViewOnly: boolean;
+  readonly isTeamAdmin: boolean;
+  readonly teamAdminEditableFields: readonly string[];
+}
+
+export const canManageProjects = ({
+  userRole,
+  isViewOnly,
+  isTeamAdmin,
+  teamAdminEditableFields,
+}: ProjectManager): boolean =>
+  !isViewOnly &&
+  (isProxyAdminRole(userRole) || (isTeamAdmin && teamAdminEditableFields.includes(TEAM_ADMIN_PROJECTS_PERMISSION)));
+
+export const useCanManageProjects = (isTeamAdmin: boolean): boolean => {
+  const { userRole, isViewOnly } = useAuthorized();
+  const { data: uiSettings } = useUISettings();
+  return canManageProjects({
+    userRole,
+    isViewOnly,
+    isTeamAdmin,
+    teamAdminEditableFields: parseTeamAdminEditableFields(uiSettings?.values),
+  });
+};

ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjectDetails.test.ts

@@ -103,8 +103,18 @@ describe("useProjectDetails", () => {
     expect(global.fetch).not.toHaveBeenCalled();
   });
 
-  it("should not fetch when userRole is not an admin role", () => {
-    mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole: "Internal User" });
+  it.each(["Internal User", "Org Admin"])("should fetch when userRole is %s", async (userRole) => {
+    mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole });
+    (global.fetch as any).mockResolvedValue({ ok: true, json: async () => mockProject });
+    const { result } = renderHook(() => useProjectDetails("proj-1"), {
+      wrapper: makeWrapper(queryClient),
+    });
+    await waitFor(() => expect(result.current.isSuccess).toBe(true));
+    expect(global.fetch).toHaveBeenCalled();
+  });
+
+  it("should not fetch when userRole cannot read projects", () => {
+    mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole: "regular_user" });
     const { result } = renderHook(() => useProjectDetails("proj-1"), {
       wrapper: makeWrapper(queryClient),
     });

ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjectDetails.ts

@@ -1,7 +1,7 @@
 import { useQuery, useQueryClient } from "@tanstack/react-query";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName, deriveErrorMessage, handleError } from "@/components/networking";
-import { all_admin_roles } from "@/utils/roles";
 import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
+import { canReadProjects } from "./projectAccess";
 import { ProjectResponse, projectKeys } from "./useProjects";
 
 // ── Fetch function ───────────────────────────────────────────────────────────
@@ -37,7 +37,7 @@ export const useProjectDetails = (projectId?: string) => {
   return useQuery<ProjectResponse>({
     queryKey: projectKeys.detail(projectId!),
     queryFn: async () => fetchProjectDetails(accessToken!, projectId!),
-    enabled: Boolean(accessToken && projectId) && all_admin_roles.includes(userRole || ""),
+    enabled: Boolean(accessToken && projectId) && canReadProjects(userRole),
 
     // Seed from the list cache when available
     initialData: () => {

ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.test.ts

@@ -115,8 +115,8 @@ describe("useProjects", () => {
     expect(global.fetch).not.toHaveBeenCalled();
   });
 
-  it("should fetch when userRole is an internal user role", async () => {
-    mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole: "Internal User" });
+  it.each(["Internal User", "Org Admin"])("should fetch when userRole is %s", async (userRole) => {
+    mockUseAuthorized.mockReturnValue({ accessToken: "test-token", userRole });
     (global.fetch as any).mockResolvedValue({ ok: true, json: async () => mockProjects });
     const { result } = renderHook(() => useProjects(), { wrapper: makeWrapper(queryClient) });
     await waitFor(() => expect(result.current.isSuccess).toBe(true));

ui/litellm-dashboard/src/app/(dashboard)/hooks/projects/useProjects.ts

@@ -2,7 +2,7 @@ import { useQuery } from "@tanstack/react-query";
 import { createQueryKeys } from "../common/queryKeysFactory";
 import { getProxyBaseUrl, getGlobalLitellmHeaderName, deriveErrorMessage, handleError } from "@/components/networking";
 import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized";
-import { all_admin_roles, internalUserRoles } from "@/utils/roles";
+import { canReadProjects } from "./projectAccess";
 
 // ── Types ────────────────────────────────────────────────────────────────────
 
@@ -42,8 +42,6 @@ export interface ProjectResponse {
 
 export const projectKeys = createQueryKeys("projects");
 
-const projectReaderRoles = [...all_admin_roles, ...internalUserRoles];
-
 // ── Fetch function ───────────────────────────────────────────────────────────
 
 const fetchProjects = async (accessToken: string): Promise<ProjectResponse[]> => {
@@ -76,6 +74,6 @@ export const useProjects = () => {
   return useQuery<ProjectResponse[]>({
     queryKey: projectKeys.list({}),
     queryFn: async () => fetchProjects(accessToken!),
-    enabled: Boolean(accessToken) && projectReaderRoles.includes(userRole!),
+    enabled: Boolean(accessToken) && canReadProjects(userRole),
   });
 };