Merge remote-tracking branch 'origin/main' into litellm_trace_canonical_content

# Conflicts:
#	litellm-rust/crates/traces-clickhouse/tests/reads.rs

litellm-rust/crates/callbacks-legacy-python/python_contract.json

@@ -88,7 +88,8 @@
     "call_type"
   ],
   "stream_opened": [
-    "logger"
+    "logger",
+    "head"
   ],
   "stream_success": [
     "logger",

litellm-rust/crates/callbacks-legacy-python/src/adapter.rs

@@ -536,7 +536,7 @@ impl LegacyLogging {
             head.bind(py).set_item("cache_key", key)?;
             head.bind(py).set_item("cache_hit", true)?;
         }
-        Streaming::Opened.call(py, (self.logger()?.object(py),))?;
+        Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?;
         self.stream = Some(DeliveredStream {
             chunks: PyList::empty(py).unbind(),
             first_chunk: None,
@@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response
     fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() {
         Python::initialize();
         Python::attach(|py| {
-            let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None");
+            let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}");
             let mut logging = LegacyLogging {
                 operation: crate::LoggingOperation::Messages,
                 ..logged(py, &locals, true)
             };
             logging
-                .on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind())
+                .on_stream_open(py, &local(&locals, "head").unbind())
                 .unwrap();
             logging
                 .on_stream_chunk(py, &local(&locals, "first").unbind())
@@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response
                 &locals,
                 c"
 assert logger.names() == ['stream_opened', 'stream_success'], logger.calls
+assert logger.calls[0][1] is head
 chunks = logger.calls[1][1]
 assert len(chunks) == 2
 assert chunks[0] is first

litellm-rust/crates/callbacks-legacy-python/src/test_support.rs

@@ -84,7 +84,7 @@ FAKES = {
         'success', response, call_type
     ),
     'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type),
-    'stream_opened': lambda logger: logger.record('stream_opened', None),
+    'stream_opened': lambda logger, head: logger.record('stream_opened', head),
     'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record(
         'stream_success', list(chunks)
     ),

litellm-rust/crates/traces-cache/src/cache.rs

@@ -298,6 +298,7 @@ mod tests {
     fn row(span_id: &str) -> TraceSpansRow {
         TraceSpansRow {
             trace_id: String::new(),
+            original_trace_id: String::new(),
             span_id: span_id.into(),
             parent_span_id: String::new(),
             name: "run".into(),

litellm-rust/crates/traces-cache/src/spend.rs

@@ -48,6 +48,7 @@ pub(super) async fn spend<S: TraceStore>(
     let params = SpendByResponseIdsParams {
         access: access.clone(),
         response_ids: lookup.response_ids,
+        provider_request_ids: lookup.provider_request_ids,
         request_ids: lookup.request_ids,
         trace_ids: lookup.trace_ids,
         start_ms: window.start,

litellm-rust/crates/traces-cache/tests/read.rs

@@ -270,6 +270,7 @@ fn access() -> ReadAccessParams {
 fn span(index: usize) -> TraceSpansRow {
     TraceSpansRow {
         trace_id: "trace".into(),
+        original_trace_id: String::new(),
         span_id: format!("span-{index}"),
         parent_span_id: if index == 0 {
             String::new()
@@ -533,6 +534,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow {
         litellm_call_id: String::new(),
         response_id: response_id.into(),
         upstream_response_id: String::new(),
+        provider_request_id: String::new(),
         trace_id: String::new(),
         span_id: String::new(),
         team_id: "team".into(),

litellm-rust/crates/traces-cache/tests/snapshots.rs

@@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120);
 fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow {
     TraceSpansRow {
         trace_id: String::new(),
+        original_trace_id: String::new(),
         span_id: span_id.into(),
         parent_span_id: parent.into(),
         name: name.into(),

litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql

@@ -0,0 +1,4 @@
+ALTER TABLE {database}.spend_logs
+    ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id,
+    ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id
+        TYPE bloom_filter(0.001) GRANULARITY 1

litellm-rust/crates/traces-clickhouse/query/spend_batch.sql

@@ -1,5 +1,5 @@
 SELECT * FROM (
-SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
+SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend,
        toUnixTimestamp64Milli(start_time) AS start_ms
 FROM (
     SELECT *,
@@ -15,7 +15,8 @@ FROM (
            OR ({user_id:String} != '' AND user = {user_id:String})
            OR has({team_ids:Array(String)}, team_id))
 )
-WHERE response_id IN {response_ids:Array(String)}
+WHERE provider_request_id IN {provider_request_ids:Array(String)}
+   OR response_id IN {response_ids:Array(String)}
    OR upstream_response_id IN {response_ids:Array(String)}
    OR litellm_call_id IN {request_ids:Array(String)}
    OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})

litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql

@@ -1,4 +1,4 @@
-SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend,
+SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend,
        toUnixTimestamp64Milli(start_time) AS start_ms
 FROM (
     SELECT *,
@@ -14,7 +14,8 @@ FROM (
            OR ({user_id:String} != '' AND user = {user_id:String})
            OR has({team_ids:Array(String)}, team_id))
 )
-WHERE response_id IN {response_ids:Array(String)}
+WHERE provider_request_id IN {provider_request_ids:Array(String)}
+   OR response_id IN {response_ids:Array(String)}
    OR upstream_response_id IN {response_ids:Array(String)}
    OR litellm_call_id IN {request_ids:Array(String)}
    OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)})

litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql

@@ -1,5 +1,5 @@
 SELECT * FROM (
-SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
        o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
        o.Framework AS framework, o.StatusCode AS status,
        substringUTF8(o.StatusMessage, 1, 128) AS status_message,

litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql

@@ -1,4 +1,4 @@
-SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
        o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
        o.Framework AS framework, o.StatusCode AS status,
        substringUTF8(o.StatusMessage, 1, 128) AS status_message,

litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql

@@ -1,5 +1,5 @@
 SELECT * FROM (
-SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
        o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
        o.Framework AS framework, o.StatusCode AS status,
        substringUTF8(o.StatusMessage, 1, 128) AS status_message,

litellm-rust/crates/traces-clickhouse/query/trace_spans.sql

@@ -1,4 +1,4 @@
-SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
+SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name,
        o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent,
        o.Framework AS framework, o.StatusCode AS status,
        substringUTF8(o.StatusMessage, 1, 128) AS status_message,

litellm-rust/crates/traces-clickhouse/src/query/named.rs

@@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams;
 struct TraceSpansRowEncoding {
     #[serde(default)]
     pub trace_id: String,
+    #[serde(default)]
+    pub original_trace_id: String,
     pub span_id: String,
     pub parent_span_id: String,
     pub name: String,
@@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding {
     #[serde(flatten)]
     pub access: contracts::ReadAccessParams,
     pub response_ids: Vec<String>,
+    pub provider_request_ids: Vec<String>,
     pub request_ids: Vec<String>,
     pub trace_ids: Vec<String>,
     #[serde(deserialize_with = "super::number::deserialize")]
@@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding {
     pub litellm_call_id: String,
     pub response_id: String,
     pub upstream_response_id: String,
+    #[serde(default)]
+    pub provider_request_id: String,
     pub trace_id: String,
     pub span_id: String,
     pub team_id: String,
@@ -359,7 +364,7 @@ mod tests {
             quoted,
         );
         round_trip::<TraceSpansRow>(
-            json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
+            json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
             quoted,
         );
         round_trip::<SpanDetailRow>(
@@ -371,7 +376,7 @@ mod tests {
             quoted,
         );
         round_trip::<SpendByResponseIdsRow>(
-            json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
+            json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
             quoted,
         );
     }
@@ -389,7 +394,7 @@ mod tests {
             quoted,
         );
         round_trip::<SpendByResponseIdsParams>(
-            json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
+            json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
             quoted,
         );
     }

litellm-rust/crates/traces-clickhouse/src/span_row.rs

@@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str {
         .flatten()
         .find_map(|key| match key {
             CallKey::ProviderResponse(id) => Some(id.as_str()),
-            CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None,
+            CallKey::LiteLlmRequest(_)
+            | CallKey::ProviderRequest(_)
+            | CallKey::Transport
+            | CallKey::GatewayAttempt => None,
         })
         .unwrap_or_default()
 }

litellm-rust/crates/traces-clickhouse/tests/migrations.rs

@@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins(
             Parameter::Strings(vec!["response-1".into()]),
         ),
         ("request_ids".into(), Parameter::Strings(Vec::new())),
+        (
+            "provider_request_ids".into(),
+            Parameter::Strings(Vec::new()),
+        ),
         ("trace_ids".into(), Parameter::Strings(Vec::new())),
         ("all_teams".into(), Parameter::Integer(0)),
         ("user_id".into(), Parameter::Text(String::new())),
@@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility(
                 "api_key_hash": legacy_key.unwrap_or_default(),
             }))?,
             response_ids: vec!["shared-response".into()],
+            provider_request_ids: Vec::new(),
             request_ids: Vec::new(),
             trace_ids: Vec::new(),
             start_ms: timestamp / 1_000_000 - 1,

litellm-rust/crates/traces-clickhouse/tests/queries/support.rs

@@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap<String, Value
                     .find_map(|key| match key {
                         litellm_traces::CallKey::LiteLlmRequest(id)
                         | litellm_traces::CallKey::ProviderResponse(id) => Some(id.as_str()),
-                        litellm_traces::CallKey::Transport
+                        litellm_traces::CallKey::ProviderRequest(_)
+                        | litellm_traces::CallKey::Transport
                         | litellm_traces::CallKey::GatewayAttempt => None,
                     })
                     .unwrap_or_default()

litellm-rust/crates/traces-clickhouse/tests/reads.rs

@@ -601,7 +601,7 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals(
 
 #[rstest]
 #[tokio::test]
-async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback(
+async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads(
     #[future(awt)] migrated_database: TestResult<SeededDatabase>,
 ) -> TestResult {
     let fixture = migrated_database?;
@@ -610,31 +610,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
     let start_ms = 1_790_000_000_000_i64;
     let cases = [
         (
-            "gateway",
-            "provider-request",
-            "gateway",
+            "same-key",
+            "provider_response:same-key",
             "team-a",
             "key-a",
             Some(0.25),
         ),
-        ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)),
-        ("conflict", "conflict", "different", "team-a", "key-a", None),
+        (
+            "other-key",
+            "provider_response:other-key",
+            "team-a",
+            "key-b",
+            None,
+        ),
         (
             "foreign-team",
-            "request",
-            "foreign-team",
+            "provider_response:foreign-team",
             "team-b",
             "key-a",
             None,
         ),
         (
-            "foreign-key",
-            "request",
-            "foreign-key",
+            "call-id-other-key",
+            "litellm_request:call-id-other-key",
             "team-a",
             "key-b",
             None,
         ),
+        (
+            "call-id-foreign-team",
+            "litellm_request:call-id-foreign-team",
+            "team-b",
+            "key-a",
+            None,
+        ),
+        ("transport-only", "transport:", "team-a", "key-a", None),
     ];
     insert_rows(
         client,
@@ -643,7 +653,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
         InsertTable::OtelTraces,
         cases
             .iter()
-            .map(|(id, _, _, _, _, _)| {
+            .map(|(id, key, _, _, _)| {
                 BTreeMap::from([
                     ("Timestamp".into(), json!(start_ms * 1_000_000)),
                     ("Duration".into(), json!(1_000_000)),
@@ -652,7 +662,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
                     ("ObservationType".into(), json!("llm")),
                     ("TeamId".into(), json!("team-a")),
                     ("ApiKeyHash".into(), json!("key-a")),
-                    ("CallKeys".into(), json!([format!("litellm_request:{id}")])),
+                    ("CallKeys".into(), json!([key])),
                     ("CallEvidence".into(), json!("complete")),
                 ])
             })
@@ -666,11 +676,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
         InsertTable::SpendLogs,
         cases
             .iter()
-            .map(|(_, request, call_id, team, key, _)| {
+            .map(|(id, _, team, key, _)| {
                 BTreeMap::from([
-                    ("request_id".into(), json!(request)),
-                    ("response_id".into(), json!("provider-response")),
-                    ("litellm_call_id".into(), json!(call_id)),
+                    ("request_id".into(), json!(format!("request-{id}"))),
+                    ("response_id".into(), json!(id)),
+                    ("litellm_call_id".into(), json!(id)),
                     ("team_id".into(), json!(team)),
                     ("api_key".into(), json!(key)),
                     ("start_time".into(), json!(start_ms)),
@@ -689,13 +699,13 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
     let access = ReadAccessParams {
         all_teams: false,
         user_id: String::new(),
-        team_ids: vec!["team-a".into()],
+        team_ids: vec!["team-a".into(), "team-b".into()],
     };
     let page = reader
         .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
         .await?;
     assert_eq!(page.data.len(), cases.len());
-    for (id, _, _, _, _, expected) in cases {
+    for (id, _, _, _, expected) in cases {
         let summary = page
             .data
             .iter()
@@ -707,10 +717,126 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback
             .ok_or("missing trace")?;
         assert_eq!(detail.summary.spend, expected, "{id}");
         assert_eq!(summary.spend, expected, "{id}");
+        assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}");
     }
     Ok(())
 }
 
+#[rstest]
+#[case::provider_request(true)]
+#[case::transport(false)]
+#[tokio::test]
+async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners(
+    #[future(awt)] migrated_database: TestResult<SeededDatabase>,
+    #[case] response_header: bool,
+    #[values(false, true)] grouped: bool,
+) -> TestResult {
+    let fixture = migrated_database?;
+    let client = &fixture.database.client;
+    let writer = Connection::writer(&fixture.database.url)?;
+    let trace_id = if grouped {
+        "grouped-trace"
+    } else {
+        "original-trace"
+    };
+    let start_ms = 1_790_000_000_000_i64;
+    let keys = if response_header {
+        vec!["provider_response:req_native"]
+    } else {
+        Vec::new()
+    };
+    insert_rows(
+        client,
+        &writer,
+        DATABASE,
+        InsertTable::OtelTraces,
+        vec![BTreeMap::from([
+            ("Timestamp".into(), json!(start_ms * 1_000_000)),
+            ("TraceId".into(), json!(trace_id)),
+            ("SpanId".into(), json!("native-call")),
+            ("SpanName".into(), json!("claude_code.llm_request")),
+            ("ObservationType".into(), json!("llm")),
+            ("Framework".into(), json!("claude-code")),
+            ("TeamId".into(), json!("team-a")),
+            ("ApiKeyHash".into(), json!("key-a")),
+            ("CallKeys".into(), json!(keys)),
+            (
+                "CallEvidence".into(),
+                json!(if response_header {
+                    "complete"
+                } else {
+                    "unknown"
+                }),
+            ),
+            (
+                "SpanAttributes".into(),
+                json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}),
+            ),
+        ])],
+    )
+    .await?;
+    insert_rows(
+        client,
+        &writer,
+        DATABASE,
+        InsertTable::SpendLogs,
+        ["key-a", "key-b"]
+            .into_iter()
+            .map(|key| {
+                BTreeMap::from([
+                    ("request_id".into(), json!(format!("log-{key}"))),
+                    ("response_id".into(), json!("msg_native")),
+                    ("provider_request_id".into(), json!("req_native")),
+                    (
+                        "trace_id".into(),
+                        json!(if response_header {
+                            ""
+                        } else {
+                            "original-trace"
+                        }),
+                    ),
+                    (
+                        "span_id".into(),
+                        json!(if response_header { "" } else { "native-call" }),
+                    ),
+                    ("team_id".into(), json!("team-a")),
+                    ("api_key".into(), json!(key)),
+                    ("start_time".into(), json!(start_ms)),
+                    ("end_time".into(), json!(start_ms + 1)),
+                    ("spend".into(), json!(0.25)),
+                ])
+            })
+            .collect(),
+    )
+    .await?;
+    let connection = fixture
+        .readers
+        .connection(client, &QueryScope::All, "fixture-secret")
+        .await?;
+    let (reader, store) = make_reader(client, connection);
+    let access = ReadAccessParams {
+        all_teams: true,
+        user_id: String::new(),
+        team_ids: Vec::new(),
+    };
+    let page = reader
+        .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50)
+        .await?;
+    assert_eq!(page.data.len(), 1);
+    let summary = &page.data[0];
+    let detail = reader
+        .get_trace(&store, &access, trace_id, &summary.trace_ref)
+        .await?
+        .ok_or("missing trace")?;
+    assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1));
+    assert_eq!(detail.summary.spend, summary.spend);
+    assert_eq!(
+        detail.spans[0].spend_log_request_id.as_deref(),
+        Some("log-key-a")
+    );
+    Ok(())
+}
+
 #[rstest]
 #[case::recorded(false)]
 #[case::conflicting(true)]

litellm-rust/crates/traces/src/lib.rs

@@ -44,5 +44,6 @@ pub use tenant::Tenant;
 pub use truncate::{truncate_messages, truncate_value};
 pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content};
 pub use view::{
-    AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary,
+    AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage,
+    TraceSummary,
 };

litellm-rust/crates/traces/src/normalize/format/claude_code.rs

@@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts};
 use crate::{
     Error,
     normalize::{
-        CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey,
+        CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence,
         ObservationType, RoleEvidence, SpanContext, attr, present, tokens,
     },
     otlp::DecodedEvent,
@@ -265,7 +265,7 @@ impl Format for ClaudeCode {
                     output: llm_output(attributes),
                     calls: present(attributes, &["gen_ai.response.id", "request_id"])
                         .map_or(CallEvidence::Unknown, |id| {
-                            CallEvidence::complete(CallKey::ProviderResponse(id))
+                            CallEvidence::complete(crate::normalize::claude_call_key(id))
                         }),
                     ..base
                 },

litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs

@@ -216,7 +216,16 @@ impl Instrumentation {
 
 fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts {
     let calls = [
-        present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse),
+        present(context.attributes, &["gen_ai.response.id"]).map(|id| {
+            if matches!(
+                context.scope,
+                crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE
+            ) {
+                crate::normalize::claude_call_key(id)
+            } else {
+                CallKey::ProviderResponse(id)
+            }
+        }),
         present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest),
     ]
     .into_iter()

litellm-rust/crates/traces/src/normalize/mod.rs

@@ -65,16 +65,26 @@ pub enum CallKey {
     LiteLlmRequest(String),
     /// The provider response id returned to the caller (`spend_logs.response_id`).
     ProviderResponse(String),
+    ProviderRequest(String),
     /// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id.
     Transport,
     GatewayAttempt,
 }
 
+pub(crate) fn claude_call_key(id: String) -> CallKey {
+    if id.starts_with("msg_") {
+        CallKey::ProviderResponse(id)
+    } else {
+        CallKey::ProviderRequest(id)
+    }
+}
+
 impl fmt::Display for CallKey {
     fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
         match self {
             Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"),
             Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"),
+            Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"),
             Self::Transport => formatter.write_str("transport:"),
             Self::GatewayAttempt => formatter.write_str("gateway_attempt:"),
         }
@@ -89,6 +99,9 @@ impl FromStr for CallKey {
             Some(("provider_response", id)) if !id.is_empty() => {
                 Ok(Self::ProviderResponse(id.to_owned()))
             }
+            Some(("provider_request", id)) if !id.is_empty() => {
+                Ok(Self::ProviderRequest(id.to_owned()))
+            }
             Some(("litellm_request", id)) if !id.is_empty() => {
                 Ok(Self::LiteLlmRequest(id.to_owned()))
             }
@@ -127,24 +140,45 @@ pub enum CallEvidence {
 
 impl CallEvidence {
     pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet<CallKey> {
-        if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() {
+        let native = Self::native_request(row);
+        let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() {
             BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())])
         } else {
             row.call_keys.iter().cloned().collect()
+        };
+        if !native {
+            return keys;
         }
+        if keys.is_empty() {
+            return BTreeSet::from([CallKey::Transport]);
+        }
+        keys.into_iter()
+            .map(|key| match key {
+                CallKey::ProviderResponse(id) => claude_call_key(id),
+                key => key,
+            })
+            .collect()
+    }
+
+    fn native_request(row: &crate::query::named::TraceSpansRow) -> bool {
+        matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk")
+            && row.name == "claude_code.llm_request"
     }
 
     pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self {
-        let kind = row
-            .call_evidence
-            .unwrap_or(if Self::row_keys(row).is_empty() {
+        let keys = Self::row_keys(row);
+        let kind = if Self::native_request(row) {
+            CallEvidenceKind::Complete
+        } else {
+            row.call_evidence.unwrap_or(if keys.is_empty() {
                 CallEvidenceKind::Unknown
             } else {
                 CallEvidenceKind::Complete
-            });
+            })
+        };
         match kind {
-            CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)),
-            CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)),
+            CallEvidenceKind::Complete => Self::Complete(keys),
+            CallEvidenceKind::Partial => Self::Partial(keys),
             CallEvidenceKind::Unknown => Self::Unknown,
         }
     }

litellm-rust/crates/traces/src/otlp/span.rs

@@ -193,9 +193,9 @@ fn decoded_span(
                 .into_iter()
                 .flatten()
                 .map(|key| match key {
-                    crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => {
-                        id.len() + size_of::<crate::CallKey>()
-                    }
+                    crate::CallKey::LiteLlmRequest(id)
+                    | crate::CallKey::ProviderResponse(id)
+                    | crate::CallKey::ProviderRequest(id) => id.len() + size_of::<crate::CallKey>(),
                     crate::CallKey::Transport | crate::CallKey::GatewayAttempt => {
                         size_of::<crate::CallKey>()
                     }

litellm-rust/crates/traces/src/query/named.rs

@@ -68,6 +68,8 @@ pub struct TraceSpansParams {
 pub struct TraceSpansRow {
     #[serde(default)]
     pub trace_id: String,
+    #[serde(default)]
+    pub original_trace_id: String,
     pub span_id: String,
     pub parent_span_id: String,
     pub name: String,
@@ -113,6 +115,16 @@ pub struct TraceSpansRow {
     pub user_id: String,
 }
 
+impl TraceSpansRow {
+    pub(crate) fn transport_trace_id(&self) -> &str {
+        if self.original_trace_id.is_empty() {
+            &self.trace_id
+        } else {
+            &self.original_trace_id
+        }
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct TracePageSpansParams {
     #[serde(flatten)]
@@ -172,6 +184,7 @@ pub struct SpendByResponseIdsParams {
     #[serde(flatten)]
     pub access: ReadAccessParams,
     pub response_ids: Vec<String>,
+    pub provider_request_ids: Vec<String>,
     pub request_ids: Vec<String>,
     pub trace_ids: Vec<String>,
     pub start_ms: i64,
@@ -184,6 +197,8 @@ pub struct SpendByResponseIdsRow {
     pub litellm_call_id: String,
     pub response_id: String,
     pub upstream_response_id: String,
+    #[serde(default)]
+    pub provider_request_id: String,
     pub trace_id: String,
     pub span_id: String,
     pub team_id: String,

litellm-rust/crates/traces/src/resolve/resolution.rs

@@ -3,6 +3,7 @@ use std::collections::HashMap;
 use indexmap::IndexMap;
 
 use crate::{
+    SpendMatch,
     normalize::{CallKey, ObservationType},
     query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow},
 };
@@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> {
     types: HashMap<&'a str, ObservationType>,
     tool_failures: HashMap<&'a str, &'a TraceSpansRow>,
     pub(super) model_calls: Vec<usize>,
+    call_matches: HashMap<usize, CallMatch<'a>>,
 }
 
+pub(super) type CallMatch<'a> = (Option<Requests<'a>>, SpendMatch);
+
 impl<'a> Resolution<'a> {
     pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self {
         let graph = Graph::new(rows);
@@ -36,7 +40,7 @@ impl<'a> Resolution<'a> {
         let types: HashMap<&str, ObservationType> = (0..rows.len())
             .map(|index| (graph.id(index), resolved_type(&graph, index, named_agents)))
             .collect();
-        let model_calls = (0..rows.len())
+        let model_calls: Vec<usize> = (0..rows.len())
             .filter(|index| {
                 types[graph.id(*index)] == ObservationType::Llm
                     && !graph
@@ -45,7 +49,7 @@ impl<'a> Resolution<'a> {
                         .any(|descendant| types[graph.id(descendant)] == ObservationType::Llm)
             })
             .collect();
-        Self {
+        let resolution = Self {
             ownership: Ownership {
                 team_id: &rows[0].team_id,
                 api_key_hash: &rows[0].api_key_hash,
@@ -65,6 +69,16 @@ impl<'a> Resolution<'a> {
                 .map(|row| (row.tool_call_id.as_str(), row))
                 .collect(),
             model_calls,
+            call_matches: HashMap::new(),
+        };
+        let call_matches = resolution
+            .model_calls
+            .iter()
+            .map(|call| (*call, resolution.resolve_call_match(*call)))
+            .collect();
+        Self {
+            call_matches,
+            ..resolution
         }
     }
 
@@ -118,7 +132,24 @@ impl<'a> Resolution<'a> {
         spend::requests(self.row(index), &self.ownership, self.spend)
     }
 
+    pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> {
+        self.call_matches.get(&call)
+    }
+
     pub(super) fn call_requests(&self, call: usize) -> Option<Requests<'a>> {
+        self.call_match(call)
+            .and_then(|(requests, _)| requests.clone())
+    }
+
+    fn resolve_call_match(&self, call: usize) -> CallMatch<'a> {
+        if let Some(requests) = self.resolve_call_requests(call) {
+            return (Some(requests), SpendMatch::Matched);
+        }
+        let evidence = self.requests(call);
+        (None, evidence.unmatched_reason())
+    }
+
+    fn resolve_call_requests(&self, call: usize) -> Option<Requests<'a>> {
         let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| {
             self.kind(*ancestor) == ObservationType::Llm
                 && self

litellm-rust/crates/traces/src/resolve/spend.rs

@@ -3,7 +3,7 @@ use std::collections::BTreeSet;
 use indexmap::IndexMap;
 
 use crate::{
-    CallEvidence, CallEvidenceKind, CallKey,
+    CallEvidence, CallEvidenceKind, CallKey, SpendMatch,
     query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow},
 };
 
@@ -12,6 +12,7 @@ use crate::{
 pub struct SpendLookup {
     pub response_ids: Vec<String>,
     pub request_ids: Vec<String>,
+    pub provider_request_ids: Vec<String>,
     /// Traces whose transport spans LiteLLM logged by `traceparent`.
     pub trace_ids: Vec<String>,
 }
@@ -45,13 +46,21 @@ impl SpendLookup {
                     })
                     .collect(),
             ),
+            provider_request_ids: sorted(
+                keys()
+                    .filter_map(|(_, key)| match key {
+                        CallKey::ProviderRequest(id) => Some(id.clone()),
+                        _ => None,
+                    })
+                    .collect(),
+            ),
             trace_ids: sorted(
                 keys()
                     .filter_map(|(row, key)| match key {
                         CallKey::Transport | CallKey::GatewayAttempt
-                            if !row.trace_id.is_empty() =>
+                            if !row.transport_trace_id().is_empty() =>
                         {
-                            Some(row.trace_id.clone())
+                            Some(row.transport_trace_id().to_owned())
                         }
                         _ => None,
                     })
@@ -61,7 +70,10 @@ impl SpendLookup {
     }
 
     pub fn is_empty(&self) -> bool {
-        self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty()
+        self.response_ids.is_empty()
+            && self.request_ids.is_empty()
+            && self.provider_request_ids.is_empty()
+            && self.trace_ids.is_empty()
     }
 }
 
@@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>;
 enum KeyFamily {
     GatewayCall,
     ProviderResponse,
+    ProviderRequest,
     Transport,
 }
 
@@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily {
     match key {
         CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall,
         CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse,
+        CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest,
         CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport,
     }
 }
 
 pub(super) enum KeyMatch<'a> {
     Missing,
+    Conflicting,
     Unique(&'a SpendRow),
     Ambiguous(Requests<'a>),
 }
@@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> {
     fn unique(&self) -> Option<&'a SpendRow> {
         match self {
             Self::Unique(request) => Some(request),
-            Self::Missing | Self::Ambiguous(_) => None,
+            Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None,
         }
     }
 
     fn agrees_with(&self, selected: &[&SpendRow]) -> bool {
         match self {
-            Self::Missing => false,
+            Self::Missing | Self::Conflicting => false,
             Self::Unique(request) => selected
                 .iter()
                 .any(|row| row.identity() == request.identity()),
@@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> {
 }
 
 impl<'a> SpendEvidence<'a> {
+    pub(super) fn unmatched_reason(&self) -> SpendMatch {
+        match self {
+            Self::Unknown => SpendMatch::NoCallId,
+            Self::Partial(_) => SpendMatch::IncompleteEvidence,
+            Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId,
+            Self::Complete(matches)
+                if matches.iter().any(|evidence| {
+                    matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_))
+                }) =>
+            {
+                SpendMatch::Ambiguous
+            }
+            Self::Complete(matches)
+                if matches
+                    .iter()
+                    .any(|evidence| matches!(evidence, KeyMatch::Missing)) =>
+            {
+                SpendMatch::NoSpendLog
+            }
+            Self::Complete(_) => SpendMatch::Ambiguous,
+        }
+    }
+
     pub(super) fn complete_requests(&self) -> Option<Requests<'a>> {
         match self {
             Self::Complete(matches) if !matches.is_empty() => {
@@ -186,15 +224,16 @@ fn matches<'a>(
         CallKey::ProviderResponse(id) => {
             !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id)
         }
+        CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id,
         CallKey::LiteLlmRequest(id) => {
             !id.is_empty()
                 && (spend.litellm_call_id == *id
                     || (spend.litellm_call_id.is_empty() && spend.request_id == *id))
         }
         CallKey::Transport | CallKey::GatewayAttempt => {
-            !row.trace_id.is_empty()
+            !row.transport_trace_id().is_empty()
                 && !row.span_id.is_empty()
-                && spend.trace_id == row.trace_id
+                && spend.trace_id == row.transport_trace_id()
                 && spend.span_id == row.span_id
         }
     };
@@ -253,7 +292,8 @@ pub(super) fn requests<'a>(
     let matches = aliases
         .into_iter()
         .map(|(_, requests)| {
-            KeyMatch::new(
+            let had_candidates = !requests.is_empty();
+            let matched = KeyMatch::new(
                 requests
                     .into_iter()
                     .filter(|request| {
@@ -262,7 +302,12 @@ pub(super) fn requests<'a>(
                             .all(|family| family.contains(&request.identity()))
                     })
                     .collect(),
-            )
+            );
+            if had_candidates && matches!(matched, KeyMatch::Missing) {
+                KeyMatch::Conflicting
+            } else {
+                matched
+            }
         })
         .collect();
     match evidence.kind() {
@@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option<f64> {
     })
 }
 
-pub(super) fn total(calls: &[Option<Requests<'_>>]) -> Option<f64> {
-    if calls.is_empty() {
-        return None;
-    }
-    let requests: Option<Vec<&SpendRow>> = calls
-        .iter()
-        .map(|requests| requests.as_ref())
-        .collect::<Option<Vec<_>>>()
-        .map(|calls| calls.into_iter().flatten().copied().collect());
-    let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests?
+pub(super) fn unique<'a>(requests: impl IntoIterator<Item = &'a SpendRow>) -> Requests<'a> {
+    requests
         .into_iter()
         .map(|request| (request.identity(), request))
-        .collect();
-    request_cost(&unique.into_values().collect::<Vec<_>>())
+        .collect::<IndexMap<_, _>>()
+        .into_values()
+        .collect()
+}
+
+pub(super) struct Priced {
+    pub(super) spend: Option<f64>,
+    pub(super) priced_calls: u64,
+}
+
+pub(super) fn total(calls: &[Option<Requests<'_>>]) -> Priced {
+    let priced: Vec<&Requests<'_>> = calls
+        .iter()
+        .flatten()
+        .filter(|requests| request_cost(requests).is_some())
+        .collect();
+    let spend = if priced.is_empty() {
+        None
+    } else {
+        request_cost(&unique(
+            priced.iter().flat_map(|requests| requests.iter().copied()),
+        ))
+    };
+    Priced {
+        spend,
+        priced_calls: priced.len() as u64,
+    }
 }

litellm-rust/crates/traces/src/resolve/view.rs

@@ -6,7 +6,7 @@ use time::OffsetDateTime;
 use crate::{
     normalize::ObservationType,
     query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow},
-    view::{AgentNode, Span, SpanStatus, Trace, TraceSummary},
+    view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary},
 };
 
 use super::{
@@ -23,7 +23,18 @@ fn optional(value: &str) -> Option<String> {
 fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span {
     let row = resolution.row(index);
     let status = resolution.status_source(index);
-    let requests = resolution.requests(index).complete_requests();
+    let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) {
+        (requests.clone(), Some(*matched))
+    } else {
+        (resolution.requests(index).complete_requests(), None)
+    };
+    let spend = requests
+        .as_ref()
+        .and_then(|requests| request_cost(requests));
+    let spend_log_request_id = match (spend_match, requests.as_deref()) {
+        (Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()),
+        _ => None,
+    };
     Span {
         span_id: row.span_id.clone(),
         parent_span_id: optional(&row.parent_span_id),
@@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span
         input_tokens: row.input_tokens,
         output_tokens: row.output_tokens,
         litellm_request_id: optional(&row.litellm_request_id),
-        spend: requests
-            .as_ref()
-            .and_then(|requests| request_cost(requests)),
+        spend,
+        spend_log_request_id,
+        spend_match,
     }
 }
 
@@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
                 .filter(|(owner, _)| *owner == name)
                 .map(|(_, requests)| requests.clone())
                 .collect();
+            let priced = total(&owned_calls);
             AgentNode {
                 name: name.to_owned(),
                 parent_agent,
@@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec<AgentNode> {
                     .map(|span| graph.rows[*span].duration_ns)
                     .sum::<u64>() as f64
                     / NANOS_PER_MS,
-                spend: total(&owned_calls),
+                spend: priced.spend,
+                priced_calls: priced.priced_calls,
             }
         })
         .collect()
@@ -160,6 +173,12 @@ pub fn resolve_trace(
     } else {
         calls.iter().map(|call| &rows[*call]).collect()
     };
+    let priced = total(
+        &calls
+            .iter()
+            .map(|call| resolution.call_requests(*call))
+            .collect::<Vec<_>>(),
+    );
     let first_input = spans
         .iter()
         .zip(rows)
@@ -208,12 +227,8 @@ pub fn resolve_trace(
         input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(),
         output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(),
         models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())),
-        spend: total(
-            &calls
-                .iter()
-                .map(|call| resolution.call_requests(*call))
-                .collect::<Vec<_>>(),
-        ),
+        spend: priced.spend,
+        priced_calls: priced.priced_calls,
     };
     Some(Trace {
         summary,
@@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary {
         output_tokens: row.output_tokens,
         models: row.models.clone(),
         spend: None,
+        priced_calls: 0,
     }
 }

litellm-rust/crates/traces/src/view.rs

@@ -37,6 +37,19 @@ pub struct Span {
     pub output_tokens: u32,
     pub litellm_request_id: Option<String>,
     pub spend: Option<f64>,
+    pub spend_log_request_id: Option<String>,
+    pub spend_match: Option<SpendMatch>,
+}
+
+#[macro_rules_attribute::apply(response_type)]
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+#[serde(rename_all = "snake_case")]
+pub enum SpendMatch {
+    Matched,
+    NoCallId,
+    NoSpendLog,
+    Ambiguous,
+    IncompleteEvidence,
 }
 
 /// One distinct agent in a trace: 200 invocations of `researcher` are one node.
@@ -50,6 +63,7 @@ pub struct AgentNode {
     pub tool_calls: u64,
     pub duration_ms: f64,
     pub spend: Option<f64>,
+    pub priced_calls: u64,
 }
 
 #[macro_rules_attribute::apply(response_type)]
@@ -80,6 +94,7 @@ pub struct TraceSummary {
     pub output_tokens: u64,
     pub models: Vec<String>,
     pub spend: Option<f64>,
+    pub priced_calls: u64,
 }
 
 #[macro_rules_attribute::apply(response_type)]

litellm-rust/crates/traces/tests/captures.rs

@@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec<SpendByResponse
                 litellm_call_id: record.litellm_call_id,
                 response_id: record.response_id,
                 upstream_response_id,
+                provider_request_id: String::new(),
                 trace_id: record.trace_id,
                 span_id: record.span_id,
                 team_id: record.team_id,
@@ -171,12 +172,13 @@ fn trace_span(span: DecodedSpan) -> TraceSpansRow {
         .into_iter()
         .flatten()
         .find_map(|key| match key {
-            CallKey::ProviderResponse(id) => Some(id.clone()),
+            CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()),
             CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None,
         })
         .unwrap_or_default();
     TraceSpansRow {
         trace_id: span.trace_id,
+        original_trace_id: String::new(),
         span_id: span.span_id,
         parent_span_id: span.parent_span_id,
         name: span.name,
@@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow {
             .expect("valid unrelated transport timestamp");
     TraceSpansRow {
         trace_id: call.trace_id.clone(),
+        original_trace_id: call.original_trace_id.clone(),
         span_id: format!("unrelated-transport-{}", call.span_id),
         parent_span_id: call.parent_span_id.clone(),
         name: "unrelated-http".into(),
@@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence(
             let response_ids: Vec<_> = keys
                 .iter()
                 .filter_map(|key| match key {
-                    CallKey::ProviderResponse(id) => Some(id.clone()),
+                    CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => {
+                        Some(id.clone())
+                    }
                     CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => {
                         None
                     }

litellm-rust/crates/traces/tests/normalization_formats.rs

@@ -928,3 +928,28 @@ fn convention_markers_keep_genai_call_evidence(
     let marked = decode(span, scope, &marked_attributes, vec![]).unwrap();
     assert_eq!(marked.normalized.calls, plain.normalized.calls);
 }
+
+#[rstest]
+#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))]
+#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))]
+fn native_claude_preserves_the_provider_id_family(
+    span: Span,
+    #[case] id: &str,
+    #[case] key: CallKey,
+) {
+    let native = Span {
+        name: "claude_code.llm_request".into(),
+        ..span
+    };
+    let decoded = decode(
+        native,
+        "com.anthropic.claude_code.tracing",
+        &[("gen_ai.response.id", id)],
+        Vec::new(),
+    )
+    .unwrap();
+    assert_eq!(
+        decoded.normalized.calls,
+        CallEvidence::Complete(std::collections::BTreeSet::from([key]))
+    );
+}

litellm-rust/crates/traces/tests/normalize.rs

@@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) {
 #[rstest]
 #[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))]
 #[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))]
+#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))]
 #[case::transport(litellm_traces::CallKey::Transport)]
 #[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)]
 fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) {
@@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) {
 #[rstest]
 #[case::missing_separator("provider_response")]
 #[case::missing_response("provider_response:")]
+#[case::missing_provider_request("provider_request:")]
 #[case::missing_request("litellm_request:")]
 #[case::transport_id("transport:unexpected")]
 #[case::gateway_attempt_separator("gateway_attempt")]

litellm-rust/crates/traces/tests/query/named.rs

@@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases(
         json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}),
     ));
     round_trip::<SpendByResponseIdsParams>(request(
-        json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
+        json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}),
     ));
 }
 
@@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() {
         json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}),
     );
     round_trip::<TraceSpansRow>(
-        json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
+        json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}),
     );
     round_trip::<SpanDetailRow>(
         json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}),
@@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() {
         json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}),
     );
     round_trip::<SpendByResponseIdsRow>(
-        json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
+        json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}),
     );
 }

litellm/caching/caching_handler.py

@@ -35,7 +35,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
 from litellm.litellm_core_utils.logging_utils import (
     _assemble_complete_response_from_streaming_chunks,
 )
-from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding
 from litellm.types.integrations.custom_logger import converted_stream_requested
 from litellm.types.llms.base import LiteLLMBaseModel
 from litellm.types.llms.openai import ResponsesAPIResponse
@@ -43,9 +43,11 @@ from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
     CachingDetails,
     CallTypes,
+    Choices,
     Embedding,
     EmbeddingResponse,
     ModelResponse,
+    TextChoices,
     TextCompletionResponse,
     TranscriptionResponse,
     Usage,
@@ -111,6 +113,43 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool:
     return "choices" in cached_result
 
 
+def _starts_content_block(event: object) -> bool:
+    text: Final = event.decode("utf-8", errors="replace") if isinstance(event, bytes) else event
+    return isinstance(text, str) and "event: content_block_start" in text.splitlines()
+
+
+def is_response_without_output(result: object) -> bool:
+    if isinstance(result, (ModelResponse, TextCompletionResponse)):
+        return not result.choices
+    if isinstance(result, ResponsesAPIResponse):
+        return not getattr(result, "output", None)
+    if not isinstance(result, dict):
+        return False
+    cached_stream_events: Final = result.get(CACHED_STREAM_EVENTS_KEY)
+    if isinstance(cached_stream_events, (list, tuple)):
+        return not any(_starts_content_block(event) for event in cached_stream_events)
+    if "choices" in result:
+        return not result["choices"]
+    if result.get("object") == "response":
+        return not result.get("output")
+    if result.get("type") == "message":
+        return not result.get("content")
+    return False
+
+
+def _choice_carries_output(choice: Choices | TextChoices) -> bool:
+    payload: Final[Mapping[str, object]] = (
+        choice.message.model_dump(exclude_none=True) if isinstance(choice, Choices) else {"text": choice.text}
+    )
+    return any(key != "role" and value not in ("", [], {}) for key, value in payload.items())
+
+
+def _assembled_stream_without_output(response: ModelResponse | TextCompletionResponse) -> bool:
+    """The stream wrapper closes a stream whose chunks carried no choice with an empty choice of its own, so the
+    assembled response has choices and the no-output check on it alone would store that empty answer."""
+    return not any(_choice_carries_output(choice) for choice in response.choices)
+
+
 def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool:
     if kwargs.get("stream", False) is True:
         return True
@@ -398,6 +437,9 @@ class LLMCachingHandler:
             print_verbose("Checking Sync Cache")
             with response_cache_phase("get"):
                 cached_result = litellm.cache.get_cache(**new_kwargs)
+            if is_response_without_output(cached_result):
+                verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
+                return CachingHandlerResponse(cached_result=None)
             if cached_result is not None:
                 if "detail" in cached_result:
                     # implies an error occurred
@@ -836,8 +878,19 @@ class LLMCachingHandler:
                         cache_key=self.preset_cache_key,
                         **request_kwargs,
                     )
+        if is_response_without_output(cached_result):
+            verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss")
+            self._forget_worker_copy()
+            return None
         return cached_result
 
+    def _forget_worker_copy(self) -> None:
+        """Drop this worker's memory copy of a stored response without output, so the next read reaches Redis,
+        where the refill lands: the text completion and messages writers skip the memory tier."""
+        if self.dual_cache is None or self.preset_cache_key is None:
+            return
+        self.dual_cache.in_memory_cache.delete_cache(self.preset_cache_key)
+
     def _convert_cached_result_to_model_response(
         self,
         cached_result: Any,
@@ -1063,6 +1116,9 @@ class LLMCachingHandler:
 
         if litellm.cache is None:
             return
+        if is_response_without_output(result):
+            verbose_logger.debug("LiteLLM Cache: not caching a response with no output")
+            return
         cache: Final = litellm.cache
 
         new_kwargs: Final = kwargs.copy()
@@ -1113,6 +1169,11 @@ class LLMCachingHandler:
         """
         Sync internal method to add the result to the cache
         """
+        if litellm.cache is None:
+            return
+        if is_response_without_output(result):
+            verbose_logger.debug("LiteLLM Cache: not caching a response with no output")
+            return
 
         new_kwargs: Final = kwargs.copy()
         new_kwargs.update(
@@ -1121,8 +1182,6 @@ class LLMCachingHandler:
                 args,
             )
         )
-        if litellm.cache is None:
-            return
 
         if self._should_store_result_in_cache(original_function=self.original_function, kwargs=new_kwargs):
             with response_cache_phase("set"):
@@ -1201,13 +1260,13 @@ class LLMCachingHandler:
                 is_async=True,
             )
         )
-        # if a complete_streaming_response is assembled, add it to the cache
-        if complete_streaming_response is not None:
-            await self.async_set_cache(
-                result=complete_streaming_response,
-                original_function=self.original_function,
-                kwargs=self.request_kwargs,
-            )
+        if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response):
+            return
+        await self.async_set_cache(
+            result=complete_streaming_response,
+            original_function=self.original_function,
+            kwargs=self.request_kwargs,
+        )
 
     def _sync_add_streaming_response_to_cache(self, processed_chunk: ModelResponse):
         """
@@ -1224,12 +1283,12 @@ class LLMCachingHandler:
             )
         )
 
-        # if a complete_streaming_response is assembled, add it to the cache
-        if complete_streaming_response is not None:
-            self.sync_set_cache(
-                result=complete_streaming_response,
-                kwargs=self.request_kwargs,
-            )
+        if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response):
+            return
+        self.sync_set_cache(
+            result=complete_streaming_response,
+            kwargs=self.request_kwargs,
+        )
 
     def _update_litellm_logging_obj_environment(
         self,

litellm/integrations/arize/arize.py

@@ -17,7 +17,7 @@ from litellm.integrations.arize._utils import ArizeOTELAttributes
 from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig
 from litellm.types.integrations.arize import ArizeConfig
 from litellm.types.services import ServiceLoggerPayload
-from litellm.types.utils import StandardCallbackDynamicParams
+from litellm.types.utils import ArizeOtlpProtocol, StandardCallbackDynamicParams
 
 if TYPE_CHECKING:
     from opentelemetry.trace import Span as _Span
@@ -174,7 +174,7 @@ class ArizeLogger(OpenTelemetry):
         _utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes)
 
     @staticmethod
-    def get_arize_config() -> ArizeConfig:
+    def get_arize_config(otlp_protocol: ArizeOtlpProtocol | None = None) -> ArizeConfig:
         """
         Helper function to get Arize configuration.
 
@@ -195,7 +195,19 @@ class ArizeLogger(OpenTelemetry):
         endpoint = None
         protocol: Protocol = "otlp_grpc"
 
-        if grpc_endpoint:
+        if otlp_protocol == "http/protobuf":
+            protocol = "otlp_http"
+            if http_endpoint:
+                endpoint = http_endpoint
+            elif grpc_endpoint:
+                base: Final = grpc_endpoint.rstrip("/")
+                endpoint = f"{base}/traces" if base.endswith("/v1") else base
+            else:
+                endpoint = "https://otlp.arize.com/v1/traces"
+        elif otlp_protocol == "grpc":
+            protocol = "otlp_grpc"
+            endpoint = grpc_endpoint or http_endpoint or "https://otlp.arize.com/v1"
+        elif grpc_endpoint:
             protocol = "otlp_grpc"
             endpoint = grpc_endpoint
         elif http_endpoint:

litellm/integrations/clickhouse/clickhouse_spend_logger.py

@@ -19,6 +19,7 @@ from litellm._logging import verbose_logger
 from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger
 from litellm.integrations.clickhouse.context import is_lens_analysis
 from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE
+from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
 from litellm.tracing.types import SpendLogRecord
 from litellm.types.utils import StandardLoggingPayload
@@ -155,6 +156,11 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[
     return SpendLogRecord(
         request_id=request_id,
         response_id=strip_cache_hit_suffix(request_id),
+        provider_request_id=(
+            get_provider_request_id(kwargs.get("response_headers"))
+            or get_provider_request_id(hidden_params.get("additional_headers"))
+            or ""
+        ),
         litellm_call_id=payload.get("litellm_call_id") or "",
         call_type=payload.get("call_type") or "",
         api_key=metadata.get("user_api_key_hash") or "",

litellm/integrations/otel/presets/destinations.py

@@ -63,7 +63,7 @@ def _langfuse_destination(params: StandardCallbackDynamicParams) -> "_Destinatio
 def _arize_destination(params: StandardCallbackDynamicParams) -> "_Destination | None":
     from litellm.integrations.arize.arize import ArizeLogger
 
-    config: Final = ArizeLogger.get_arize_config()
+    config: Final = ArizeLogger.get_arize_config(params.get("arize_otlp_protocol"))
     return (config.endpoint, config.protocol)
 
 

litellm/integrations/websearch_interception/handler.py

@@ -1557,8 +1557,8 @@ class WebSearchInterceptionLogger(CustomLogger):
                 search_litellm_params = dict[str, object](tool_params)
                 search_provider = tool_params.get("search_provider")
 
-            # Fallback to perplexity if no router or no search tools configured
             if not search_provider:
+                self._authorize_unregistered_search_fallback(kwargs=kwargs)
                 search_provider = "perplexity"
                 verbose_logger.debug(
                     "WebSearchInterception: No search tools configured in router, using default provider '%s'",
@@ -1623,6 +1623,18 @@ class WebSearchInterceptionLogger(CustomLogger):
             verbose_logger.error("WebSearchInterception: Search failed for '%s': %s", query, e)
             raise
 
+    def _authorize_unregistered_search_fallback(self, kwargs: Mapping[str, object] | None) -> None:
+        user_api_key_auth: Final = self._get_user_api_key_auth_from_kwargs(kwargs)
+        if user_api_key_auth is None:
+            return
+
+        from litellm.proxy.auth.auth_checks import check_unregistered_search_fallback, typed_general_settings
+        from litellm.proxy.proxy_server import general_settings
+
+        check_unregistered_search_fallback(
+            valid_token=user_api_key_auth, general_settings=typed_general_settings(general_settings)
+        )
+
     async def _authorize_search_tool(
         self,
         search_tool: Mapping[str, object],
@@ -1636,36 +1648,9 @@ class WebSearchInterceptionLogger(CustomLogger):
         if user_api_key_auth is None:
             return
 
-        from litellm.proxy.auth.auth_checks import (
-            can_key_call_search_tool,
-            can_team_call_search_tool,
-            get_team_object,
-        )
+        from litellm.proxy.auth.auth_checks import can_token_call_search_tool
 
-        await can_key_call_search_tool(
-            search_tool_name=search_tool_name,
-            valid_token=user_api_key_auth,
-        )
-
-        team_id: Final[str | None] = getattr(user_api_key_auth, "team_id", None)
-        if team_id:
-            from litellm.proxy.proxy_server import (
-                prisma_client,
-                proxy_logging_obj,
-                user_api_key_cache,
-            )
-
-            team_object: Final = await get_team_object(
-                team_id=team_id,
-                prisma_client=prisma_client,
-                user_api_key_cache=user_api_key_cache,
-                parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None),
-                proxy_logging_obj=proxy_logging_obj,
-            )
-            await can_team_call_search_tool(
-                search_tool_name=search_tool_name,
-                team_object=team_object,
-            )
+        await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=user_api_key_auth)
 
     @staticmethod
     def _build_search_request_metadata(

litellm/litellm_core_utils/core_helpers.py

@@ -11,6 +11,7 @@ import httpx
 from pydantic import TypeAdapter, ValidationError
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id
 from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason
 
 if TYPE_CHECKING:
@@ -495,7 +496,8 @@ def process_response_headers(
         **processed_headers,
         **additional_headers,
     }
-    return additional_headers
+    request_id: Final = get_provider_request_id(response_headers)
+    return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})}
 
 
 def preserve_upstream_non_openai_attributes(

litellm/litellm_core_utils/error_normalization.py

@@ -107,6 +107,10 @@ _PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType(
         "key_vector_store_access_denied": PERMISSION_DENIED,
         "team_vector_store_access_denied": PERMISSION_DENIED,
         "org_vector_store_access_denied": PERMISSION_DENIED,
+        "user_vector_store_access_denied": PERMISSION_DENIED,
+        "key_search_tool_access_denied": PERMISSION_DENIED,
+        "team_search_tool_access_denied": PERMISSION_DENIED,
+        "user_search_tool_access_denied": PERMISSION_DENIED,
         "tool_access_denied": PERMISSION_DENIED,
         "team_member_permission_error": PERMISSION_DENIED,
         "not_found_error": RESOURCE_NOT_FOUND,

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -4,7 +4,12 @@ from contextlib import contextmanager
 from contextvars import ContextVar
 from typing import Any, Final
 
-from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams
+from litellm.types.utils import (
+    ARIZE_OTLP_PROTOCOLS,
+    OTEL_SPAN_SCOPES,
+    TRUSTED_CALLBACK_VARS_FIELD,
+    StandardCallbackDynamicParams,
+)
 
 _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata")
 _inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar(
@@ -81,6 +86,11 @@ def validate_langfuse_span_scope_value(value: str) -> None:
         raise ValueError(f"Invalid langfuse_span_scope {value!r}: must be one of {sorted(OTEL_SPAN_SCOPES)}")
 
 
+def validate_arize_otlp_protocol_value(value: str) -> None:
+    if value not in ARIZE_OTLP_PROTOCOLS:
+        raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}")
+
+
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
 _supported_callback_params: Final[tuple[str, ...]] = (
     "langfuse_public_key",

litellm/litellm_core_utils/llm_response_utils/get_headers.py

@@ -1,6 +1,20 @@
 from collections.abc import Mapping
 from typing import Final
 
+from pydantic import TypeAdapter, ValidationError
+
+_HEADERS: Final = TypeAdapter(Mapping[str, str])
+_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id")
+
+
+def get_provider_request_id(headers: object) -> str | None:
+    try:
+        values: Final = _HEADERS.validate_python(headers)
+    except ValidationError:
+        return None
+    normalized: Final = {name.lower(): value for name, value in values.items()}
+    return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None)
+
 
 def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict:
     """
@@ -29,7 +43,9 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) ->
     if "x-ratelimit-remaining-tokens" in _response_headers:
         openai_headers["x-ratelimit-remaining-tokens"] = _response_headers["x-ratelimit-remaining-tokens"]
     llm_provider_headers: Final = _get_llm_provider_headers(_response_headers)
-    return {**llm_provider_headers, **openai_headers}
+    request_id: Final = get_provider_request_id(_response_headers)
+    native_headers: Final = {"request-id": request_id} if request_id is not None else {}
+    return {**llm_provider_headers, **openai_headers, **native_headers}
 
 
 def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict:

litellm/llms/anthropic/pass_through/messages/response_cache.py

@@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
 
 import litellm
 from litellm._logging import verbose_logger
-from litellm.caching.caching_handler import create_cache_write_task
+from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
 from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
     AnthropicMessagesStreamingResponse,
     BaseAnthropicMessagesStreamingIterator,
@@ -13,6 +13,7 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import (
     _is_provider_error_chunk,
     aclose_if_supported,
 )
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY
 
 if TYPE_CHECKING:
     from litellm.caching.caching_handler import LLMCachingHandler
@@ -20,8 +21,6 @@ if TYPE_CHECKING:
     from litellm.types.llms.openai import AllMessageValues
     from litellm.types.utils import ModelResponseStream
 
-CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events"
-
 _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({})
 
 _SSE_EVENT_BOUNDARY: Final = re.compile(r"(?<=\n\n)")
@@ -114,6 +113,9 @@ class AnthropicMessagesStreamCacheWriter:
             verbose_logger.exception("Anthropic Messages stream cache write failed: %s", e)
             return
         cached_payload: Final = {CACHED_STREAM_EVENTS_KEY: events}
+        if is_response_without_output(cached_payload):
+            verbose_logger.debug("LiteLLM Cache: not caching a stream with no content blocks")
+            return
         dual_cache: Final = self.caching_handler.dual_cache
 
         async def _write() -> None:

litellm/llms/custom_httpx/llm_http_handler.py

@@ -2214,6 +2214,7 @@ class BaseLLMHTTPHandler:
 
         # used for logging + cost tracking
         logging_obj.model_call_details["httpx_response"] = response
+        logging_obj.model_call_details["response_headers"] = dict(response.headers)
 
         initial_response: AsyncIterator | AnthropicMessagesResponse
         if stream:

litellm/proxy/_types.py

@@ -25,6 +25,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict
 from litellm._uuid import uuid
 from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS
 from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+    validate_arize_otlp_protocol_value,
     validate_langfuse_environment_value,
     validate_langfuse_span_scope_value,
     validate_no_callback_env_reference,
@@ -2383,6 +2384,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase):
                 validate_langfuse_environment_value(callback_vars[key])
             if key == "langfuse_span_scope":
                 validate_langfuse_span_scope_value(callback_vars[key])
+            if key == "arize_otlp_protocol":
+                validate_arize_otlp_protocol_value(callback_vars[key])
         return values
 
 
@@ -3019,6 +3022,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
         None,
         description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.",
     )
+    vector_store_deny_by_default: bool = Field(
+        default=False,
+        description="When True, a vector store must be explicitly listed in object_permission.vector_stores: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing. Dashboard session keys are not yet covered",
+    )
+    search_tool_deny_by_default: bool = Field(
+        default=False,
+        description="When True, a search tool must be explicitly listed in object_permission.search_tools: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing, and the unregistered search fallback is denied. The master key and dashboard sessions are exempt",
+    )
     missing_session_id: Literal["generate", "reject", "omit"] | None = Field(
         None,
         description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.",
@@ -4509,6 +4520,26 @@ class ProxyErrorTypes(str, enum.Enum):
     Organization does not have access to the vector store
     """
 
+    user_vector_store_access_denied = "user_vector_store_access_denied"
+    """
+    User does not have access to the vector store
+    """
+
+    key_search_tool_access_denied = "key_search_tool_access_denied"
+    """
+    Key does not have access to the search tool
+    """
+
+    team_search_tool_access_denied = "team_search_tool_access_denied"
+    """
+    Team does not have access to the search tool
+    """
+
+    user_search_tool_access_denied = "user_search_tool_access_denied"
+    """
+    User does not have access to the search tool
+    """
+
     team_member_already_in_team = "team_member_already_in_team"
     """
     Team member is already in team
@@ -4543,7 +4574,7 @@ class ProxyErrorTypes(str, enum.Enum):
 
     @classmethod
     def get_vector_store_access_error_type_for_object(
-        cls, object_type: Literal["key", "team", "org"]
+        cls, object_type: Literal["key", "team", "org", "user"]
     ) -> "ProxyErrorTypes":
         """
         Get the vector store access error type for object_type
@@ -4554,6 +4585,18 @@ class ProxyErrorTypes(str, enum.Enum):
             return cls.team_vector_store_access_denied
         elif object_type == "org":
             return cls.org_vector_store_access_denied
+        elif object_type == "user":
+            return cls.user_vector_store_access_denied
+
+    @classmethod
+    def get_search_tool_access_error_type_for_object(
+        cls, object_type: Literal["key", "team", "user"]
+    ) -> "ProxyErrorTypes":
+        return {
+            "key": cls.key_search_tool_access_denied,
+            "team": cls.team_search_tool_access_denied,
+            "user": cls.user_search_tool_access_denied,
+        }[object_type]
 
 
 DB_CONNECTION_ERROR_TYPES: Final = (

litellm/proxy/auth/auth_checks.py

@@ -14,7 +14,9 @@ import math
 import re
 import time
 from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
+from dataclasses import dataclass
 from functools import partial
+from itertools import chain
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, Protocol, TypeAlias
 
@@ -34,6 +36,7 @@ from litellm.constants import (
     DEFAULT_MAX_RECURSE_DEPTH,
     EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE,
     END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
+    LITELLM_PROXY_MASTER_KEY_ALIAS,
     MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE,
     REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
     TAG_REGISTRY_MAX_SIZE,
@@ -44,7 +47,9 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 from litellm.models.project import LiteLLM_ProjectTable
 from litellm.proxy._types import (
     RBAC_ROLES,
+    UI_TEAM_ID,
     CallInfo,
+    ConfigGeneralSettings,
     LiteLLM_AccessGroupTable,
     LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
@@ -308,17 +313,14 @@ def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_P
     return _DeadlineBoundedTable(repo.table, "user")
 
 
+GrantLayer = Literal["key", "team", "user"]
+
+
 class _VectorStorePermissionsRow(Protocol):
     @property
     def vector_stores(self) -> Sequence[str] | None: ...
 
 
-def _object_permission_table(
-    repo: _PrismaTableHolder[_VectorStorePermissionsRow],
-) -> _PrismaAuthTable[_VectorStorePermissionsRow]:
-    return _DeadlineBoundedTable(repo.table, "object_permission")
-
-
 class _PrismaTagRow(Protocol):
     tag_name: str
 
@@ -388,6 +390,9 @@ def _typed_request_body(request_body: dict) -> Mapping[str, object]:
     return request_body
 
 
+typed_general_settings: Final = _typed_request_body
+
+
 class _JsonLoadsObj(Protocol):
     def __call__(self, data: str) -> object: ...
 
@@ -1353,6 +1358,8 @@ async def common_checks(
             request_body=request_body,
             team_object=team_object,
             valid_token=valid_token,
+            user_object=user_object,
+            deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)),
         )
 
     # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path)
@@ -5481,10 +5488,10 @@ def _search_tool_names_from_object_permission(
 def _can_object_call_search_tool(
     search_tool_name: str,
     allowed_search_tools: list[str],
-    object_type: Literal["key", "team", "project"],
+    object_type: Literal["key", "team", "project", "user"],
 ) -> Literal[True]:
     """
-    Check if an object (key/team/project) can access a specific search tool.
+    Check if an object (key/team/project/user) can access a specific search tool.
 
     Similar to _can_object_call_model but for search tools.
 
@@ -5517,84 +5524,188 @@ def _can_object_call_search_tool(
     )
 
 
-async def can_key_call_search_tool(
-    search_tool_name: str,
-    valid_token: UserAPIKeyAuth,
-) -> Literal[True]:
+TeamObjectLoader: TypeAlias = Callable[[], Awaitable[LiteLLM_TeamTable | None]]
+
+
+@dataclass(frozen=True)
+class SearchToolGrants:
     """
-    Check if a key can access a specific search tool.
-
-    Similar to can_key_call_model but for search tools.
-
-    Args:
-        search_tool_name: The search tool being requested
-        valid_token: The authenticated key
-
-    Returns:
-        True if access is allowed
-
-    Raises:
-        ProxyException if access is denied
+    The key, team and user grants that scope one caller's search tools. Every layer in `strict_layers`
+    must list the tool, and any other loaded grant only narrows when its list is nonempty
     """
-    return _can_object_call_search_tool(
-        search_tool_name=search_tool_name,
-        allowed_search_tools=_search_tool_names_from_object_permission(valid_token.object_permission),
-        object_type="key",
-    )
+
+    grants: Mapping[GrantLayer, LiteLLM_ObjectPermissionTable | None]
+    strict_layers: frozenset[GrantLayer]
 
 
-async def can_team_call_search_tool(
-    search_tool_name: str,
-    team_object: LiteLLM_TeamTable | None,
-) -> Literal[True]:
+def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool:
     """
-    Check if a team can access a specific search tool.
-
-    Similar to can_team_access_model but for search tools.
-
-    Args:
-        search_tool_name: The search tool being requested
-        team_object: The team object
-
-    Returns:
-        True if access is allowed
-
-    Raises:
-        ProxyException if access is denied
-    """
-    if team_object is None:
-        return True
-
-    return _can_object_call_search_tool(
-        search_tool_name=search_tool_name,
-        allowed_search_tools=_search_tool_names_from_object_permission(team_object.object_permission),
-        object_type="team",
-    )
-
-
-async def can_user_view_search_tool(
-    search_tool_name: str,
-    valid_token: UserAPIKeyAuth,
-    team_object: LiteLLM_TeamTable | None,
-) -> bool:
-    """
-    Boolean variant of the key + team authorization enforced on /search, used to
-    scope /search_tools/list so a non-admin caller only sees tools it may invoke.
+    Startup rejects a non-boolean value from the config file. A non-boolean value that reaches
+    general_settings another way enables the policy, so only search tool requests are denied.
     """
     try:
-        await can_key_call_search_tool(
-            search_tool_name=search_tool_name,
-            valid_token=valid_token,
+        return ConfigGeneralSettings.model_validate(
+            MappingProxyType(
+                {"search_tool_deny_by_default": general_settings.get("search_tool_deny_by_default", False)}
+            )
+        ).search_tool_deny_by_default
+    except ValidationError:
+        return True
+
+
+def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool:
+    return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token)
+
+
+def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException:
+    return ProxyException(
+        message=message,
+        type=ProxyErrorTypes.get_search_tool_access_error_type_for_object(object_type),
+        param="search_tool_name",
+        code=status.HTTP_403_FORBIDDEN,
+    )
+
+
+async def _strict_team_object(load_team_object: TeamObjectLoader) -> LiteLLM_TeamTable | None:
+    try:
+        return await load_team_object()
+    except Exception as e:  # noqa: BLE001  # an unresolved team grants nothing under deny-by-default
+        verbose_proxy_logger.debug("Team lookup failed under search_tool_deny_by_default: %s", e)
+        return None
+
+
+async def _strict_user_object(
+    valid_token: UserAPIKeyAuth, prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache
+) -> LiteLLM_UserTable | None:
+    if valid_token.user_id is None:
+        return None
+    try:
+        return await get_user_object(
+            user_id=valid_token.user_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            user_id_upsert=False,
+            parent_otel_span=valid_token.parent_otel_span,
         )
-        await can_team_call_search_tool(
-            search_tool_name=search_tool_name,
-            team_object=team_object,
+    except Exception as e:  # noqa: BLE001  # an unresolved user grants nothing under deny-by-default
+        verbose_proxy_logger.debug("User lookup failed under search_tool_deny_by_default: %s", e)
+        return None
+
+
+async def resolve_search_tool_grants(
+    valid_token: UserAPIKeyAuth,
+    general_settings: Mapping[str, object],
+    load_team_object: TeamObjectLoader,
+) -> SearchToolGrants:
+    """
+    Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions,
+    the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names
+    must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing
+    """
+    if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
+        team_object: Final = await load_team_object()
+        return SearchToolGrants(
+            grants=MappingProxyType(
+                {"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None}
+            ),
+            strict_layers=frozenset(),
         )
+
+    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
+
+    strict_team_object: Final = await _strict_team_object(load_team_object)
+    strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object)
+    if prisma_client is None:
+        return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers)
+    user_object: Final = (
+        await _strict_user_object(valid_token, prisma_client, user_api_key_cache) if "user" in strict_layers else None
+    )
+    return SearchToolGrants(
+        grants=MappingProxyType(
+            dict(
+                await _identity_grants(valid_token, strict_team_object, user_object, prisma_client, user_api_key_cache)
+            )
+        ),
+        strict_layers=strict_layers,
+    )
+
+
+def check_search_tool_grants(search_tool_name: str, grants: SearchToolGrants) -> Literal[True]:
+    """Raises a 403 ProxyException naming the first key, team or user layer that does not grant the tool"""
+    for layer in ("key", "team", "user"):
+        grant = grants.grants.get(layer)
+        if layer in grants.strict_layers:
+            if grant is None or search_tool_name not in (grant.search_tools or ()):
+                raise _search_tool_denied(
+                    layer,
+                    f"{layer.capitalize()} not allowed to access search tool: {search_tool_name}. "
+                    f"search_tool_deny_by_default is enabled and the {layer} does not grant it",
+                )
+        elif grant is not None:
+            _can_object_call_search_tool(
+                search_tool_name=search_tool_name,
+                allowed_search_tools=_search_tool_names_from_object_permission(grant),
+                object_type=layer,
+            )
+    return True
+
+
+async def can_caller_call_search_tool(
+    search_tool_name: str,
+    valid_token: UserAPIKeyAuth,
+    general_settings: Mapping[str, object],
+    load_team_object: TeamObjectLoader,
+) -> Literal[True]:
+    """Key, team and user search tool authorization shared by /search, web search interception and discovery"""
+    return check_search_tool_grants(
+        search_tool_name, await resolve_search_tool_grants(valid_token, general_settings, load_team_object)
+    )
+
+
+async def can_token_call_search_tool(search_tool_name: str, valid_token: UserAPIKeyAuth) -> Literal[True]:
+    """`can_caller_call_search_tool` against the proxy's own settings, team cache and database"""
+    from litellm.proxy.proxy_server import general_settings, prisma_client, proxy_logging_obj, user_api_key_cache
+
+    async def _load_team_object() -> LiteLLM_TeamTable | None:
+        if not valid_token.team_id:
+            return None
+        return await get_team_object(
+            team_id=valid_token.team_id,
+            prisma_client=prisma_client,
+            user_api_key_cache=user_api_key_cache,
+            parent_otel_span=valid_token.parent_otel_span,
+            proxy_logging_obj=proxy_logging_obj,
+        )
+
+    return await can_caller_call_search_tool(
+        search_tool_name=search_tool_name,
+        valid_token=valid_token,
+        general_settings=typed_general_settings(general_settings),
+        load_team_object=_load_team_object,
+    )
+
+
+def can_grants_view_search_tool(search_tool_name: str, grants: SearchToolGrants) -> bool:
+    try:
+        check_search_tool_grants(search_tool_name, grants)
     except ProxyException:
         return False
     return True
 
 
+def check_unregistered_search_fallback(
+    valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]
+) -> Literal[True]:
+    """The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it"""
+    if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
+        return True
+    layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index)
+    raise _search_tool_denied(
+        layer,
+        "No registered search tool is available and search_tool_deny_by_default is enabled",
+    )
+
+
 async def is_valid_fallback_model(
     model: str,
     llm_router: Router | None,
@@ -6792,17 +6903,207 @@ def _get_rag_query_vector_store_id(request_body: Mapping[str, object]) -> str |
     return vector_store_id if isinstance(vector_store_id, str) and vector_store_id else None
 
 
+def _is_strict_grant_identity(valid_token: UserAPIKeyAuth | None) -> bool:
+    return (
+        valid_token is not None
+        and valid_token.api_key != LITELLM_PROXY_MASTER_KEY_ALIAS
+        and valid_token.team_id != UI_TEAM_ID
+    )
+
+
+def _strict_grant_layers(
+    deny_by_default: bool, valid_token: UserAPIKeyAuth | None, team_object: LiteLLM_TeamTable | None
+) -> frozenset[GrantLayer]:
+    """
+    The identities that must each grant an object under a deny-by-default policy: a virtual key and its team,
+    a keyless team member's team, or a keyless user's own grant. The master key and dashboard sessions need none
+    """
+    if not deny_by_default or valid_token is None or not _is_strict_grant_identity(valid_token):
+        return frozenset()
+    virtual_key: Final = valid_token.via_virtual_key and not valid_token.is_session_token
+    has_team: Final = team_object is not None or valid_token.team_id is not None
+    if not virtual_key and not has_team:
+        return frozenset(("user",))
+    return frozenset(layer for layer, required in (("key", virtual_key), ("team", has_team)) if required)
+
+
+async def _identity_grants(
+    valid_token: UserAPIKeyAuth | None,
+    team_object: LiteLLM_TeamTable | None,
+    user_object: LiteLLM_UserTable | None,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> tuple[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None], ...]:
+    return (
+        (
+            "key",
+            None
+            if valid_token is None
+            else await _cached_object_permission(
+                valid_token.object_permission_id, valid_token.object_permission, prisma_client, user_api_key_cache
+            ),
+        ),
+        (
+            "team",
+            None
+            if team_object is None
+            else await _cached_object_permission(
+                team_object.object_permission_id, team_object.object_permission, prisma_client, user_api_key_cache
+            ),
+        ),
+        (
+            "user",
+            None
+            if user_object is None
+            else await _cached_object_permission(
+                user_object.object_permission_id, user_object.object_permission, prisma_client, user_api_key_cache
+            ),
+        ),
+    )
+
+
+def _vector_store_deny_by_default(general_settings: Mapping[str, object]) -> bool:
+    """
+    Startup rejects a non-boolean value from the config file. A non-boolean value that reaches
+    general_settings another way enables the policy, so only vector store requests are denied.
+    """
+    try:
+        return ConfigGeneralSettings.model_validate(
+            MappingProxyType(
+                {"vector_store_deny_by_default": general_settings.get("vector_store_deny_by_default", False)}
+            )
+        ).vector_store_deny_by_default
+    except ValidationError:
+        return True
+
+
+_VECTOR_STORE_IDS_ADAPTER: Final[TypeAdapter[Sequence[str]]] = TypeAdapter(Sequence[str])
+_TOOLS_ADAPTER: Final[TypeAdapter[Sequence[object]]] = TypeAdapter(Sequence[object])
+_TOOL_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object])
+
+
+def _validated_vector_store_ids(value: object) -> tuple[str, ...]:
+    if value is None:
+        return ()
+    try:
+        return tuple(_VECTOR_STORE_IDS_ADAPTER.validate_python(value, strict=True))
+    except ValidationError:
+        raise _malformed_vector_store_ids() from None
+
+
+def _malformed_vector_store_ids() -> ProxyException:
+    return ProxyException(
+        message="vector_store_ids must be a list of strings",
+        type="invalid_request_error",
+        param="vector_store_ids",
+        code=status.HTTP_400_BAD_REQUEST,
+    )
+
+
+def _tools(tools: object) -> tuple[object, ...]:
+    try:
+        return tuple(_TOOLS_ADAPTER.validate_python(tools, strict=True))
+    except ValidationError:
+        return ()
+
+
+def _tool_vector_store_ids(tool: object) -> tuple[str, ...]:
+    try:
+        tool_fields: Final = _TOOL_ADAPTER.validate_python(tool, strict=True)
+    except ValidationError:
+        return ()
+    return _validated_vector_store_ids(tool_fields.get("vector_store_ids"))
+
+
+def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tuple[str, ...]:
+    """
+    Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, but a vector_store_ids that is
+    not a list of strings is a 400 instead of being skipped or iterated, and tools that are not
+    objects name no store.
+    """
+    return tuple(
+        chain(
+            _validated_vector_store_ids(request_body.get("vector_store_ids")),
+            chain.from_iterable(_tool_vector_store_ids(tool) for tool in _tools(request_body.get("tools"))),
+        )
+    )
+
+
+def _require_vector_store_grant(
+    object_type: GrantLayer,
+    vector_store_ids_to_run: Sequence[str],
+    object_permission: _VectorStorePermissionsRow | None,
+) -> None:
+    if object_permission is None or not object_permission.vector_stores:
+        raise ProxyException(
+            message=f"{object_type.capitalize()} not allowed to access vector store. Tried to access {vector_store_ids_to_run[0]}. vector_store_deny_by_default is enabled and the {object_type} has no vector store grants",
+            type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type),
+            param="vector_store",
+            code=status.HTTP_401_UNAUTHORIZED,
+        )
+    _can_object_call_vector_stores(
+        object_type=object_type,
+        vector_store_ids_to_run=vector_store_ids_to_run,
+        object_permissions=object_permission,
+    )
+
+
+async def _cached_object_permission(
+    object_permission_id: str | None,
+    loaded: LiteLLM_ObjectPermissionTable | None,
+    prisma_client: PrismaClient,
+    user_api_key_cache: UserApiKeyCache,
+) -> LiteLLM_ObjectPermissionTable | None:
+    """
+    The grant row auth already attached to the key, team or user, else the cached row by id. Both are
+    evicted on every worker when the grant changes, so the request path never reads the table directly.
+    """
+    if object_permission_id is None:
+        return None
+    if loaded is not None:
+        return loaded
+    return await get_object_permission(
+        object_permission_id=object_permission_id,
+        prisma_client=prisma_client,
+        user_api_key_cache=user_api_key_cache,
+    )
+
+
 async def vector_store_access_check(
     request_body: dict,
     team_object: LiteLLM_TeamTable | None,
     valid_token: UserAPIKeyAuth | None,
+    *,
+    user_object: LiteLLM_UserTable | None = None,
+    deny_by_default: bool = False,
 ):
     """
-    Checks if the object (key, team, org) has access to the vector store.
+    Checks whether the caller may use every vector store the request names.
 
-    Raises ProxyException if the object (key, team, org) cannot access the specific vector store.
+    Requested stores come from `vector_store_ids`, `tools[].vector_store_ids` and the RAG
+    `retrieval_config.vector_store_id`. Grants come from each identity's
+    `object_permission.vector_stores`.
+
+    With `deny_by_default=False` (legacy), a key or team only restricts access when its list is
+    nonempty, and stores are only read from the request when a vector store registry is loaded.
+
+    With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read
+    even without a registry, and a missing record, `null` or `[]` grants nothing:
+
+    - virtual key: the key must grant every store, and so must its team when it has one, even if
+      the team failed to load
+    - keyless team member (JWT, `lite login` session token): only the resolved team is checked
+    - keyless user with no team: the user's own grant is checked
+    - master key and dashboard sessions: legacy behavior
+
+    The user's personal grant is only consulted in the keyless no-team case, so it can neither
+    rescue nor restrict a key or team request.
+
+    Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity
+    that does not grant a requested store, and with the flag on, ProxyException (400,
+    `invalid_request_error`) when a `vector_store_ids` field is not a list of strings.
     """
-    from litellm.proxy.proxy_server import prisma_client
+    from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
 
     #########################################################
     # Get the vector store the user is trying to access
@@ -6812,12 +7113,17 @@ async def vector_store_access_check(
         return True
 
     registry_ids: Final = (
-        litellm.vector_store_registry.get_vector_store_ids_to_run(
-            non_default_params=request_body, tools=request_body.get("tools", None)
+        _strict_requested_vector_store_ids(_typed_request_body(request_body))
+        if deny_by_default
+        else (
+            litellm.vector_store_registry.get_vector_store_ids_to_run(
+                non_default_params=request_body, tools=request_body.get("tools", None)
+            )
+            if litellm.vector_store_registry is not None
+            else None
         )
-        if litellm.vector_store_registry is not None
-        else None
-    ) or ()
+        or ()
+    )
     rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body))
     rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else ()
     vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids)))
@@ -6828,38 +7134,28 @@ async def vector_store_access_check(
     #########################################################
     # Check if the object (key, team, org) has access to the vector store
     #########################################################
-    # Check if the key can access the vector store
-    if valid_token is not None and valid_token.object_permission_id is not None:
-        key_object_permission: Final = await _object_permission_table(
-            ObjectPermissionRepository(prisma_client)
-        ).find_unique(
-            where={"object_permission_id": valid_token.object_permission_id},
-        )
-        if key_object_permission is not None:
+    strict_layers: Final = _strict_grant_layers(deny_by_default, valid_token, team_object)
+    grants: Final = await _identity_grants(
+        valid_token,
+        team_object,
+        user_object if "user" in strict_layers else None,
+        prisma_client,
+        user_api_key_cache,
+    )
+    for layer, grant in grants:
+        if layer in strict_layers:
+            _require_vector_store_grant(layer, vector_store_ids_to_run, grant)
+        elif grant is not None:
             _can_object_call_vector_stores(
-                object_type="key",
+                object_type=layer,
                 vector_store_ids_to_run=vector_store_ids_to_run,
-                object_permissions=key_object_permission,
-            )
-
-    # Check if the team can access the vector store
-    if team_object is not None and team_object.object_permission_id is not None:
-        team_object_permission: Final = await _object_permission_table(
-            ObjectPermissionRepository(prisma_client)
-        ).find_unique(
-            where={"object_permission_id": team_object.object_permission_id},
-        )
-        if team_object_permission is not None:
-            _can_object_call_vector_stores(
-                object_type="team",
-                vector_store_ids_to_run=vector_store_ids_to_run,
-                object_permissions=team_object_permission,
+                object_permissions=grant,
             )
     return True
 
 
 def _can_object_call_vector_stores(
-    object_type: Literal["key", "team", "org"],
+    object_type: Literal["key", "team", "org", "user"],
     vector_store_ids_to_run: Sequence[str],
     object_permissions: _VectorStorePermissionsRow | None,
 ):

litellm/proxy/auth/fallback_model_access.py

@@ -5,7 +5,8 @@ Authorize router fallback targets against the caller's key, team and project mod
 Fallbacks configured on the router (`router_settings.fallbacks` and friends) are chosen after auth,
 inside the router, so this predicate is injected into the router to re-run the same model access
 checks for each fallback target before it is attempted. Opt-in via
-`general_settings.enforce_fallback_model_access: true`.
+`general_settings.enforce_fallback_model_access: true`. Fallbacks of a search request always run the
+key, team and user search tool grants instead, the same check /search runs on the requested tool.
 """
 
 from collections.abc import Callable, Mapping
@@ -16,8 +17,9 @@ from pydantic import ValidationError
 
 from litellm._logging import verbose_proxy_logger
 from litellm.proxy._types import ProxyException, UserAPIKeyAuth
-from litellm.proxy.auth.auth_checks import can_key_call_resolved_model
+from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool
 from litellm.router import Router
+from litellm.search import asearch
 from litellm.types.llms.base import LiteLLMBaseModel
 
 
@@ -45,6 +47,23 @@ async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAu
     return True
 
 
+async def is_search_tool_authorized_for_token(*, search_tool_name: str, valid_token: UserAPIKeyAuth) -> bool:
+    try:
+        await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=valid_token)
+    except ProxyException:
+        return False
+    except Exception as e:  # noqa: BLE001  # fail closed: a lookup failure must neither run the fallback nor replace the provider error
+        verbose_proxy_logger.warning(
+            "Skipping fallback to search tool=%s: authorization lookup failed: %s", search_tool_name, e
+        )
+        return False
+    return True
+
+
+def _is_search_request(request_kwargs: Mapping[str, object]) -> bool:
+    return request_kwargs.get("original_generic_function") is asearch
+
+
 def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None:
     try:
         return _RequestMetadata.model_validate(metadata).user_api_key_auth
@@ -72,19 +91,21 @@ def _enforced_by_general_settings() -> bool:
 @dataclass(frozen=True, slots=True)
 class RouterFallbackAccessCheck:
     """
-    `FallbackAccessCheck` for the proxy's router: while `is_enforced()` is true, a fallback target
-    is attempted only when the key behind the request could have requested it directly. Requests
-    that carry no key (for example internal health checks) are not restricted.
+    `FallbackAccessCheck` for the proxy's router: a fallback of a search request, and while `is_enforced()`
+    is true any other fallback, is attempted only when the key behind the request could have requested it
+    directly. Requests that carry no key (for example internal health checks) are not restricted.
     """
 
     is_enforced: Callable[[], bool]
 
     async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool:
-        if not self.is_enforced():
-            return True
         valid_token: Final = _user_api_key_auth_from_request(request_kwargs)
         if valid_token is None:
             return True
+        if _is_search_request(request_kwargs):
+            return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token)
+        if not self.is_enforced():
+            return True
         return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router)
 
 

litellm/proxy/auth/user_api_key_auth.py

@@ -3063,6 +3063,9 @@ async def _run_centralized_common_checks(
             user_id=user_api_key_auth_obj.user_id or litellm_proxy_admin_name,
             user_role=LitellmUserRoles.PROXY_ADMIN,
             spend=user_object.spend if user_object is not None else 0.0,
+            object_permission_id=(
+                user_object.object_permission_id if isinstance(user_object, LiteLLM_UserTable) else None
+            ),
         )
 
     if project_object is not None:

litellm/proxy/common_utils/callback_config_validation.py

@@ -15,15 +15,20 @@ _NEWRELIC_VAR_PREFIX: Final = "newrelic_"
 _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel"
 _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope"
 _ARIZE_CALLBACK: Final = "arize"
+_ARIZE_OTLP_PROTOCOL_VAR: Final = "arize_otlp_protocol"
 _ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset(
     {"arize_success_sampling_rate", "arize_error_sampling_rate"}
 )
 
 
-def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None:
+def callback_config_error(
+    callback_name: str | None, callback_vars: Mapping[str, str] | None, callback_type: str | None = None
+) -> str | None:
     if not callback_vars:
         return None
-    arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars)
+    arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) or _arize_otlp_protocol_error(
+        callback_name, callback_vars, callback_type
+    )
     if arize_error is not None:
         return arize_error
     langfuse_error: Final = _langfuse_environment_error(callback_vars) or _langfuse_span_scope_error(
@@ -94,7 +99,9 @@ _VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType(
     }
 )
 
-_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR, *_ARIZE_SAMPLING_RATE_VARS})
+_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset(
+    {_LANGFUSE_SPAN_SCOPE_VAR, _ARIZE_OTLP_PROTOCOL_VAR, *_ARIZE_SAMPLING_RATE_VARS}
+)
 
 
 def _family_of(var: str) -> str | None:
@@ -217,7 +224,10 @@ def _logging_entry_error(entry: object) -> str | None:
     callback_name: Final = entry.get("callback_name")
     if not isinstance(callback_name, str) or not isinstance(entry.get("callback_vars"), Mapping):
         return None
-    return callback_config_error(callback_name, _entry_callback_vars(entry))
+    callback_type_raw: Final = entry.get("callback_type")
+    return callback_config_error(
+        callback_name, _entry_callback_vars(entry), callback_type_raw if isinstance(callback_type_raw, str) else None
+    )
 
 
 def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None:
@@ -236,6 +246,30 @@ def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping
     return None
 
 
+def _arize_otlp_protocol_error(
+    callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None = None
+) -> str | None:
+    value: Final = callback_vars.get(_ARIZE_OTLP_PROTOCOL_VAR)
+    if value is None:
+        return None
+    if callback_name != _ARIZE_CALLBACK:
+        return f"{_ARIZE_OTLP_PROTOCOL_VAR} applies to the {_ARIZE_CALLBACK} callback only, not {callback_name!r}"
+    from litellm.integrations.otel.model.config import is_otel_v2_enabled
+    from litellm.litellm_core_utils.initialize_dynamic_callback_params import (
+        validate_arize_otlp_protocol_value,
+    )
+
+    try:
+        validate_arize_otlp_protocol_value(value)
+    except ValueError as e:
+        return str(e)
+    if callback_type == "failure":
+        return f"{_ARIZE_OTLP_PROTOCOL_VAR} needs callback_type 'success' or 'success_and_failure'; failure-only Arize callbacks export over the proxy's own Arize transport"
+    if not is_otel_v2_enabled():
+        return "Per-team Arize transport selection requires the proxy to run with LITELLM_OTEL_V2=true."
+    return None
+
+
 def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None:
     """Per-team New Relic routing runs on the OTel v2 path only.
 

litellm/proxy/management_endpoints/internal_user_endpoints.py

@@ -1458,11 +1458,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis
         *(object_permission_cache_key(permission_id) for permission_id in dict.fromkeys(object_permission_ids)),
         *((user_object_permission_id_cache_key(user_id), user_id) if user_id is not None else ()),
     )
-    for key in keys:
-        try:
-            await user_api_key_cache.async_delete_cache(key=key)
-        except Exception as e:  # noqa: BLE001  # a cache we cannot clear still expires; never fail the write
-            verbose_proxy_logger.warning("Failed to invalidate cached entitlement key %r: %s", key, e)
+    await evict_and_broadcast(cache_keys=keys, user_api_key_cache=user_api_key_cache)
 
 
 async def _update_single_user_helper(

litellm/proxy/management_endpoints/team_callback_endpoints.py

@@ -61,7 +61,7 @@ def _callback_config_error(message: str) -> HTTPException:
 
 
 def _validate_team_callback(data: "AddTeamCallback") -> None:
-    error: Final = callback_config_error(data.callback_name, data.callback_vars)
+    error: Final = callback_config_error(data.callback_name, data.callback_vars, data.callback_type)
     if error is not None:
         raise _callback_config_error(error)
 

litellm/proxy/management_endpoints/team_endpoints.py

@@ -165,6 +165,7 @@ from litellm.proxy.management_helpers.object_permission_utils import (
     _set_object_permission,
     enforce_all_proxy_mcp_servers_grant_is_admin_only,
     handle_update_object_permission_common,
+    invalidate_cached_object_permissions,
 )
 from litellm.proxy.management_helpers.team_member_permission_checks import (
     TeamMemberPermissionChecks,
@@ -2546,6 +2547,10 @@ async def update_team(
 
         verbose_proxy_logger.info("Successfully updated team - %s, info", team_row.team_id)
         await sync_team_access_group_membership(prisma_client=prisma_client, team_id=team_row.team_id)
+        await invalidate_cached_object_permissions(
+            object_permission_ids=(existing_team.object_permission_id, team_row.object_permission_id),
+            user_api_key_cache=user_api_key_cache,
+        )
         await _refresh_cached_team(
             team_row=team_row,
             user_api_key_cache=user_api_key_cache,

litellm/proxy/proxy_server.py

@@ -6713,6 +6713,17 @@ class ProxyConfig:
         if general_settings is None:
             general_settings = {}
 
+        typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings)
+        ConfigGeneralSettings.model_validate(
+            MappingProxyType(
+                {
+                    name: typed_general_settings[name]
+                    for name in ("vector_store_deny_by_default", "search_tool_deny_by_default")
+                    if name in typed_general_settings
+                }
+            )
+        )
+
         if general_settings.get("mcp_advertised_versions") is not None:
             from litellm.types.mcp import MCPAdvertisedVersions
 

litellm/proxy/search_endpoints/endpoints.py

@@ -135,52 +135,24 @@ async def search(
     if search_tool_name is not None:
         data["search_tool_name"] = search_tool_name
 
-    if not (
-        data.get("search_tool_name") or data.get("model") or general_settings.get("completion_model") or user_model
-    ):
+    from litellm.proxy.auth.auth_checks import can_token_call_search_tool
+    from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model
+
+    routed_search_tool_name: Final = data.get("search_tool_name") or resolve_inference_model(
+        data.get("model"), general_settings, user_model
+    )
+    if not isinstance(routed_search_tool_name, str) or not routed_search_tool_name:
         raise ProxyMissingRequiredParamError(route="/search", param="search_tool_name")
+    try:
+        await can_token_call_search_tool(search_tool_name=routed_search_tool_name, valid_token=user_api_key_dict)
+    except ProxyException as e:
+        verbose_proxy_logger.debug("Search tool authorization denied: %s", e.type)
+        raise
 
     if "search_tool_name" in data and data["search_tool_name"]:
         data["model"] = data["search_tool_name"]
         search_tool_name_value: Final = data["search_tool_name"]
 
-        # Authorization check: verify key can access this search tool
-        from litellm.proxy.auth.auth_checks import (
-            can_key_call_search_tool,
-            can_team_call_search_tool,
-            get_team_object,
-        )
-
-        try:
-            # Check key-level access
-            await can_key_call_search_tool(
-                search_tool_name=search_tool_name_value,
-                valid_token=user_api_key_dict,
-            )
-
-            # Check team-level access if key is associated with a team
-            if user_api_key_dict.team_id:
-                from litellm.proxy.proxy_server import (
-                    prisma_client,
-                    proxy_logging_obj,
-                    user_api_key_cache,
-                )
-
-                team_object: Final = await get_team_object(
-                    team_id=user_api_key_dict.team_id,
-                    prisma_client=prisma_client,
-                    user_api_key_cache=user_api_key_cache,
-                    parent_otel_span=user_api_key_dict.parent_otel_span,
-                    proxy_logging_obj=proxy_logging_obj,
-                )
-                await can_team_call_search_tool(
-                    search_tool_name=search_tool_name_value,
-                    team_object=team_object,
-                )
-        except Exception as e:
-            verbose_proxy_logger.error("Search tool authorization failed for %s: %s", search_tool_name_value, e)
-            raise
-
         if llm_router is not None and hasattr(llm_router, "search_tools"):
             verbose_proxy_logger.debug(
                 "Search endpoint - Looking for search_tool_name: %s. Available search tools in router: %s. Total search tools: %s",

litellm/proxy/search_endpoints/search_tool_management.py

@@ -2,7 +2,7 @@
 CRUD ENDPOINTS FOR SEARCH TOOLS
 """
 
-from collections.abc import Awaitable, Callable, Sequence
+from collections.abc import Awaitable, Callable, Mapping, Sequence
 from datetime import datetime
 from typing import Any, Final, TypeAlias
 
@@ -117,34 +117,39 @@ async def _filter_visible_search_tools(
     search_tools: list[SearchToolInfoResponse],
     user_api_key_dict: UserAPIKeyAuth,
     lookup_team_object: TeamObjectLookup = _team_object_from_db,
+    general_settings: Mapping[str, object] | None = None,
 ) -> list[SearchToolInfoResponse]:
     """
     Drop search tools the caller is not authorized to invoke, applying the same
-    key/team object_permission allowlists enforced on /search. Admins see all tools.
+    key/team/user grants enforced on /search. Admins see all tools unless
+    search_tool_deny_by_default applies to their credential.
     """
+    from litellm.proxy.auth.auth_checks import (
+        can_grants_view_search_tool,
+        is_search_tool_deny_by_default_applied,
+        resolve_search_tool_grants,
+        typed_general_settings,
+    )
+    from litellm.proxy.proxy_server import general_settings as proxy_general_settings
+
+    settings: Final = typed_general_settings(proxy_general_settings) if general_settings is None else general_settings
     if user_api_key_dict.user_role in (
         LitellmUserRoles.PROXY_ADMIN,
         LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
-    ):
+    ) and not is_search_tool_deny_by_default_applied(user_api_key_dict, settings):
         return search_tools
 
-    from litellm.proxy.auth.auth_checks import can_user_view_search_tool
-
     allowlist_team_id: Final = _allowlist_team_id(user_api_key_dict)
-    team_object: Final[LiteLLM_TeamTable | None] = (
-        await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None
-    )
 
-    visible: Final[list[SearchToolInfoResponse]] = []
-    for tool in search_tools:
-        tool_name = tool.get("search_tool_name")
-        if tool_name and await can_user_view_search_tool(
-            search_tool_name=tool_name,
-            valid_token=user_api_key_dict,
-            team_object=team_object,
-        ):
-            visible.append(tool)
-    return visible
+    async def _load_team_object() -> LiteLLM_TeamTable | None:
+        return await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None
+
+    grants: Final = await resolve_search_tool_grants(user_api_key_dict, settings, _load_team_object)
+    return [
+        tool
+        for tool in search_tools
+        if (tool_name := tool.get("search_tool_name")) and can_grants_view_search_tool(tool_name, grants)
+    ]
 
 
 @router.get(

litellm/responses/streaming_iterator.py

@@ -691,8 +691,10 @@ class BaseResponsesAPIStreamingIterator:
         if getattr(completed_response, "type", None) != openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED:
             return
 
+        from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output
+
         response_obj: Final = self._get_completed_response_object()
-        if response_obj is None:
+        if response_obj is None or is_response_without_output(response_obj):
             return
 
         caching_handler: Final[LLMCachingHandler | None] = getattr(self.logging_obj, "_llm_caching_handler", None)
@@ -732,8 +734,6 @@ class BaseResponsesAPIStreamingIterator:
         if cached_response is None:
             return
         if is_async:
-            from litellm.caching.caching_handler import create_cache_write_task
-
             cache_write_task: Final = create_cache_write_task(
                 lambda: cache.async_add_cache(
                     cached_response,

litellm/rust_bridge/callbacks_legacy_python.py

@@ -21,6 +21,8 @@ from typing import (
     cast,  # noqa: TID251  # bounded compatibility calls into legacy Python integrations
 )
 
+from pydantic import TypeAdapter
+
 if TYPE_CHECKING:
     from litellm.litellm_core_utils.litellm_logging import Logging
 
@@ -337,9 +339,15 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t
     return hook(kwargs, error, call_type)
 
 
-def stream_opened(logger: Logging) -> None:
+_STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str])
+
+
+def stream_opened(logger: Logging, head: Mapping[str, object]) -> None:
     logger.stream = True
     logger.model_call_details["stream"] = True
+    logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python(
+        head.get("additional_headers") or {}
+    )
 
 
 def stream_success(

litellm/rust_bridge/trace/generated/types.py

@@ -59,6 +59,7 @@ class AgentNode(typing_extensions.TypedDict):
     tool_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]]
     duration_ms: ReadOnly[float]
     spend: ReadOnly[float | None]
+    priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]]
 
 
 SpanType: TypeAlias = Literal[
@@ -77,6 +78,9 @@ SpanType: TypeAlias = Literal[
 ]
 
 
+SpendMatch: TypeAlias = Literal["matched", "no_call_id", "no_spend_log", "ambiguous", "incomplete_evidence"]
+
+
 class TraceScope(typing_extensions.TypedDict):
     all_teams: ReadOnly[Literal[0, 1]]
     user_id: ReadOnly[str]
@@ -120,6 +124,7 @@ class TraceSummary(typing_extensions.TypedDict):
     output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]]
     models: ReadOnly[tuple[str, ...]]
     spend: ReadOnly[float | None]
+    priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]]
 
 
 class Span(typing_extensions.TypedDict):
@@ -140,6 +145,8 @@ class Span(typing_extensions.TypedDict):
     output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=4294967295)]]
     litellm_request_id: ReadOnly[str | None]
     spend: ReadOnly[float | None]
+    spend_log_request_id: ReadOnly[str | None]
+    spend_match: ReadOnly[SpendMatch | None | None]
 
 
 class Trace(typing_extensions.TypedDict):

litellm/tracing/types.py

@@ -8,6 +8,7 @@ class SpendLogRecord(TypedDict):
 
     request_id: ReadOnly[str]
     response_id: ReadOnly[str]
+    provider_request_id: ReadOnly[str]
     litellm_call_id: ReadOnly[str]
     call_type: ReadOnly[str]
     api_key: ReadOnly[str]

litellm/types/caching.py

@@ -140,6 +140,8 @@ class HealthCheckCacheParams(LiteLLMBaseModel):
 
 EMBEDDING_CACHE_FORMAT_VERSION: Final = 2
 
+CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events"
+
 
 class CachedEmbedding(TypedDict):
     """Type definition for cached embedding objects"""

litellm/types/utils.py

@@ -3648,6 +3648,9 @@ OPENAI_RESPONSE_HEADERS: Final = [
 OtelSpanScope = Literal["full", "llm_only"]
 OTEL_SPAN_SCOPES: Final[frozenset[str]] = frozenset(get_args(OtelSpanScope))
 
+ArizeOtlpProtocol = Literal["grpc", "http/protobuf"]
+ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol))
+
 
 class StandardCallbackDynamicParams(TypedDict, total=False):
     # Langfuse dynamic params
@@ -3681,6 +3684,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False):
     arize_space_id: str | None
     arize_success_sampling_rate: ReadOnly[float | None]
     arize_error_sampling_rate: ReadOnly[float | None]
+    arize_otlp_protocol: ReadOnly[ArizeOtlpProtocol | None]
 
     # PostHog dynamic params
     posthog_api_key: str | None

scripts/trace_codegen/schemas/traces/Trace.json

@@ -28,6 +28,12 @@
             "null"
           ]
         },
+        "priced_calls": {
+          "format": "uint64",
+          "maximum": 18446744073709551615,
+          "minimum": 0,
+          "type": "integer"
+        },
         "spend": {
           "format": "double",
           "type": [
@@ -49,7 +55,8 @@
         "llm_calls",
         "tool_calls",
         "duration_ms",
-        "spend"
+        "spend",
+        "priced_calls"
       ],
       "type": "object"
     },
@@ -120,6 +127,22 @@
             "null"
           ]
         },
+        "spend_log_request_id": {
+          "type": [
+            "string",
+            "null"
+          ]
+        },
+        "spend_match": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/SpendMatch"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
         "start_offset_ms": {
           "format": "double",
           "type": "number"
@@ -148,7 +171,9 @@
         "input_tokens",
         "output_tokens",
         "litellm_request_id",
-        "spend"
+        "spend",
+        "spend_log_request_id",
+        "spend_match"
       ],
       "type": "object"
     },
@@ -177,6 +202,16 @@
       ],
       "type": "string"
     },
+    "SpendMatch": {
+      "enum": [
+        "matched",
+        "no_call_id",
+        "no_spend_log",
+        "ambiguous",
+        "incomplete_evidence"
+      ],
+      "type": "string"
+    },
     "TraceSummary": {
       "properties": {
         "agent_count": {
@@ -245,6 +280,12 @@
           "minimum": 0,
           "type": "integer"
         },
+        "priced_calls": {
+          "format": "uint64",
+          "maximum": 18446744073709551615,
+          "minimum": 0,
+          "type": "integer"
+        },
         "resolution_limited": {
           "type": "boolean",
           "x-python-optional": true
@@ -306,7 +347,8 @@
         "input_tokens",
         "output_tokens",
         "models",
-        "spend"
+        "spend",
+        "priced_calls"
       ],
       "type": "object"
     }

scripts/trace_codegen/schemas/traces/TracePage.json

@@ -76,6 +76,12 @@
           "minimum": 0,
           "type": "integer"
         },
+        "priced_calls": {
+          "format": "uint64",
+          "maximum": 18446744073709551615,
+          "minimum": 0,
+          "type": "integer"
+        },
         "resolution_limited": {
           "type": "boolean",
           "x-python-optional": true
@@ -137,7 +143,8 @@
         "input_tokens",
         "output_tokens",
         "models",
-        "spend"
+        "spend",
+        "priced_calls"
       ],
       "type": "object"
     }

tests/integration/_support/manifest.py

@@ -18,6 +18,7 @@ OWNED_DIRECTORIES: Final = frozenset(
         "compatibility",
         "sdk",
         "cost_calculation",
+        "caching",
         "security",
     }
 )

tests/integration/authorization/test_object_permission_lookup.py

@@ -57,18 +57,18 @@ def _assert_forbidden_vector_store_denied(gateway: Gateway, model: str, key: str
 
 
 @pytest.mark.covers("authorization.vector_store.plain_request_skips_object_permission_lookup")
-def test_chat_request_without_vector_stores_does_not_read_object_permission_table(gateway: Gateway) -> None:
+def test_requests_do_not_read_object_permission_table_once_the_grant_is_cached(gateway: Gateway) -> None:
     with gateway.scenario() as scenario:
         model: Final = scenario.model()
         key: Final = scenario.key(models=[model], object_permission={"vector_stores": ["vs_allowed"]})
-        _assert_plain_chat_served(gateway, model, key)
-        before_control: Final = _object_permission_reads()
+        before_first_use: Final = _object_permission_reads()
         _assert_forbidden_vector_store_denied(gateway, model, key)
-        eventually(_object_permission_reads, lambda reads: reads > before_control, seconds=15)
+        eventually(_object_permission_reads, lambda reads: reads > before_first_use, seconds=15)
         baseline: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic())
         for _ in range(PLAIN_REQUESTS):
             _assert_plain_chat_served(gateway, model, key)
+            _assert_forbidden_vector_store_denied(gateway, model, key)
         after: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic())
         assert after - baseline < PLAIN_REQUESTS, (
-            f"{PLAIN_REQUESTS} plain chat requests added {after - baseline} object permission reads"
+            f"{2 * PLAIN_REQUESTS} requests after the first added {after - baseline} object permission reads"
         )

tests/integration/authorization/test_rag_query_vector_store_allowlist.py

@@ -1,5 +1,8 @@
 from __future__ import annotations
 
+import json
+import os
+import time
 import uuid
 from collections.abc import Iterator, Mapping
 from pathlib import Path
@@ -7,16 +10,22 @@ from types import MappingProxyType
 from typing import Final, Literal, TypeAlias
 
 import httpx
+import jwt
 import pytest
 import yaml
-from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value
+from cryptography.hazmat.primitives.asymmetric import rsa
+from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value
 from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, wire_server
 from integration.authorization._guardrail_opt_out import upstream_observations
 from pydantic import JsonValue
+from redis import Redis
 
 CONFIG_STORE_ID: Final = "vs_integration_config_store"
 PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
 REMOVE_OPENAI_API_BASE: Final = ("OPENAI_API_BASE",)
+JWT_KEY_ID: Final = "integration-vector-store-jwt-key"
+AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation"
 JsonObject: TypeAlias = dict[str, JsonValue]
 
 
@@ -99,6 +108,211 @@ def no_registry_gateways(tmp_path_factory: pytest.TempPathFactory) -> Iterator[t
             yield no_registry_gateway, upstream_gateway
 
 
+class StrictGateway:
+    def __init__(
+        self,
+        gateway: Gateway,
+        upstream: Gateway,
+        signing_key: rsa.RSAPrivateKey,
+        config: Path,
+        environment: Mapping[str, str],
+    ) -> None:
+        self.gateway: Final = gateway
+        self.upstream: Final = upstream
+        self._signing_key: Final = signing_key
+        self.config: Final = config
+        self.environment: Final = environment
+
+    def jwt(self, subject: str, groups: tuple[str, ...] = ()) -> str:
+        claims: Final[JsonObject] = {
+            "sub": subject,
+            "groups": _json_array(*groups),
+            "iat": int(time.time()),
+            "exp": int(time.time()) + 300,
+        }
+        return jwt.encode(claims, self._signing_key, algorithm="RS256", headers={"kid": JWT_KEY_ID})
+
+
+def _strict_config(directory: Path, *, deny_by_default: bool = True) -> Path:
+    config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text()))
+    general_settings: Final = object_value(config["general_settings"])
+    strict: Final[JsonObject] = {
+        **config,
+        "general_settings": {
+            **general_settings,
+            "vector_store_deny_by_default": deny_by_default,
+            "enable_jwt_auth": True,
+            "litellm_jwtauth": {
+                "user_id_jwt_field": "sub",
+                "team_ids_jwt_field": "groups",
+                "team_allowed_routes": _json_array("openai_routes", "info_routes", "/v1/rag/query"),
+            },
+        },
+    }
+    path: Final = directory / f"proxy_vector_store_deny_by_default_{deny_by_default}.yaml"
+    path.write_text(yaml.safe_dump(strict))
+    return path
+
+
+@pytest.fixture(scope="module")
+def strict_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[StrictGateway]:
+    signing_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048)
+    public_jwk: Final = jwt.algorithms.RSAAlgorithm.to_jwk(signing_key.public_key())
+    jwks_body: Final = json.dumps({"keys": [{**json.loads(public_jwk), "kid": JWT_KEY_ID}]}).encode()
+
+    def respond(request: Request) -> Reply:
+        assert request.method == "GET", request
+        return Reply(body=jwks_body)
+
+    with gateway_from_environment() as upstream_gateway, wire_server(respond) as jwks:
+        directory: Final = tmp_path_factory.mktemp("rag_query_deny_by_default")
+        config: Final = _strict_config(directory)
+        environment: Final = MappingProxyType({**_openai_environment(upstream_gateway), "JWT_PUBLIC_KEY_URL": jwks.url})
+        with owned_proxy(
+            upstream_gateway,
+            directory,
+            environment,
+            config=config,
+            remove_environment=REMOVE_OPENAI_API_BASE,
+        ) as gateway:
+            yield StrictGateway(gateway, upstream_gateway, signing_key, config, environment)
+
+
+StrictCase: TypeAlias = Literal[
+    "standalone_key_no_permission",
+    "team_key_empty_key_grants",
+    "team_key_empty_team_grants",
+    "multi_store_one_ungranted",
+    "jwt_user_without_grant",
+]
+
+
+def _strict_denied_request(
+    strict: StrictGateway, scenario: Scenario, case: StrictCase, model: str, marker: str, store_id: str
+) -> tuple[httpx.Response, str]:
+    models: Final = _json_array(model)
+    granted: Final = _permission_for_stores(store_id)
+    empty: Final = _permission_for_stores()
+    if case == "standalone_key_no_permission":
+        standalone_key: Final = scenario.key(models=models)
+        return strict.gateway.request(
+            "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=standalone_key
+        ), ("key_vector_store_access_denied")
+    if case in ("team_key_empty_key_grants", "team_key_empty_team_grants"):
+        key_grants_store: Final = case == "team_key_empty_team_grants"
+        team: Final = scenario.team(models=models, object_permission=empty if key_grants_store else granted)
+        team_key: Final = scenario.key(
+            team_id=team, models=models, object_permission=granted if key_grants_store else empty
+        )
+        error_type: Final = "team_vector_store_access_denied" if key_grants_store else "key_vector_store_access_denied"
+        return strict.gateway.request(
+            "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=team_key
+        ), (error_type)
+    if case == "multi_store_one_ungranted":
+        partial_key: Final = scenario.key(models=models, object_permission=_permission_for_stores(store_id))
+        body: Final[JsonObject] = {
+            **_rag_query_body(model, marker, store_id),
+            "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(CONFIG_STORE_ID)}),
+        }
+        return strict.gateway.request(
+            "POST", "/v1/chat/completions", body, key=partial_key
+        ), "key_vector_store_access_denied"
+    user: Final = scenario.user(user_role="internal_user")
+    return (
+        strict.gateway.request("POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=strict.jwt(user)),
+        "user_vector_store_access_denied",
+    )
+
+
+@pytest.mark.parametrize(
+    "case",
+    (
+        "standalone_key_no_permission",
+        "team_key_empty_key_grants",
+        "team_key_empty_team_grants",
+        "multi_store_one_ungranted",
+        "jwt_user_without_grant",
+    ),
+)
+def test_deny_by_default_rejects_ungranted_store_before_upstream_search(
+    strict_gateway: StrictGateway, case: StrictCase
+) -> None:
+    with strict_gateway.gateway.scenario() as scenario:
+        model: Final = scenario.model()
+        store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}"
+        marker: Final = f"lit6035 deny by default {case} {uuid.uuid4().hex}"
+
+        response, error_type = _strict_denied_request(strict_gateway, scenario, case, model, marker, store_id)
+        observations: Final = tuple(
+            observation
+            for observation in upstream_observations(strict_gateway.upstream)
+            if marker in str(observation["body"])
+        )
+
+        assert response.status_code == 401, f"{response.text}; scripted_upstream_observations={observations!r}"
+        assert response.json()["error"]["type"] == error_type, response.text
+        assert observations == ()
+
+
+GrantedCase: TypeAlias = Literal[
+    "standalone_key_granted_registered_store",
+    "team_key_both_grant_unregistered_store",
+    "jwt_team_member_team_grant_only",
+    "jwt_user_personal_grant",
+    "master_key_without_grants",
+]
+
+
+def _strict_granted_request(
+    strict: StrictGateway, scenario: Scenario, case: GrantedCase, model: str, marker: str, store_id: str
+) -> httpx.Response:
+    models: Final = _json_array(model)
+    granted: Final = _permission_for_stores(store_id)
+    body: Final = _rag_query_body(model, marker, store_id)
+    if case in ("standalone_key_granted_registered_store", "team_key_both_grant_unregistered_store"):
+        key_team: Final = scenario.team(models=models, object_permission=granted) if case.startswith("team") else None
+        granted_key: Final = scenario.key(
+            models=models, object_permission=granted, **({} if key_team is None else {"team_id": key_team})
+        )
+        return strict.gateway.request("POST", "/v1/rag/query", body, key=granted_key)
+    if case == "jwt_team_member_team_grant_only":
+        member_team: Final = scenario.team(models=models, object_permission=granted)
+        member: Final = scenario.member(member_team)
+        return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(member, (member_team,)))
+    if case == "jwt_user_personal_grant":
+        user: Final = scenario.user(user_role="internal_user", object_permission=granted)
+        return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(user))
+    return strict.gateway.request("POST", "/v1/rag/query", body)
+
+
+@pytest.mark.parametrize(
+    "case",
+    (
+        "standalone_key_granted_registered_store",
+        "team_key_both_grant_unregistered_store",
+        "jwt_team_member_team_grant_only",
+        "jwt_user_personal_grant",
+        "master_key_without_grants",
+    ),
+)
+def test_deny_by_default_searches_explicitly_granted_store(strict_gateway: StrictGateway, case: GrantedCase) -> None:
+    with strict_gateway.gateway.scenario() as scenario:
+        model: Final = scenario.model()
+        store_id: Final = (
+            CONFIG_STORE_ID
+            if case == "standalone_key_granted_registered_store"
+            else f"vs_unregistered_{uuid.uuid4().hex}"
+        )
+        marker: Final = f"lit6035 granted {case} {uuid.uuid4().hex}"
+
+        response: Final = _strict_granted_request(strict_gateway, scenario, case, model, marker, store_id)
+        assert response.status_code == 200, response.text
+
+        searches: Final = _searches_for_marker(strict_gateway.upstream, marker, store_id)
+        assert len(searches) == 1, searches
+        assert marker in str(object_value(searches[0]["body"])["query"]), searches
+
+
 @pytest.mark.parametrize(
     ("scope", "error_type"),
     (("key", "key_vector_store_access_denied"), ("team", "team_vector_store_access_denied")),
@@ -220,3 +434,136 @@ def test_rag_query_alias_denies_store_when_team_allowlist_excludes(gateway: Gate
         assert response.status_code == 401, response.text
         assert response.json()["error"]["type"] == "team_vector_store_access_denied", response.text
         assert _searches_for_marker(gateway, marker) == ()
+
+
+def test_explicit_false_flag_keeps_legacy_vector_store_outcomes(tmp_path: Path) -> None:
+    with gateway_from_environment() as upstream_gateway:
+        with owned_proxy(
+            upstream_gateway,
+            tmp_path,
+            _openai_environment(upstream_gateway),
+            config=_strict_config(tmp_path, deny_by_default=False),
+            remove_environment=REMOVE_OPENAI_API_BASE,
+        ) as gateway:
+            with gateway.scenario() as scenario:
+                model: Final = scenario.model()
+                allowed_marker: Final = f"flag-false-allowed-{uuid.uuid4().hex}"
+                denied_marker: Final = f"flag-false-denied-{uuid.uuid4().hex}"
+                no_permission_key: Final = scenario.key(models=_json_array(model))
+                excluding_key: Final = scenario.key(
+                    models=_json_array(model), object_permission=_permission_for_stores("vs_some_other_store")
+                )
+
+                allowed: Final = _rag_query(gateway, model, allowed_marker, no_permission_key)
+                denied: Final = _rag_query(gateway, model, denied_marker, excluding_key)
+
+            assert allowed.status_code == 200, allowed.text
+            assert len(_searches_for_marker(upstream_gateway, allowed_marker)) == 1
+            assert denied.status_code == 401, denied.text
+            assert denied.json()["error"]["type"] == "key_vector_store_access_denied"
+            assert _searches_for_marker(upstream_gateway, denied_marker) == ()
+
+
+def _strict_no_registry_config(directory: Path) -> Path:
+    config: Final = object_value(yaml.safe_load(_no_registry_config(directory).read_text()))
+    general_settings: Final = object_value(config["general_settings"])
+    strict: Final[JsonObject] = {
+        **config,
+        "general_settings": {**general_settings, "vector_store_deny_by_default": True},
+    }
+    path: Final = directory / "proxy_vector_store_deny_by_default_no_registry.yaml"
+    path.write_text(yaml.safe_dump(strict))
+    return path
+
+
+def test_deny_by_default_without_registry_checks_search_route_and_file_search_tools(tmp_path: Path) -> None:
+    with gateway_from_environment() as upstream_gateway:
+        with owned_proxy(
+            upstream_gateway,
+            tmp_path,
+            _openai_environment(upstream_gateway),
+            config=_strict_no_registry_config(tmp_path),
+            remove_environment=REMOVE_OPENAI_API_BASE,
+        ) as gateway:
+            with gateway.scenario() as scenario:
+                model: Final = scenario.model()
+                store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}"
+                search_marker: Final = f"lit6035 no registry search {uuid.uuid4().hex}"
+                responses_marker: Final = f"lit6035 no registry responses {uuid.uuid4().hex}"
+                granted_marker: Final = f"lit6035 no registry granted search {uuid.uuid4().hex}"
+                ungranted_key: Final = scenario.key(models=_json_array(model))
+                granted_key: Final = scenario.key(
+                    models=_json_array(model), object_permission=_permission_for_stores(store_id)
+                )
+
+                search_denied: Final = gateway.request(
+                    "POST", f"/v1/vector_stores/{store_id}/search", {"query": search_marker}, key=ungranted_key
+                )
+                responses_denied: Final = gateway.request(
+                    "POST",
+                    "/v1/responses",
+                    {
+                        "model": model,
+                        "input": responses_marker,
+                        "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(store_id)}),
+                    },
+                    key=ungranted_key,
+                )
+                search_granted: Final = gateway.request(
+                    "POST", f"/v1/vector_stores/{store_id}/search", {"query": granted_marker}, key=granted_key
+                )
+
+            observations: Final = upstream_observations(upstream_gateway)
+            denied_observations: Final = tuple(
+                observation
+                for observation in observations
+                if search_marker in str(observation["body"]) or responses_marker in str(observation["body"])
+            )
+            granted_searches: Final = tuple(
+                observation
+                for observation in observations
+                if observation["path"] == f"/vector_stores/{store_id}/search"
+                and granted_marker in str(observation["body"])
+            )
+            assert search_denied.status_code == 401, search_denied.text
+            assert search_denied.json()["error"]["type"] == "key_vector_store_access_denied", search_denied.text
+            assert responses_denied.status_code == 401, responses_denied.text
+            assert responses_denied.json()["error"]["type"] == "key_vector_store_access_denied", responses_denied.text
+            assert denied_observations == ()
+            assert search_granted.status_code == 200, search_granted.text
+            assert len(granted_searches) == 1, granted_searches
+
+
+def _auth_cache_subscribers() -> int:
+    with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache:
+        return int(cache.pubsub_numsub(AUTH_CACHE_INVALIDATION_CHANNEL)[0][1])
+
+
+def test_revoked_user_grant_stops_working_on_another_proxy(strict_gateway: StrictGateway, tmp_path: Path) -> None:
+    subscribers_before_peer: Final = _auth_cache_subscribers()
+    with (
+        owned_proxy(
+            strict_gateway.upstream,
+            tmp_path,
+            strict_gateway.environment,
+            config=strict_gateway.config,
+            remove_environment=REMOVE_OPENAI_API_BASE,
+        ) as peer,
+        strict_gateway.gateway.scenario() as scenario,
+    ):
+        eventually(_auth_cache_subscribers, lambda count: count > subscribers_before_peer)
+        model: Final = scenario.model()
+        store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}"
+        user: Final = scenario.user(user_role="internal_user", object_permission=_permission_for_stores(store_id))
+        token: Final = strict_gateway.jwt(user)
+
+        def peer_status() -> int:
+            marker: Final = f"lit6035 revoked user grant {uuid.uuid4().hex}"
+            return peer.request(
+                "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=token
+            ).status_code
+
+        assert eventually(peer_status, lambda status: status == 200, seconds=30, return_last_on_timeout=True) == 200
+        strict_gateway.gateway.post("/user/update", {"user_id": user, "object_permission": _permission_for_stores()})
+
+        assert eventually(peer_status, lambda status: status == 401, seconds=10, return_last_on_timeout=True) == 401

tests/integration/authorization/test_search_tool_deny_by_default.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+import json
+import uuid
+from collections.abc import Iterator
+from pathlib import Path
+from typing import Final, Literal, TypeAlias
+
+import httpx
+import pytest
+import yaml
+from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value
+from integration._support.process import owned_proxy
+from integration._support.wire import Reply, Request, Wire, wire_server
+from pydantic import JsonValue
+
+PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml"
+SEARCH_TOOL: Final = "integration-search"
+OTHER_SEARCH_TOOL: Final = "integration-other-search"
+FAILING_SEARCH_TOOL: Final = "integration-failing-search"
+SEARCH_RESULT: Final = {"title": "Synthetic result", "url": "https://example.test/result", "content": "snippet"}
+JsonObject: TypeAlias = dict[str, JsonValue]
+
+
+def _json_array(*values: JsonValue) -> JsonValue:
+    return [*values]  # mutable-ok: request payloads and YAML sequences require list values
+
+
+def _grant(*search_tools: str) -> JsonObject:
+    permission: Final[JsonObject] = {"search_tools": _json_array(*search_tools)}
+    return permission
+
+
+def _respond(request: Request) -> Reply:
+    if (request.method, request.target) == ("POST", "/failing/search"):
+        return Reply(status=500, body=b'{"error": "synthetic outage"}')
+    assert (request.method, request.target) == ("POST", "/tavily/search"), request
+    query: Final = json.loads(request.body)["query"]
+    return Reply(body=json.dumps({"query": query, "results": [SEARCH_RESULT]}).encode())
+
+
+def _config(directory: Path, wire: Wire) -> Path:
+    config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text()))
+    tool_params: Final[JsonObject] = {
+        "search_provider": "tavily",
+        "api_key": "synthetic-tavily-key",
+        "api_base": f"{wire.url}/tavily",
+    }
+    failing_tool_params: Final[JsonObject] = {**tool_params, "api_base": f"{wire.url}/failing"}
+    strict: Final[JsonObject] = {
+        **config,
+        "general_settings": {**object_value(config["general_settings"]), "search_tool_deny_by_default": True},
+        "router_settings": {
+            **object_value(config["router_settings"]),
+            "fallbacks": _json_array({FAILING_SEARCH_TOOL: _json_array(OTHER_SEARCH_TOOL)}),
+        },
+        "search_tools": _json_array(
+            {"search_tool_name": SEARCH_TOOL, "litellm_params": tool_params},
+            {"search_tool_name": OTHER_SEARCH_TOOL, "litellm_params": tool_params},
+            {"search_tool_name": FAILING_SEARCH_TOOL, "litellm_params": failing_tool_params},
+        ),
+    }
+    path: Final = directory / "proxy_search_tool_deny_by_default.yaml"
+    path.write_text(yaml.safe_dump(strict))
+    return path
+
+
+@pytest.fixture(scope="module")
+def strict_search(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Wire]]:
+    with gateway_from_environment() as upstream, wire_server(_respond) as wire:
+        directory: Final = tmp_path_factory.mktemp("search_tool_deny_by_default")
+        with owned_proxy(upstream, directory, {}, config=_config(directory, wire)) as gateway:
+            yield gateway, wire
+
+
+def _search(gateway: Gateway, key: str, marker: str) -> httpx.Response:
+    return gateway.request("POST", f"/v1/search/{SEARCH_TOOL}", {"query": marker}, key=key)
+
+
+def _searched(wire: Wire, marker: str) -> tuple[Request, ...]:
+    return tuple(request for request in wire.drain() if marker in request.body.decode())
+
+
+Caller: TypeAlias = Literal[
+    "standalone_key_no_permission",
+    "standalone_key_empty_grant",
+    "standalone_key_other_tool",
+    "team_key_empty_key_grant",
+    "team_key_empty_team_grant",
+    "proxy_admin_key_no_grant",
+]
+
+
+def _denied_key(scenario: Scenario, caller: Caller) -> tuple[str, str]:
+    if caller == "standalone_key_no_permission":
+        return scenario.key(), "key_search_tool_access_denied"
+    if caller == "standalone_key_empty_grant":
+        return scenario.key(object_permission=_grant()), "key_search_tool_access_denied"
+    if caller == "standalone_key_other_tool":
+        return scenario.key(object_permission=_grant(OTHER_SEARCH_TOOL)), "key_search_tool_access_denied"
+    if caller == "team_key_empty_key_grant":
+        team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL))
+        return scenario.key(team_id=team, object_permission=_grant()), "key_search_tool_access_denied"
+    if caller == "team_key_empty_team_grant":
+        empty_team: Final = scenario.team(object_permission=_grant())
+        return scenario.key(team_id=empty_team, object_permission=_grant(SEARCH_TOOL)), "team_search_tool_access_denied"
+    admin: Final = scenario.user(user_role="proxy_admin")
+    return scenario.key(user_id=admin), "key_search_tool_access_denied"
+
+
+@pytest.mark.parametrize(
+    "caller",
+    [
+        "standalone_key_no_permission",
+        "standalone_key_empty_grant",
+        "standalone_key_other_tool",
+        "team_key_empty_key_grant",
+        "team_key_empty_team_grant",
+        "proxy_admin_key_no_grant",
+    ],
+)
+def test_deny_by_default_rejects_ungranted_search_before_the_provider_is_called(
+    strict_search: tuple[Gateway, Wire], caller: Caller
+) -> None:
+    gateway, wire = strict_search
+    with gateway.scenario() as scenario:
+        key, error_type = _denied_key(scenario, caller)
+        marker: Final = f"search deny {caller} {uuid.uuid4().hex}"
+
+        response: Final = _search(gateway, key, marker)
+
+        assert response.status_code == 403, response.text
+        assert response.json()["error"]["type"] == error_type, response.text
+        assert _searched(wire, marker) == ()
+
+
+@pytest.mark.parametrize("scope", ["standalone_key", "team_key", "master_key"])
+def test_deny_by_default_serves_explicitly_granted_search(
+    strict_search: tuple[Gateway, Wire], scope: Literal["standalone_key", "team_key", "master_key"]
+) -> None:
+    gateway, wire = strict_search
+    with gateway.scenario() as scenario:
+        if scope == "standalone_key":
+            key = scenario.key(object_permission=_grant(SEARCH_TOOL))
+        elif scope == "team_key":
+            team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL))
+            key = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL))
+        else:
+            key = gateway.key
+        marker: Final = f"search allow {scope} {uuid.uuid4().hex}"
+
+        response: Final = _search(gateway, key, marker)
+
+        assert response.status_code == 200, response.text
+        assert response.json()["results"][0]["url"] == SEARCH_RESULT["url"]
+        assert len(_searched(wire, marker)) == 1
+
+
+def test_search_tools_list_shows_only_the_tools_the_key_and_its_team_both_grant(
+    strict_search: tuple[Gateway, Wire],
+) -> None:
+    gateway, _ = strict_search
+    with gateway.scenario() as scenario:
+        team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL, OTHER_SEARCH_TOOL))
+        member: Final = scenario.member(team)
+        key: Final = scenario.key(team_id=team, user_id=member, object_permission=_grant(SEARCH_TOOL))
+
+        response: Final = gateway.request("GET", "/search_tools/list", key=key)
+
+        assert response.status_code == 200, response.text
+        assert [tool["search_tool_name"] for tool in response.json()["search_tools"]] == [SEARCH_TOOL]
+
+
+def test_revoking_a_team_search_grant_takes_effect_on_the_next_request(strict_search: tuple[Gateway, Wire]) -> None:
+    gateway, wire = strict_search
+    with gateway.scenario() as scenario:
+        team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL))
+        key: Final = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL))
+        assert _search(gateway, key, f"search warm {uuid.uuid4().hex}").status_code == 200
+
+        gateway.post("/team/update", {"team_id": team, "object_permission": _grant()})
+        marker: Final = f"search revoked {uuid.uuid4().hex}"
+        response: Final = _search(gateway, key, marker)
+
+        assert response.status_code == 403, response.text
+        assert response.json()["error"]["type"] == "team_search_tool_access_denied", response.text
+        assert _searched(wire, marker) == ()
+
+
+def test_deny_by_default_authorizes_a_search_tool_named_by_the_model_field(strict_search: tuple[Gateway, Wire]) -> None:
+    gateway, wire = strict_search
+    with gateway.scenario() as scenario:
+        key: Final = scenario.key()
+        marker: Final = f"search model field {uuid.uuid4().hex}"
+
+        response: Final = gateway.request("POST", "/v1/search", {"model": SEARCH_TOOL, "query": marker}, key=key)
+
+        assert response.status_code == 403, response.text
+        assert response.json()["error"]["type"] == "key_search_tool_access_denied", response.text
+        assert _searched(wire, marker) == ()
+
+
+@pytest.mark.parametrize("fallback_granted", [False, True])
+def test_router_search_fallback_only_reaches_a_granted_tool(
+    strict_search: tuple[Gateway, Wire], fallback_granted: bool
+) -> None:
+    gateway, wire = strict_search
+    with gateway.scenario() as scenario:
+        granted: Final = (FAILING_SEARCH_TOOL, OTHER_SEARCH_TOOL) if fallback_granted else (FAILING_SEARCH_TOOL,)
+        key: Final = scenario.key(object_permission=_grant(*granted))
+        marker: Final = f"search fallback {fallback_granted} {uuid.uuid4().hex}"
+
+        response: Final = gateway.request("POST", f"/v1/search/{FAILING_SEARCH_TOOL}", {"query": marker}, key=key)
+
+        searched: Final = tuple(request.target for request in _searched(wire, marker))
+        if fallback_granted:
+            assert response.status_code == 200, response.text
+            assert searched == ("/failing/search", "/tavily/search")
+        else:
+            assert response.status_code == 500, response.text
+            assert searched == ("/failing/search",)

tests/integration/caching/response_cache_case.py

@@ -0,0 +1,364 @@
+"""Scripted provider answers, upstream observations and Redis helpers shared by the response cache cells.
+
+The bodies carry ``$UNIQUE_ID`` so every upstream answer has its own id: an answer served from the
+cache repeats the id of the request that filled it, an answer that reached the upstream again does not.
+"""
+
+from __future__ import annotations
+
+import ast
+import json
+import os
+import re
+import uuid
+from collections.abc import Iterator, Mapping
+from contextlib import contextmanager
+from typing import Final
+
+import httpx
+from pydantic import JsonValue
+from redis import Redis
+
+from tests.integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value
+from tests.integration._support.upstream import CONTROL_URL, ScenarioHandle, delete_scenario, register_scenario
+from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
+
+CACHE_KEY: Final = re.compile(r"^[0-9a-f]{64}$")
+PROVIDER_KEY: Final = "integration-provider-key"
+CHAT_MODEL: Final = "openai/gpt-4o-mini"
+TEXT_MODEL: Final = "openai/gpt-3.5-turbo-instruct"
+MESSAGES_MODEL: Final = "anthropic/claude-haiku-4-5"
+
+CHAT: Final[dict[str, JsonValue]] = {
+    "id": "chatcmpl-$UNIQUE_ID",
+    "object": "chat.completion",
+    "created": 1,
+    "model": "gpt-4o-mini",
+    "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+TEXT: Final[dict[str, JsonValue]] = {
+    "id": "cmpl-$UNIQUE_ID",
+    "object": "text_completion",
+    "created": 1,
+    "model": "gpt-3.5-turbo-instruct",
+    "choices": [{"text": "scripted", "index": 0, "finish_reason": "stop"}],
+    "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
+}
+RESPONSE: Final[dict[str, JsonValue]] = {
+    "id": "resp_$UNIQUE_ID",
+    "object": "response",
+    "created_at": 1,
+    "status": "completed",
+    "model": "gpt-4o-mini",
+    "output": [
+        {
+            "id": "msg_$UNIQUE_ID",
+            "type": "message",
+            "status": "completed",
+            "role": "assistant",
+            "content": [{"type": "output_text", "text": "scripted", "annotations": []}],
+        }
+    ],
+    "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2},
+}
+MESSAGE: Final[dict[str, JsonValue]] = {
+    "id": "msg-$UNIQUE_ID",
+    "type": "message",
+    "role": "assistant",
+    "model": "claude-haiku-4-5",
+    "content": [{"type": "text", "text": "scripted"}],
+    "stop_reason": "end_turn",
+    "stop_sequence": None,
+    "usage": {"input_tokens": 1, "output_tokens": 1},
+}
+EMBEDDING: Final[dict[str, JsonValue]] = {
+    "object": "list",
+    "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}],
+    "model": "text-embedding-$UNIQUE_ID",
+    "usage": {"prompt_tokens": 1, "total_tokens": 1},
+}
+RERANK: Final[dict[str, JsonValue]] = {
+    "id": "rerank-$UNIQUE_ID",
+    "results": [{"index": 0, "relevance_score": 0.5}],
+    "meta": {},
+}
+TRANSCRIPT: Final[dict[str, JsonValue]] = {"text": "scripted $UNIQUE_ID"}
+
+CHAT_CHUNK: Final = (
+    'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":'
+)
+CHAT_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        CHAT_CHUNK + '[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{"content":"streamed "},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{"content":"response"},"finish_reason":null}]}',
+        CHAT_CHUNK + '[{"index":0,"delta":{},"finish_reason":"stop"}]}',
+        "data: [DONE]",
+    ),
+)
+CHAT_STREAM_WITHOUT_CHOICES: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(CHAT_CHUNK + "[]}", "data: [DONE]"),
+)
+TEXT_CHUNK: Final = (
+    'data: {"id":"cmpl-$UNIQUE_ID","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":'
+)
+TEXT_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        TEXT_CHUNK + '[{"text":"streamed ","index":0,"finish_reason":null}]}',
+        TEXT_CHUNK + '[{"text":"response","index":0,"finish_reason":null}]}',
+        TEXT_CHUNK + '[{"text":"","index":0,"finish_reason":"stop"}]}',
+        "data: [DONE]",
+    ),
+)
+TEXT_STREAM_WITHOUT_CHOICES: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(TEXT_CHUNK + "[]}", "data: [DONE]"),
+)
+RESPONSE_CREATED: Final = (
+    "event: response.created\n"
+    'data: {"type":"response.created","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+    '"created_at":1,"status":"in_progress","model":"gpt-4o-mini","output":[],"usage":null}}'
+)
+RESPONSE_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        RESPONSE_CREATED,
+        (
+            "event: response.output_item.added\n"
+            'data: {"type":"response.output_item.added","output_index":0,'
+            '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"in_progress","role":"assistant","content":[]}}'
+        ),
+        (
+            "event: response.content_part.added\n"
+            'data: {"type":"response.content_part.added","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}'
+        ),
+        (
+            "event: response.output_text.delta\n"
+            'data: {"type":"response.output_text.delta","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"delta":"streamed response"}'
+        ),
+        (
+            "event: response.output_text.done\n"
+            'data: {"type":"response.output_text.done","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"text":"streamed response"}'
+        ),
+        (
+            "event: response.content_part.done\n"
+            'data: {"type":"response.content_part.done","item_id":"msg_$UNIQUE_ID","output_index":0,'
+            '"content_index":0,"part":{"type":"output_text","text":"streamed response","annotations":[]}}'
+        ),
+        (
+            "event: response.output_item.done\n"
+            'data: {"type":"response.output_item.done","output_index":0,'
+            '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"completed","role":"assistant",'
+            '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}}'
+        ),
+        (
+            "event: response.completed\n"
+            'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+            '"created_at":1,"status":"completed","model":"gpt-4o-mini",'
+            '"output":[{"id":"msg_$UNIQUE_ID","type":"message","status":"completed","role":"assistant",'
+            '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}],'
+            '"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}'
+        ),
+    ),
+)
+RESPONSE_STREAM_WITHOUT_OUTPUT: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        RESPONSE_CREATED,
+        (
+            "event: response.completed\n"
+            'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",'
+            '"created_at":1,"status":"completed","model":"gpt-4o-mini","output":[],'
+            '"usage":{"input_tokens":1,"output_tokens":0,"total_tokens":1}}}'
+        ),
+    ),
+)
+MESSAGE_START: Final = (
+    "event: message_start\n"
+    'data: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant",'
+    '"model":"claude-haiku-4-5","content":[],"stop_reason":null,"stop_sequence":null,'
+    '"usage":{"input_tokens":1,"output_tokens":0}}}'
+)
+MESSAGE_END: Final = (
+    (
+        "event: message_delta\n"
+        'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},'
+        '"usage":{"output_tokens":2}}'
+    ),
+    'event: message_stop\ndata: {"type":"message_stop"}',
+)
+MESSAGE_STREAM: Final = SseResponse(
+    content_type="text/event-stream",
+    frames=(
+        MESSAGE_START,
+        (
+            "event: content_block_start\n"
+            'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}'
+        ),
+        (
+            "event: content_block_delta\n"
+            'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"streamed response"}}'
+        ),
+        'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}',
+        *MESSAGE_END,
+    ),
+)
+MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS: Final = SseResponse(
+    content_type="text/event-stream", frames=(MESSAGE_START, *MESSAGE_END)
+)
+
+
+class Upstream:
+    """The owned upstream's observations, drained once per read and kept, filtered per deployment."""
+
+    def __init__(self, url: str = CONTROL_URL) -> None:
+        self._url: Final = url.rstrip("/")
+        self._seen: tuple[dict[str, JsonValue], ...] = ()
+
+    def received(self, identity: str) -> tuple[dict[str, JsonValue], ...]:
+        with httpx.Client(timeout=10, trust_env=False) as client:
+            payload: Final = JSON_OBJECT.validate_json(client.get(f"{self._url}/__observations").content)
+        requests: Final = payload["requests"]
+        assert isinstance(requests, list)
+        self._seen = (
+            *self._seen,
+            *(object_value(item) for item in requests),
+        )  # rebind-ok: drained observations accumulate
+        return tuple(item for item in self._seen if f"/{identity}/" in string_value(item.get("path", "")))
+
+    def calls(self, identity: str) -> int:
+        """The LLM calls the deployment took; the proxy's boot-time ``GET /v1/models`` discovery of a config
+        deployment is not one."""
+        return sum(1 for item in self.received(identity) if item.get("method") == "POST")
+
+
+def json_response(body: Mapping[str, JsonValue], status: int = 200) -> JsonResponse:
+    return JsonResponse(content_type="application/json", body=dict(body), status=status)
+
+
+def slowed(stream: SseResponse, frame_delay_ms: int) -> SseResponse:
+    return stream.model_copy(update={"frame_delay_ms": frame_delay_ms})
+
+
+def emptied(body: Mapping[str, JsonValue], field: str) -> dict[str, JsonValue]:
+    return {**body, field: []}
+
+
+def scenario_id() -> str:
+    return f"cache-{uuid.uuid4().hex[:12]}"
+
+
+def scripted(scenario: Scenario, response: JsonResponse | SseResponse) -> ScenarioHandle:
+    handle: Final = register_scenario(scenario_id(), response, control_url=scenario.gateway.upstream_url)
+    scenario.cleanups.callback(delete_scenario, handle)
+    return handle
+
+
+@contextmanager
+def scripted_in_process(response: JsonResponse | SseResponse) -> Iterator[ScenarioHandle]:
+    handle: Final = register_scenario(scenario_id(), response)
+    try:
+        yield handle
+    finally:
+        delete_scenario(handle)
+
+
+def rescript(handle: ScenarioHandle, response: JsonResponse | SseResponse) -> None:
+    register_scenario(handle.scenario_id, response, control_url=handle.control_url)
+
+
+def prompt() -> str:
+    return f"cache probe {uuid.uuid4().hex}"
+
+
+def redis_store() -> Redis:
+    return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]))
+
+
+def entry_key(store: Redis, needle: str) -> str | None:
+    """The response cache key (a bare sha256) whose stored value names ``needle``, or ``None``."""
+    for raw_key in store.scan_iter(count=1000):
+        key: Final = raw_key.decode()
+        if not CACHE_KEY.match(key) or store.type(key) != b"string":
+            continue
+        value: Final = store.get(key)
+        if value is not None and needle.encode() in value:
+            return key
+    return None
+
+
+def await_entry(store: Redis, needle: str) -> str:
+    key: Final = eventually(lambda: entry_key(store, needle), lambda found: found is not None)
+    assert key is not None
+    return key
+
+
+def _decode_entry(raw: bytes) -> tuple[dict[str, JsonValue], bool]:
+    """The stored entry and whether it is JSON; the sync SDK path stores ``str(dict)`` instead."""
+    text: Final = raw.decode()
+    if text.startswith('{"'):
+        return object_value(json.loads(text)), True
+    return object_value(ast.literal_eval(text)), False
+
+
+SHORT_TTL_SECONDS: Final = 2
+SHORT_TTL_CACHE_CONTROL: Final[dict[str, JsonValue]] = {"ttl": SHORT_TTL_SECONDS}
+STALE_ENTRY_TTL_SECONDS: Final = 60
+STALE_ID_SUFFIX: Final = "-stale"
+
+
+def flip_entry_to_empty(store: Redis, key: str, field: str) -> str:
+    """Rewrite the stored response so ``field`` is ``[]`` and its id carries ``-stale``, keeping the encoding.
+
+    Returns the stale id. The proxy worker that wrote the entry keeps a copy in its own memory for the entry's ttl,
+    so a cell fills the entry with ``SHORT_TTL_CACHE_CONTROL`` and reads the rewritten entry once that copy lapsed.
+    """
+    raw: Final = store.get(key)
+    assert raw is not None, key
+    entry, is_json = _decode_entry(raw)
+    response: Final = entry["response"]
+    stored: Final = object_value(json.loads(response) if isinstance(response, str) else response)
+    assert field in stored, f"{field!r} missing from the cached response {sorted(stored)}"
+    stale_id: Final = f"{string_value(stored['id'])}{STALE_ID_SUFFIX}"
+    flipped: Final[dict[str, JsonValue]] = {**stored, field: [], "id": stale_id}
+    rewritten: Final[dict[str, JsonValue]] = {
+        **entry,
+        "response": json.dumps(flipped) if isinstance(response, str) else flipped,
+    }
+    store.set(key, json.dumps(rewritten) if is_json else str(rewritten), ex=STALE_ENTRY_TTL_SECONDS)
+    return stale_id
+
+
+def chat_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]:
+    return {"model": model, "messages": [{"role": "user", "content": text}], **extra}
+
+
+def message_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]:
+    return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}], **extra}
+
+
+def response_id(response: httpx.Response) -> str:
+    assert response.status_code == 200, response.text
+    return string_value(JSON_OBJECT.validate_json(response.content)["id"])
+
+
+def choices(response: httpx.Response) -> list[JsonValue]:
+    assert response.status_code == 200, response.text
+    found: Final = JSON_OBJECT.validate_json(response.content)["choices"]
+    assert isinstance(found, list), response.text
+    return found
+
+
+def content_of_first_choice(response: httpx.Response) -> JsonValue:
+    return object_value(object_value(choices(response)[0])["message"])["content"]
+
+
+def text_of_first_choice(response: httpx.Response) -> JsonValue:
+    return object_value(choices(response)[0])["text"]

tests/integration/caching/test_response_cache_chaos.py

@@ -0,0 +1,503 @@
+"""Chaos rows for the response cache on an owned two-worker proxy.
+
+Each owned proxy carries its own deployments pointed at scripted upstream scenarios, so both workers know
+them from boot. A 24-request burst (chat, text completions, Responses and Anthropic Messages, two of each
+streamed at 500 ms per frame so they are still in flight when the outage lands) runs while the cache store
+is stopped (C1), paused (C2) or while one worker has just been SIGKILLed (C3): every request answers 200
+with content and lands exactly once in the spend log. After recovery an upstream answer with no choices
+still reaches the upstream again on the next identical request. C4 runs the in-memory cache mode, where
+each worker keeps its own store and the cached object is read back without a JSON round trip.
+"""
+
+from __future__ import annotations
+
+import os
+import threading
+import uuid
+from collections.abc import Callable, Iterator, Sequence
+from concurrent.futures import Future, ThreadPoolExecutor
+from dataclasses import dataclass
+from hashlib import sha256
+from pathlib import Path
+from typing import Final
+
+import httpx
+import psutil
+import pytest
+import yaml
+from pydantic import JsonValue
+from redis import Redis
+
+from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, string_value
+from tests.integration._support.database import read_rows
+from tests.integration._support.process import owned_proxy_process
+from tests.integration._support.redis_process import owned_redis
+from tests.integration._support.upstream import ScenarioHandle
+from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    CHAT_STREAM,
+    MESSAGE,
+    MESSAGE_STREAM,
+    MESSAGES_MODEL,
+    PROVIDER_KEY,
+    RESPONSE,
+    RESPONSE_STREAM,
+    TEXT,
+    TEXT_MODEL,
+    TEXT_STREAM,
+    Upstream,
+    chat_body,
+    choices,
+    content_of_first_choice,
+    emptied,
+    json_response,
+    message_body,
+    prompt,
+    rescript,
+    response_id,
+    scripted,
+    slowed,
+)
+
+RecordProperty = Callable[[str, object], None]
+
+WORKERS: Final = 2
+REMOVE_FROM_ENVIRONMENT: Final = ("DATABASE_URL_READ_REPLICA",)
+BURST_TIMEOUT_SECONDS: Final = 60
+CHAOS_AFTER_ANSWERS: Final = 4
+STREAM_FRAME_DELAY_MS: Final = 500
+PER_ENDPOINT: Final = 6
+STREAMS_PER_ENDPOINT: Final = 2
+LOCAL_BATCH: Final = 20
+SPEND_SQL: Final = 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s'
+
+
+@dataclass(frozen=True, slots=True)
+class Deployment:
+    name: str
+    litellm_model: str
+    handle: ScenarioHandle
+
+    def entry(self) -> dict[str, JsonValue]:
+        return {
+            "model_name": self.name,
+            "litellm_params": {
+                "model": self.litellm_model,
+                "api_base": self.handle.api_base(),
+                "api_key": PROVIDER_KEY,
+            },
+        }
+
+
+@dataclass(frozen=True, slots=True)
+class Fleet:
+    chat: Deployment
+    chat_stream: Deployment
+    text: Deployment
+    text_stream: Deployment
+    responses: Deployment
+    responses_stream: Deployment
+    messages: Deployment
+    messages_stream: Deployment
+    flip: Deployment
+
+    def burst_members(self) -> tuple[Deployment, ...]:
+        return (
+            self.chat,
+            self.chat_stream,
+            self.text,
+            self.text_stream,
+            self.responses,
+            self.responses_stream,
+            self.messages,
+            self.messages_stream,
+        )
+
+    def entries(self) -> list[dict[str, JsonValue]]:
+        return [deployment.entry() for deployment in (*self.burst_members(), self.flip)]
+
+
+@dataclass(frozen=True, slots=True)
+class Call:
+    label: str
+    path: str
+    body: dict[str, JsonValue]
+
+    @property
+    def streamed(self) -> bool:
+        return self.label.startswith("stream-")
+
+
+@dataclass(frozen=True, slots=True)
+class Answer:
+    call: Call
+    status: int | None
+    text: str
+
+    @property
+    def has_content(self) -> bool:
+        marker: Final = "streamed" if self.call.streamed else "scripted"
+        return self.status == 200 and marker in self.text
+
+    def describe(self) -> str:
+        return f"{self.call.label}: {self.status} {self.text[:160]!r}"
+
+
+def _deployment(scenario: Scenario, kind: str, litellm_model: str, response: JsonResponse | SseResponse) -> Deployment:
+    return Deployment(f"chaos-{kind}-{uuid.uuid4().hex[:8]}", litellm_model, scripted(scenario, response))
+
+
+def _fleet(scenario: Scenario) -> Fleet:
+    return Fleet(
+        chat=_deployment(scenario, "chat", CHAT_MODEL, json_response(CHAT)),
+        chat_stream=_deployment(scenario, "chat-stream", CHAT_MODEL, slowed(CHAT_STREAM, STREAM_FRAME_DELAY_MS)),
+        text=_deployment(scenario, "text", TEXT_MODEL, json_response(TEXT)),
+        text_stream=_deployment(scenario, "text-stream", TEXT_MODEL, slowed(TEXT_STREAM, STREAM_FRAME_DELAY_MS)),
+        responses=_deployment(scenario, "responses", CHAT_MODEL, json_response(RESPONSE)),
+        responses_stream=_deployment(
+            scenario, "responses-stream", CHAT_MODEL, slowed(RESPONSE_STREAM, STREAM_FRAME_DELAY_MS)
+        ),
+        messages=_deployment(scenario, "messages", MESSAGES_MODEL, json_response(MESSAGE)),
+        messages_stream=_deployment(
+            scenario, "messages-stream", MESSAGES_MODEL, slowed(MESSAGE_STREAM, STREAM_FRAME_DELAY_MS)
+        ),
+        flip=_deployment(scenario, "flip", CHAT_MODEL, json_response(emptied(CHAT, "choices"))),
+    )
+
+
+def _config(directory: Path, fleet: Fleet, cache_params: dict[str, JsonValue] | None = None) -> Path:
+    """The rig's proxy config with the fleet as its model list and, when given, another response cache."""
+    rig: Final = yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text())
+    assert isinstance(rig, dict)
+    litellm_settings: Final = rig["litellm_settings"]
+    assert isinstance(litellm_settings, dict)
+    config: Final = {
+        **rig,
+        "model_list": fleet.entries(),
+        "litellm_settings": {**litellm_settings, **({"cache_params": cache_params} if cache_params else {})},
+    }
+    path: Final = directory / f"proxy_config-{uuid.uuid4().hex[:8]}.yaml"
+    path.write_text(yaml.safe_dump(config, sort_keys=False))
+    return path
+
+
+def _overrides() -> dict[str, str]:
+    return {"DATABASE_URL": os.environ["DATABASE_URL"]}
+
+
+def _burst_calls(fleet: Fleet, per_endpoint: int, streams_per_endpoint: int) -> tuple[Call, ...]:
+    def calls() -> Iterator[Call]:
+        for index in range(per_endpoint):
+            streamed: Final = index < streams_per_endpoint
+            prefix: Final = "stream" if streamed else "plain"
+            extra: Final[dict[str, JsonValue]] = {"stream": True} if streamed else {}
+            chat: Final = fleet.chat_stream if streamed else fleet.chat
+            text: Final = fleet.text_stream if streamed else fleet.text
+            responses: Final = fleet.responses_stream if streamed else fleet.responses
+            messages: Final = fleet.messages_stream if streamed else fleet.messages
+            yield Call(f"{prefix}-chat-{index}", "/v1/chat/completions", chat_body(chat.name, prompt(), **extra))
+            yield Call(f"{prefix}-text-{index}", "/v1/completions", {"model": text.name, "prompt": prompt(), **extra})
+            yield Call(
+                f"{prefix}-responses-{index}", "/v1/responses", {"model": responses.name, "input": prompt(), **extra}
+            )
+            yield Call(f"{prefix}-messages-{index}", "/v1/messages", message_body(messages.name, prompt(), **extra))
+
+    return tuple(calls())
+
+
+class Burst:
+    """Every call submitted at once against ``target``; ``chaos_point`` is set once ``CHAOS_AFTER_ANSWERS``
+    have answered (or failed), while the slowed streams are still in flight."""
+
+    def __init__(self, target: str, key: str) -> None:
+        self._target: Final = target
+        self._key: Final = key
+        self._lock: Final = threading.Lock()
+        self._answers = 0  # rebind-ok: counter behind _lock
+        self._futures: dict[str, Future[Answer]] = {}
+        self.chaos_point: Final = threading.Event()
+
+    def start(self, pool: ThreadPoolExecutor, calls: Sequence[Call]) -> None:
+        assert not self._futures, "burst already started"
+        self._futures.update((call.label, pool.submit(self._send, call)) for call in calls)
+        assert self.chaos_point.wait(BURST_TIMEOUT_SECONDS), (
+            f"fewer than {CHAOS_AFTER_ANSWERS} requests answered within {BURST_TIMEOUT_SECONDS}s"
+        )
+
+    def _send(self, call: Call) -> Answer:
+        try:
+            with httpx.Client(base_url=self._target, timeout=BURST_TIMEOUT_SECONDS, trust_env=False) as client:
+                response: Final = client.post(
+                    call.path, json=call.body, headers={"Authorization": f"Bearer {self._key}"}
+                )
+            answer = Answer(call, response.status_code, response.text)
+        except httpx.HTTPError as error:
+            answer = Answer(call, None, f"{type(error).__name__}: {error}"[:200])
+        with self._lock:
+            self._answers += 1
+            if self._answers >= CHAOS_AFTER_ANSWERS:
+                self.chaos_point.set()
+        return answer
+
+    def answered(self) -> int:
+        with self._lock:
+            return self._answers
+
+    def pending_streams(self) -> tuple[str, ...]:
+        return tuple(
+            label for label, future in self._futures.items() if label.startswith("stream-") and not future.done()
+        )
+
+    def outcomes(self) -> tuple[Answer, ...]:
+        return tuple(future.result(timeout=BURST_TIMEOUT_SECONDS + 30) for future in self._futures.values())
+
+
+def _assert_all_answered_with_content(outcomes: Sequence[Answer]) -> None:
+    failures: Final = tuple(answer.describe() for answer in outcomes if not answer.has_content)
+    assert not failures, (
+        f"{len(failures)} of {len(outcomes)} burst requests lack a 200 with content:\n  " + "\n  ".join(failures)
+    )
+
+
+def _plain_ids(outcomes: Sequence[Answer]) -> frozenset[str]:
+    """The ids of the non-streamed chat, text and Messages answers, which the spend log records verbatim."""
+    return frozenset(
+        string_value(JSON_OBJECT.validate_json(answer.text)["id"])
+        for answer in outcomes
+        if not answer.call.streamed and answer.call.path != "/v1/responses"
+    )  # comprehension-ok: one filter over the burst
+
+
+def _spend_request_ids(key: str) -> frozenset[str]:
+    return frozenset(
+        string_value(row["request_id"]) for row in read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),))
+    )
+
+
+def _assert_every_burst_request_landed_once(key: str, outcomes: Sequence[Answer]) -> None:
+    rows: Final = eventually(lambda: _spend_request_ids(key), lambda ids: len(ids) >= len(outcomes), seconds=70)
+    assert len(rows) == len(outcomes), f"{len(rows)} spend rows for {len(outcomes)} burst requests: {sorted(rows)}"
+    missing: Final = _plain_ids(outcomes) - rows
+    assert not missing, f"burst answers without a spend row: {sorted(missing)}"
+
+
+def _chat(target: str, key: str, body: dict[str, JsonValue]) -> httpx.Response:
+    with httpx.Client(base_url=target, timeout=30, trust_env=False) as client:
+        return client.post("/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"})
+
+
+def _assert_empty_answer_reaches_upstream_again(target: str, key: str, upstream: Upstream, flip: Deployment) -> None:
+    """Through ``target`` after the outage: the scripted empty answer is never served again once the
+    upstream answers for real, and the real answer is the one served from the cache afterwards."""
+    request: Final = chat_body(flip.name, prompt())
+    empty: Final = _chat(target, key, request)
+    assert choices(empty) == [], empty.text
+    assert upstream.calls(flip.handle.scenario_id) == 1
+    rescript(flip.handle, json_response(CHAT))
+    refilled: Final = _chat(target, key, request)
+    assert content_of_first_choice(refilled) == "scripted", f"the empty answer was served: {refilled.text}"
+    assert upstream.calls(flip.handle.scenario_id) == 2
+    hit: Final = eventually(lambda: _chat(target, key, request), lambda r: response_id(r) == response_id(refilled))
+    assert content_of_first_choice(hit) == "scripted", hit.text
+    assert upstream.calls(flip.handle.scenario_id) >= 2
+
+
+def _workers(root: psutil.Process) -> tuple[psutil.Process, ...]:
+    """uvicorn's worker children of the owned proxy root, spawned through ``multiprocessing.spawn``."""
+
+    def spawned() -> Iterator[psutil.Process]:
+        for child in root.children():
+            try:
+                cmdline = child.cmdline()
+            except (psutil.NoSuchProcess, psutil.AccessDenied):
+                continue
+            if any("multiprocessing.spawn" in part for part in cmdline):
+                yield child
+
+    return tuple(sorted(spawned(), key=lambda process: process.pid))
+
+
+def _cache_ping(target: str, key: str) -> httpx.Response:
+    with httpx.Client(base_url=target, timeout=30, trust_env=False) as client:
+        return client.get("/cache/ping", headers={"Authorization": f"Bearer {key}"})
+
+
+def _cache_status(response: httpx.Response) -> str:
+    assert response.status_code == 200, f"/cache/ping: {response.status_code} {response.text}"
+    return string_value(JSON_OBJECT.validate_json(response.content)["status"])
+
+
+def _proxy_url(gateway: Gateway) -> str:
+    return str(gateway.client.base_url).rstrip("/")
+
+
+@pytest.mark.timeout(300)
+def test_redis_stopped_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario, owned_redis(tmp_path) as store:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                {
+                    **_overrides(),
+                    "REDIS_HOST": store.host,
+                    "REDIS_PORT": str(store.port),
+                    "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "0",
+                },
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT))
+            pending: Final = burst.pending_streams()
+            record_property("answered_before_outage", burst.answered())
+            assert pending, "no stream was in flight when Redis stopped"
+            store.stop()
+            down: Final = _cache_ping(target, gateway.key)
+            assert down.status_code == 503, f"/cache/ping with Redis stopped: {down.status_code} {down.text}"
+            assert "Service Unhealthy" in down.text, down.text
+            outcomes: Final = burst.outcomes()
+            store.start()
+            recovered: Final = eventually(
+                lambda: _cache_ping(target, gateway.key), lambda r: r.status_code == 200, seconds=60
+            )
+            assert _cache_status(recovered) == "healthy"
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+@pytest.mark.timeout(300)
+def test_redis_paused_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario, owned_redis(tmp_path) as store:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                {**_overrides(), "REDIS_HOST": store.host, "REDIS_PORT": str(store.port)},
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+            Redis(host=store.host, port=store.port) as control,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT))
+            pending: Final = burst.pending_streams()
+            record_property("answered_before_pause", burst.answered())
+            assert pending, "no stream was in flight when Redis was paused"
+            assert control.client_pause(2000) is True
+            outcomes: Final = burst.outcomes()
+            assert _cache_status(_cache_ping(target, gateway.key)) == "healthy"
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+@pytest.mark.timeout(300)
+def test_worker_killed_before_burst_leaves_the_survivor_serving_and_never_pins_an_empty_answer(
+    gateway: Gateway, tmp_path: Path, record_property: RecordProperty
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)])
+        with (
+            owned_proxy_process(
+                gateway,
+                tmp_path,
+                _overrides(),
+                config=_config(tmp_path, fleet),
+                remove_environment=REMOVE_FROM_ENVIRONMENT,
+                workers=WORKERS,
+            ) as owned,
+            ThreadPoolExecutor(PER_ENDPOINT * 4) as pool,
+        ):
+            target: Final = _proxy_url(owned.gateway)
+            root: Final = psutil.Process(owned.process.pid)
+            before: Final = _workers(root)
+            assert len(before) == WORKERS, [process.pid for process in before]
+            victim: Final = before[0]
+            victim.kill()
+            victim.wait(timeout=10)
+            with httpx.Client(base_url=target, timeout=15, trust_env=False) as fresh:
+                readiness: Final = fresh.get("/health/readiness")
+            assert readiness.status_code == 200, f"/health/readiness with worker {victim.pid} dead: {readiness.text}"
+            burst: Final = Burst(target, key)
+            burst.start(pool, _burst_calls(fleet, PER_ENDPOINT // 2, STREAMS_PER_ENDPOINT // 2))
+            outcomes: Final = burst.outcomes()
+            respawned: Final = eventually(
+                lambda: tuple(process.pid for process in _workers(root)),
+                lambda pids: len(pids) == WORKERS and victim.pid not in pids,
+                seconds=60,
+            )
+            record_property(
+                "worker_pids", {"before": [process.pid for process in before], "killed": victim.pid, "after": respawned}
+            )
+            _assert_all_answered_with_content(outcomes)
+            _assert_every_burst_request_landed_once(key, outcomes)
+            _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip)
+
+
+def _batch(target: str, key: str, body: dict[str, JsonValue], size: int) -> tuple[httpx.Response, ...]:
+    """``size`` identical requests, each on a fresh connection so both workers take traffic."""
+    return tuple(_chat(target, key, body) for _ in range(size))
+
+
+@pytest.mark.timeout(300)
+def test_in_memory_cache_mode_never_serves_an_empty_answer_from_a_worker_store(
+    gateway: Gateway, tmp_path: Path
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        fleet: Final = _fleet(scenario)
+        key: Final = scenario.key(models=[fleet.flip.name])
+        with owned_proxy_process(
+            gateway,
+            tmp_path,
+            _overrides(),
+            config=_config(tmp_path, fleet, cache_params={"type": "local"}),
+            remove_environment=REMOVE_FROM_ENVIRONMENT,
+            workers=WORKERS,
+        ) as owned:
+            target: Final = _proxy_url(owned.gateway)
+            request: Final = chat_body(fleet.flip.name, prompt())
+            first_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            assert all(choices(response) == [] for response in first_batch), [r.text for r in first_batch]
+            assert upstream.calls(fleet.flip.handle.scenario_id) == LOCAL_BATCH, (
+                "an empty answer was served from a worker store"
+            )
+            rescript(fleet.flip.handle, json_response(CHAT))
+            second_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            served_empty: Final = tuple(response.text for response in second_batch if choices(response) == [])
+            assert not served_empty, (
+                f"{len(served_empty)} of {LOCAL_BATCH} answers still empty after the upstream recovered"
+            )
+            assert all(content_of_first_choice(response) == "scripted" for response in second_batch)
+            after_second: Final = upstream.calls(fleet.flip.handle.scenario_id)
+            assert LOCAL_BATCH < after_second <= 2 * LOCAL_BATCH, after_second
+            third_batch: Final = _batch(target, key, request, LOCAL_BATCH)
+            assert all(content_of_first_choice(response) == "scripted" for response in third_batch)
+            assert upstream.calls(fleet.flip.handle.scenario_id) - after_second <= WORKERS, (
+                "the real answer is not cached"
+            )

tests/integration/caching/test_response_cache_empty_output.py

@@ -0,0 +1,813 @@
+"""The response cache never stores an answer with no output and never serves a stored one.
+
+Every cell scripts the owned upstream for one deployment, drives the rig proxy through the client a
+user would hold (OpenAI SDK, Anthropic SDK, raw httpx) and reads three things: what the caller got,
+how often the upstream was called for that deployment (an answer served from the cache never reaches
+it), and what landed in Redis or the spend log. An upstream answer carrying no choices, no output items
+or no content blocks reaches the upstream again on the next identical request, and the real answer
+that follows is the one the cache keeps. Answers with output, errors, ``no-cache`` requests, the
+``/cache/delete`` workaround and the other cached call types are unchanged.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import uuid
+from collections.abc import Callable, Iterator, Mapping
+from dataclasses import dataclass
+from functools import partial
+from hashlib import sha256
+from typing import Final, TypeAlias, TypeVar
+
+import httpx
+import pytest
+from anthropic import Anthropic, AsyncAnthropic
+from openai import AsyncOpenAI, OpenAI
+from pydantic import JsonValue
+
+from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value
+from tests.integration._support.database import read_rows
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    CHAT_STREAM,
+    CHAT_STREAM_WITHOUT_CHOICES,
+    EMBEDDING,
+    MESSAGE,
+    MESSAGE_STREAM,
+    MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS,
+    MESSAGES_MODEL,
+    RERANK,
+    RESPONSE,
+    RESPONSE_STREAM,
+    RESPONSE_STREAM_WITHOUT_OUTPUT,
+    SHORT_TTL_CACHE_CONTROL,
+    TEXT,
+    TEXT_MODEL,
+    TEXT_STREAM_WITHOUT_CHOICES,
+    TRANSCRIPT,
+    Upstream,
+    await_entry,
+    chat_body,
+    choices,
+    content_of_first_choice,
+    emptied,
+    flip_entry_to_empty,
+    json_response,
+    message_body,
+    prompt,
+    redis_store,
+    rescript,
+    response_id,
+    scripted,
+    text_of_first_choice,
+)
+
+MISSING: Final = object()
+EMPTY_STREAMS: Final = 6
+REPLAY_ATTEMPTS: Final = 8
+SETTLE_SECONDS: Final = 15
+T = TypeVar("T")
+_Answer: TypeAlias = tuple[frozenset[str], str, tuple[str, ...]]
+SPEND_SQL: Final = (
+    'SELECT request_id, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s ORDER BY "startTime", request_id'
+)
+
+
+def _proxy_root(gateway: Gateway) -> str:
+    return str(gateway.client.base_url).rstrip("/")
+
+
+def _openai(gateway: Gateway, key: str | None = None) -> OpenAI:
+    return OpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0)
+
+
+def _async_openai(gateway: Gateway, key: str | None = None) -> AsyncOpenAI:
+    return AsyncOpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0)
+
+
+def _anthropic(gateway: Gateway) -> Anthropic:
+    return Anthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0)
+
+
+def _async_anthropic(gateway: Gateway) -> AsyncAnthropic:
+    return AsyncAnthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0)
+
+
+def _chat(gateway: Gateway, body: Mapping[str, JsonValue], *, key: str | None = None) -> httpx.Response:
+    return gateway.request("POST", "/v1/chat/completions", body, key=key)
+
+
+def _spend_rows(key: str) -> list[dict[str, JsonValue]]:
+    return read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),))
+
+
+def _assert_prompt_reached_upstream(upstream: Upstream, identity: str, text: str, times: int) -> None:
+    received: Final = upstream.received(identity)
+    assert len(received) == times, f"{identity}: {len(received)} upstream calls, expected {times}"
+    assert all(text in json.dumps(item["body"]) for item in received), received
+
+
+def _served_without_upstream(upstream: Upstream, identity: str, call: Callable[[], T]) -> T | None:
+    before: Final = upstream.calls(identity)
+    answer: Final = call()
+    return None if upstream.calls(identity) > before else answer
+
+
+@dataclass(frozen=True, slots=True)
+class Streamed:
+    ids: frozenset[str]
+    text: str
+    kinds: tuple[str, ...]
+    reached_upstream: bool
+
+
+def _streams(stream: Callable[[], _Answer], upstream: Upstream, identity: str, attempts: int) -> Iterator[Streamed]:
+    """Stream up to ``attempts`` times, stopping after the first answer served without an upstream call."""
+    for _ in range(attempts):
+        before: Final = upstream.calls(identity)
+        ids, text, kinds = stream()
+        answer: Final = Streamed(ids, text, kinds, upstream.calls(identity) > before)
+        yield answer
+        if not answer.reached_upstream:
+            return
+
+
+def _never_replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str) -> tuple[Streamed, ...]:
+    """``EMPTY_STREAMS`` identical streams, every one reaching the upstream and carrying ids of its own."""
+    answers: Final = tuple(_streams(stream, upstream, identity, EMPTY_STREAMS))
+    assert len(answers) == EMPTY_STREAMS and all(answer.reached_upstream for answer in answers), (
+        f"a stream was replayed from the cache: {answers}"
+    )
+    distinct: Final = frozenset[str]().union(*(answer.ids for answer in answers))
+    assert sum(len(answer.ids) for answer in answers) == len(distinct), answers
+    return answers
+
+
+def _replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str, produced: frozenset[str]) -> Streamed:
+    """The first of up to ``REPLAY_ATTEMPTS`` identical streams served without an upstream call.
+
+    A streamed request can reach the upstream once per proxy worker before a replay is served: the proxy adds
+    ``stream_options`` to a stream before keying it only when its own router already resolves the model, and the
+    worker that did not take the ``/model/new`` call resolves it once its router syncs. The replay carries ids an
+    upstream answer produced.
+    """
+    answers: Final = tuple(_streams(stream, upstream, identity, REPLAY_ATTEMPTS))
+    replay: Final = answers[-1]
+    assert not replay.reached_upstream, (
+        f"no stream was replayed from the cache in {REPLAY_ATTEMPTS} attempts: {answers}"
+    )
+    seen: Final = produced.union(*(answer.ids for answer in answers[:-1]))
+    assert replay.ids <= seen, (replay.ids, seen)
+    return replay
+
+
+def test_chat_empty_choices_reach_upstream_again_and_the_real_answer_is_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _openai(gateway) as client:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        empty: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}])
+        assert empty.choices == [], empty.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1)
+        rescript(handle, json_response(CHAT))
+        real: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}])
+        assert real.id != empty.id and real.choices[0].message.content == "scripted", real.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2)
+        await_entry(store, real.id)
+        hit: Final = client.chat.completions.with_raw_response.create(
+            model=model, messages=[{"role": "user", "content": text}]
+        )
+        served: Final = hit.parse()
+        assert served.id == real.id and served.choices[0].message.content == "scripted", hit.text
+        assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_async_chat_spend_rows_record_the_empty_answer_and_the_refill_as_misses(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        key: Final = scenario.key(models=[model])
+
+        async def drive() -> tuple[str, str, str]:
+            async with _async_openai(gateway, key) as client:
+                messages: Final = [{"role": "user", "content": text}]
+                empty = await client.chat.completions.create(model=model, messages=messages)
+                assert empty.choices == [], empty.model_dump_json()
+                rescript(handle, json_response(CHAT))
+                real = await client.chat.completions.create(model=model, messages=messages)
+                assert real.choices[0].message.content == "scripted", real.model_dump_json()
+                await_entry(store, real.id)
+                served = await client.chat.completions.create(model=model, messages=messages)
+                return empty.id, real.id, served.id
+
+        empty_id, real_id, served_id = asyncio.run(drive())
+        assert served_id == real_id != empty_id
+        assert upstream.calls(handle.scenario_id) == 2
+        rows: Final = eventually(lambda: _spend_rows(key), lambda found: len(found) == 3, seconds=70)
+        by_id: Final = {string_value(row["request_id"]): row for row in rows}
+        assert set(by_id) >= {empty_id, real_id}, sorted(by_id)
+        assert by_id[empty_id]["cache_hit"] != "True" and by_id[real_id]["cache_hit"] != "True", rows
+        hit_row: Final = next(row for request_id, row in by_id.items() if request_id.startswith(f"{real_id}_cache_hit"))
+        assert hit_row["cache_hit"] == "True" and float(str(hit_row["spend"])) == 0, rows
+
+
+def test_text_completion_empty_choices_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(TEXT, "choices")))
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "prompt": prompt()}
+        empty: Final = gateway.request("POST", "/v1/completions", body)
+        assert choices(empty) == [], empty.text
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(TEXT))
+        real: Final = gateway.request("POST", "/v1/completions", body)
+        assert text_of_first_choice(real) == "scripted" and response_id(real) != response_id(empty), real.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(real))
+        hit: Final = gateway.request("POST", "/v1/completions", body)
+        assert response_id(hit) == response_id(real) and text_of_first_choice(hit) == "scripted", hit.text
+        assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_responses_empty_output_reaches_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(RESPONSE, "output")))
+        model: Final = scenario.model(api_base=handle.api_base())
+
+        async def drive() -> None:
+            async with _async_openai(gateway) as client:
+                empty = await client.responses.create(model=model, input=text)
+                assert empty.output == [], empty.model_dump_json()
+                assert upstream.calls(handle.scenario_id) == 1
+                rescript(handle, json_response(RESPONSE))
+                real = await client.responses.create(model=model, input=text)
+                assert real.output_text == "scripted", real.model_dump_json()
+                assert upstream.calls(handle.scenario_id) == 2
+                await_entry(store, real.output[0].id)
+                hit = await client.responses.with_raw_response.create(model=model, input=text)
+                served = hit.parse()
+                assert served.output[0].id == real.output[0].id and served.output_text == "scripted", hit.text
+                assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers)
+                assert upstream.calls(handle.scenario_id) == 2
+
+        asyncio.run(drive())
+
+
+def test_messages_empty_content_reaches_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _anthropic(gateway) as client:
+        handle: Final = scripted(scenario, json_response(emptied(MESSAGE, "content")))
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        messages: Final = [{"role": "user", "content": text}]
+        empty: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert empty.content == [], empty.model_dump_json()
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1)
+        rescript(handle, json_response(MESSAGE))
+        real: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert real.id != empty.id and real.content[0].type == "text", real.model_dump_json()
+        assert real.content[0].text == "scripted"
+        _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2)
+        await_entry(store, real.id)
+        served: Final = client.messages.create(model=model, max_tokens=16, messages=messages)
+        assert served.id == real.id and served.content[0].type == "text", served.model_dump_json()
+        assert served.content[0].text == "scripted"
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_chat_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL))
+        assert content_of_first_choice(filled) == "scripted", filled.text
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices")
+        request: Final = chat_body(model, text)
+        refilled: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS
+        )
+        assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}"
+        assert content_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS
+        )
+        assert content_of_first_choice(hit) == "scripted", hit.text
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_text_completion_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(TEXT))
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        short_lived: Final[dict[str, JsonValue]] = {"model": model, "prompt": text, "cache": SHORT_TTL_CACHE_CONTROL}
+        filled: Final = gateway.request("POST", "/v1/completions", short_lived)
+        assert text_of_first_choice(filled) == "scripted", filled.text
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices")
+        request: Final[dict[str, JsonValue]] = {"model": model, "prompt": text}
+        refilled: Final = eventually(
+            lambda: gateway.request("POST", "/v1/completions", request),
+            lambda answer: response_id(answer) != response_id(filled),
+            SETTLE_SECONDS,
+        )
+        assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}"
+        assert text_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: gateway.request("POST", "/v1/completions", request),
+            lambda answer: response_id(answer) == response_id(refilled),
+            SETTLE_SECONDS,
+        )
+        assert text_of_first_choice(hit) == "scripted", hit.text
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_responses_entry_with_empty_output_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _openai(gateway) as client:
+        handle: Final = scripted(scenario, json_response(RESPONSE))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = client.responses.create(model=model, input=text, extra_body={"cache": SHORT_TTL_CACHE_CONTROL})
+        assert filled.output_text == "scripted", filled.model_dump_json()
+        flip_entry_to_empty(store, await_entry(store, filled.output[0].id), "output")
+        refilled: Final = eventually(
+            lambda: client.responses.create(model=model, input=text),
+            lambda answer: not answer.output or answer.output[0].id != filled.output[0].id,
+            SETTLE_SECONDS,
+        )
+        assert refilled.output, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert refilled.output_text == "scripted", refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.output[0].id)
+        hit: Final = eventually(
+            lambda: client.responses.create(model=model, input=text),
+            lambda answer: bool(answer.output) and answer.output[0].id == refilled.output[0].id,
+            SETTLE_SECONDS,
+        )
+        assert hit.output_text == "scripted", hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def test_stale_messages_entry_with_empty_content_is_a_miss(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario, _anthropic(gateway) as client:
+        handle: Final = scripted(scenario, json_response(MESSAGE))
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        messages: Final = [{"role": "user", "content": text}]
+        filled: Final = client.messages.create(
+            model=model, max_tokens=16, messages=messages, extra_body={"cache": SHORT_TTL_CACHE_CONTROL}
+        )
+        assert filled.content[0].type == "text" and filled.content[0].text == "scripted", filled.model_dump_json()
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, filled.id), "content")
+        refilled: Final = eventually(
+            lambda: client.messages.create(model=model, max_tokens=16, messages=messages),
+            lambda answer: answer.id != filled.id,
+            SETTLE_SECONDS,
+        )
+        assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert refilled.content[0].type == "text" and refilled.content[0].text == "scripted", refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.id)
+        hit: Final = eventually(
+            lambda: client.messages.create(model=model, max_tokens=16, messages=messages),
+            lambda answer: answer.id == refilled.id,
+            SETTLE_SECONDS,
+        )
+        assert hit.content[0].type == "text" and hit.content[0].text == "scripted", hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def _streamed_chat(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The chunk ids, the streamed text and the chunk kinds, after the whole stream was read."""
+    with (
+        _openai(gateway) as client,
+        client.chat.completions.create(
+            model=model, messages=[{"role": "user", "content": text}], stream=True
+        ) as stream,
+    ):
+        chunks: Final = tuple(stream)
+    ids: Final = frozenset(chunk.id for chunk in chunks)
+    streamed: Final = "".join(
+        choice.delta.content or "" for chunk in chunks for choice in chunk.choices
+    )  # comprehension-ok: flattens chunk choices
+    return ids, streamed, tuple(chunk.object for chunk in chunks)
+
+
+def test_streamed_chat_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, CHAT_STREAM_WITHOUT_CHOICES)
+        model: Final = scenario.model(api_base=handle.api_base())
+        empty: Final = _never_replayed(partial(_streamed_chat, gateway, model, text), upstream, handle.scenario_id)
+        assert all(answer.text == "" for answer in empty), empty
+
+
+def _streamed_text(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The chunk ids, the streamed text and the chunk kinds of a text completion, after the whole stream was read."""
+    with _openai(gateway) as client, client.completions.create(model=model, prompt=text, stream=True) as stream:
+        chunks: Final = tuple(stream)
+    ids: Final = frozenset(chunk.id for chunk in chunks)
+    streamed: Final = "".join(
+        choice.text or "" for chunk in chunks for choice in chunk.choices
+    )  # comprehension-ok: flattens chunk choices
+    return ids, streamed, tuple(chunk.object for chunk in chunks)
+
+
+def test_streamed_text_completion_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, TEXT_STREAM_WITHOUT_CHOICES)
+        model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base())
+        empty: Final = _never_replayed(partial(_streamed_text, gateway, model, text), upstream, handle.scenario_id)
+        assert all(answer.text == "" for answer in empty), empty
+
+
+def test_streamed_chat_with_content_is_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, CHAT_STREAM)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_chat, gateway, model, text)
+        first_ids, first_text, _ = stream()
+        assert first_text == "streamed response" and len(first_ids) == 1, first_ids
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response", replay
+
+
+def _streamed_response(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The completed response's output item ids, its text and the event kinds, after the whole stream was read."""
+
+    async def read() -> _Answer:
+        async with (
+            _async_openai(gateway) as client,
+            await client.responses.create(model=model, input=text, stream=True) as stream,
+        ):
+            events: Final = [event async for event in stream]
+        completed: Final = tuple(event for event in events if event.type == "response.completed")
+        assert len(completed) == 1, [event.type for event in events]
+        ids: Final = frozenset(item.id for item in completed[0].response.output)
+        return ids, completed[0].response.output_text, tuple(event.type for event in events)
+
+    return asyncio.run(read())
+
+
+def test_streamed_responses_completing_without_output_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, RESPONSE_STREAM_WITHOUT_OUTPUT)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_response, gateway, model, text)
+        empty: Final = _never_replayed(stream, upstream, handle.scenario_id)
+        assert all(answer.ids == frozenset() and answer.text == "" for answer in empty), empty
+        rescript(handle, RESPONSE_STREAM)
+        real_ids, real_text, _ = stream()
+        assert real_text == "streamed response", real_ids
+        assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the empty stream was replayed"
+        await_entry(store, next(iter(real_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids)
+        assert replay.text == "streamed response", replay
+
+
+def test_streamed_responses_with_output_are_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, RESPONSE_STREAM)
+        model: Final = scenario.model(api_base=handle.api_base())
+        stream: Final = partial(_streamed_response, gateway, model, text)
+        first_ids, first_text, _ = stream()
+        assert first_text == "streamed response" and len(first_ids) == 1, first_ids
+        assert all(item.startswith(f"msg_{handle.scenario_id}-") for item in first_ids), first_ids
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response", replay
+
+
+def _streamed_message(gateway: Gateway, model: str, text: str) -> _Answer:
+    """The message id, the streamed text and the event kinds, after the whole stream was read."""
+
+    async def read() -> _Answer:
+        async with _async_anthropic(gateway) as client:
+            stream: Final = await client.messages.create(
+                model=model, max_tokens=16, messages=[{"role": "user", "content": text}], stream=True
+            )
+            events: Final = [event async for event in stream]
+        starts: Final = tuple(event for event in events if event.type == "message_start")
+        assert len(starts) == 1, [event.type for event in events]
+        deltas: Final = tuple(event for event in events if event.type == "content_block_delta")
+        streamed: Final = "".join(delta.delta.text for delta in deltas if delta.delta.type == "text_delta")
+        return frozenset({starts[0].message.id}), streamed, tuple(event.type for event in events)
+
+    return asyncio.run(read())
+
+
+def test_streamed_messages_without_content_blocks_reach_upstream_again(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS)
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        stream: Final = partial(_streamed_message, gateway, model, text)
+        empty: Final = _never_replayed(stream, upstream, handle.scenario_id)
+        assert all("content_block_start" not in answer.kinds and answer.text == "" for answer in empty), empty
+        rescript(handle, MESSAGE_STREAM)
+        real_ids, real_text, real_kinds = stream()
+        assert real_text == "streamed response" and "content_block_start" in real_kinds, real_kinds
+        assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the stream without content blocks was replayed"
+        await_entry(store, next(iter(real_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids)
+        assert replay.text == "streamed response" and replay.kinds == real_kinds, replay
+
+
+def test_streamed_messages_with_content_are_replayed_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, MESSAGE_STREAM)
+        model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base())
+        stream: Final = partial(_streamed_message, gateway, model, text)
+        first_ids, first_text, first_kinds = stream()
+        assert first_text == "streamed response" and "content_block_start" in first_kinds, first_kinds
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, next(iter(first_ids)))
+        replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids)
+        assert replay.text == "streamed response" and replay.kinds == first_kinds, replay
+
+
+def test_embeddings_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(EMBEDDING))
+        model: Final = scenario.model(model="openai/text-embedding-3-small", api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "input": [prompt()]}
+        first: Final = gateway.post("/v1/embeddings", body)
+        assert first["object"] == "list" and first["data"], first
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = eventually(
+            lambda: _served_without_upstream(
+                upstream, handle.scenario_id, lambda: gateway.post("/v1/embeddings", body)
+            ),
+            lambda answer: answer is not None,
+            SETTLE_SECONDS,
+        )
+        assert second is not None and second["data"] == first["data"], second
+
+
+@pytest.mark.parametrize(
+    "choice",
+    (
+        pytest.param(
+            {"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "content_filter"},
+            id="blocked-empty-content",
+        ),
+        pytest.param(
+            {
+                "index": 0,
+                "message": {
+                    "role": "assistant",
+                    "content": None,
+                    "tool_calls": [
+                        {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}}
+                    ],
+                },
+                "finish_reason": "tool_calls",
+            },
+            id="tool-calls-only",
+        ),
+    ),
+)
+def test_chat_answers_with_a_choice_but_no_text_are_still_cached(
+    gateway: Gateway, choice: dict[str, JsonValue]
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response({**CHAT, "choices": [choice]}))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt())
+        first: Final = _chat(gateway, request)
+        assert object_value(choices(first)[0])["finish_reason"] == choice["finish_reason"], first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, response_id(first))
+        second: Final = _chat(gateway, request)
+        assert response_id(second) == response_id(first) and choices(second) == choices(first), second.text
+        assert upstream.calls(handle.scenario_id) == 1
+
+
+def test_no_cache_requests_reach_upstream_every_time(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt(), cache={"no-cache": True})
+        ids: Final = tuple(response_id(_chat(gateway, request)) for _ in range(3))
+        assert len(set(ids)) == 3, ids
+        assert upstream.calls(handle.scenario_id) == 3
+
+
+@pytest.mark.parametrize("status", (500, 429))
+def test_upstream_errors_reach_the_caller_and_are_never_cached(gateway: Gateway, status: int) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(
+            scenario, json_response({"error": {"message": "Controlled provider failure", "type": "api_error"}}, status)
+        )
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt())
+        first: Final = _chat(gateway, request)
+        assert first.status_code == status and "Controlled provider failure" in first.text, first.text
+        after_first: Final = upstream.calls(handle.scenario_id)
+        assert after_first >= 1
+        second: Final = _chat(gateway, request)
+        assert second.status_code == status and "Controlled provider failure" in second.text, second.text
+        assert upstream.calls(handle.scenario_id) > after_first, "the error was served from the cache"
+
+
+def test_cache_delete_on_the_entry_key_makes_the_next_request_reach_upstream(gateway: Gateway) -> None:
+    """``/cache/delete`` clears the Redis entry; a worker's in-memory copy of it serves until the entry's ttl lapses."""
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    text: Final = prompt()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(api_base=handle.api_base())
+        filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL))
+        key: Final = await_entry(store, response_id(filled))
+        assert gateway.post("/cache/delete", {"keys": [key]}) == {"status": "success"}
+        assert store.get(key) is None
+        request: Final = chat_body(model, text)
+        refilled: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS
+        )
+        assert content_of_first_choice(refilled) == "scripted", refilled.text
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, response_id(refilled))
+        hit: Final = eventually(
+            lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS
+        )
+        assert hit.headers.get("x-litellm-cache-key") == key, dict(hit.headers)
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+def _shape(body: Mapping[str, JsonValue], field: str, value: object) -> dict[str, JsonValue]:
+    if value is MISSING:
+        return {name: item for name, item in body.items() if name != field}
+    assert isinstance(value, (str, int)) or value is None
+    return {**body, field: value}
+
+
+def _request_for(path: str, model: str, text: str) -> dict[str, JsonValue]:
+    if path == "/v1/responses":
+        return {"model": model, "input": text}
+    if path == "/v1/messages":
+        return message_body(model, text)
+    return chat_body(model, text)
+
+
+@pytest.mark.parametrize(
+    ("path", "body", "litellm_model", "field", "value"),
+    (
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", None, id="chat-choices-null"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", MISSING, id="chat-choices-missing"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", "x", id="chat-choices-string"),
+        pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", 5, id="chat-choices-int"),
+    ),
+)
+def test_malformed_upstream_answers_are_an_error_and_never_cached(
+    gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(_shape(body, field, value)))
+        control: Final = scripted(scenario, json_response(CHAT))
+        model: Final = scenario.model(model=litellm_model, api_base=handle.api_base())
+        control_model: Final = scenario.model(api_base=control.api_base())
+        key: Final = scenario.key(models=[model, control_model])
+        request: Final = _request_for(path, model, prompt())
+        first: Final = gateway.request("POST", path, request, key=key)
+        assert first.status_code == 500, first.text
+        assert "error" in first.text.lower(), first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = gateway.request("POST", path, request, key=key)
+        assert second.status_code == 500, second.text
+        assert upstream.calls(handle.scenario_id) == 2, "the malformed answer was served from the cache"
+        unrelated: Final = _chat(gateway, chat_body(control_model, prompt()), key=key)
+        assert content_of_first_choice(unrelated) == "scripted", unrelated.text
+
+
+@pytest.mark.parametrize(
+    ("path", "body", "litellm_model", "field", "value"),
+    (
+        pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", None, id="responses-output-null"),
+        pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", MISSING, id="responses-output-missing"),
+        pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", None, id="messages-content-null"),
+        pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", MISSING, id="messages-content-missing"),
+    ),
+)
+def test_answers_without_the_output_field_reach_upstream_again(
+    gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object
+) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(_shape(body, field, value)))
+        model: Final = scenario.model(model=litellm_model, api_base=handle.api_base())
+        request: Final = _request_for(path, model, prompt())
+        first: Final = gateway.request("POST", path, request)
+        assert first.status_code == 200, first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        second: Final = gateway.request("POST", path, request)
+        assert second.status_code == 200, second.text
+        assert upstream.calls(handle.scenario_id) == 2, "the answer without output was served from the cache"
+
+
+def test_per_request_ttl_on_an_empty_answer_does_not_pin_it(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices")))
+        model: Final = scenario.model(api_base=handle.api_base())
+        request: Final = chat_body(model, prompt(), cache={"ttl": 30})
+        empty: Final = _chat(gateway, request)
+        assert choices(empty) == [], empty.text
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(CHAT))
+        real: Final = _chat(gateway, request)
+        assert content_of_first_choice(real) == "scripted", real.text
+        assert upstream.calls(handle.scenario_id) == 2, "the empty answer was pinned for the request's ttl"
+        key: Final = await_entry(store, response_id(real))
+        assert 0 < store.ttl(key) <= 30, store.ttl(key)
+
+
+def test_rerank_results_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(RERANK))
+        model: Final = scenario.model(model="cohere/rerank-v4.0", api_base=handle.api_base())
+        body: Final[dict[str, JsonValue]] = {"model": model, "query": prompt(), "documents": ["first", "second"]}
+        first: Final = gateway.post("/v1/rerank", body)
+        assert string_value(first["id"]).startswith(f"rerank-{handle.scenario_id}"), first
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, string_value(first["id"]))
+        second: Final = gateway.post("/v1/rerank", body)
+        assert second["id"] == first["id"] and second["results"] == first["results"], second
+        assert upstream.calls(handle.scenario_id) == 1
+
+
+def test_transcriptions_are_still_served_from_the_cache(gateway: Gateway) -> None:
+    upstream: Final = Upstream(gateway.upstream_url)
+    store: Final = redis_store()
+    with gateway.scenario() as scenario:
+        handle: Final = scripted(scenario, json_response(TRANSCRIPT))
+        model: Final = scenario.model(model="openai/gpt-4o-mini-transcribe", api_base=handle.api_base())
+        audio: Final = (f"{uuid.uuid4().hex}.wav", b"RIFF" + uuid.uuid4().bytes, "audio/wav")
+        first: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio})
+        assert first.status_code == 200, first.text
+        transcript: Final = string_value(JSON_OBJECT.validate_json(first.content)["text"])
+        assert transcript.startswith(f"scripted {handle.scenario_id}-"), first.text
+        assert upstream.calls(handle.scenario_id) == 1
+        await_entry(store, transcript)
+        second: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio})
+        assert second.status_code == 200 and JSON_OBJECT.validate_json(second.content)["text"] == transcript, (
+            second.text
+        )
+        assert upstream.calls(handle.scenario_id) == 1

tests/integration/run.py

@@ -13,7 +13,7 @@ from typing import Final
 GROUPS: Final = MappingProxyType(
     {
         "management": ("management", "authorization", "configuration"),
-        "accounting": ("pricing", "spend"),
+        "accounting": ("pricing", "spend", "caching"),
         "database": ("database",),
         "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation"),
         "extensions": ("observability", "compatibility"),

tests/integration/sdk/test_response_cache_sync_sdk.py

@@ -0,0 +1,125 @@
+"""The sync SDK paths of the response cache, which the proxy never reaches.
+
+``litellm.completion`` and ``litellm.text_completion`` read and write the Redis cache in process. An upstream
+answer with no choices is never stored, a stored entry whose choices were emptied is a miss, and the real answer
+the refill brings back is the one served afterwards.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Iterator
+from typing import Final
+
+import pytest
+
+import litellm
+from litellm.caching.caching import Cache
+from litellm.types.caching import LiteLLMCacheType
+from litellm.types.utils import ModelResponse, TextCompletionResponse
+from tests.integration.caching.response_cache_case import (
+    CHAT,
+    CHAT_MODEL,
+    PROVIDER_KEY,
+    TEXT,
+    TEXT_MODEL,
+    Upstream,
+    await_entry,
+    emptied,
+    flip_entry_to_empty,
+    json_response,
+    prompt,
+    redis_store,
+    rescript,
+    scripted_in_process,
+)
+
+
+@pytest.fixture
+def sdk_cache(monkeypatch: pytest.MonkeyPatch) -> Iterator[Cache]:
+    cache: Final = Cache(type=LiteLLMCacheType.REDIS, host=os.environ["REDIS_HOST"], port=os.environ["REDIS_PORT"])
+    monkeypatch.setattr(litellm, "cache", cache)
+    yield cache
+    monkeypatch.setattr(litellm, "cache", None)
+
+
+def _completion(api_base: str, text: str) -> ModelResponse:
+    response: Final = litellm.completion(
+        model=CHAT_MODEL, messages=[{"role": "user", "content": text}], api_base=api_base, api_key=PROVIDER_KEY
+    )
+    assert isinstance(response, ModelResponse), type(response)
+    return response
+
+
+def _text_completion(api_base: str, text: str) -> TextCompletionResponse:
+    response: Final = litellm.text_completion(model=TEXT_MODEL, prompt=text, api_base=api_base, api_key=PROVIDER_KEY)
+    assert isinstance(response, TextCompletionResponse), type(response)
+    return response
+
+
+def _content(response: ModelResponse) -> object:
+    assert len(response.choices) == 1, response.model_dump_json()
+    return response.choices[0].model_dump()["message"]["content"]
+
+
+def _text(response: TextCompletionResponse) -> object:
+    assert len(response.choices) == 1, response.model_dump_json()
+    return response.choices[0].model_dump()["text"]
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_completion_with_empty_choices_reaches_upstream_again_and_the_real_answer_is_cached() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(emptied(CHAT, "choices"))) as handle:
+        empty: Final = _completion(handle.api_base(), text)
+        assert empty.choices == [], empty.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(CHAT))
+        refilled: Final = _completion(handle.api_base(), text)
+        assert _content(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}"
+        assert refilled.id != empty.id
+        assert upstream.calls(handle.scenario_id) == 2
+        with redis_store() as store:
+            await_entry(store, refilled.id)
+        hit: Final = _completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert _content(hit) == "scripted"
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_completion_treats_a_stale_entry_with_empty_choices_as_a_miss() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(CHAT)) as handle, redis_store() as store:
+        first: Final = _completion(handle.api_base(), text)
+        assert _content(first) == "scripted", first.model_dump_json()
+        stale_id: Final = flip_entry_to_empty(store, await_entry(store, first.id), "choices")
+        refilled: Final = _completion(handle.api_base(), text)
+        assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}"
+        assert _content(refilled) == "scripted" and refilled.id != first.id, refilled.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+        await_entry(store, refilled.id)
+        hit: Final = _completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2
+
+
+@pytest.mark.usefixtures("sdk_cache")
+def test_sync_text_completion_with_empty_choices_reaches_upstream_again() -> None:
+    upstream: Final = Upstream()
+    text: Final = prompt()
+    with scripted_in_process(json_response(emptied(TEXT, "choices"))) as handle:
+        empty: Final = _text_completion(handle.api_base(), text)
+        assert empty.choices == [], empty.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 1
+        rescript(handle, json_response(TEXT))
+        refilled: Final = _text_completion(handle.api_base(), text)
+        assert _text(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}"
+        assert upstream.calls(handle.scenario_id) == 2
+        with redis_store() as store:
+            await_entry(store, refilled.id)
+        hit: Final = _text_completion(handle.api_base(), text)
+        assert hit.id == refilled.id, hit.model_dump_json()
+        assert upstream.calls(handle.scenario_id) == 2

tests/llm_translation/test_llm_response_utils/test_get_headers.py

@@ -8,6 +8,7 @@ import pytest
 from litellm.litellm_core_utils.llm_response_utils.get_headers import (
     get_response_headers,
     _get_llm_provider_headers,
+    get_provider_request_id,
 )
 
 
@@ -72,3 +73,17 @@ def test_get_llm_provider_headers():
     }
     result = _get_llm_provider_headers(input_headers)
     assert result == expected_output, "Unexpected output for _get_llm_provider_headers"
+
+
+@pytest.mark.parametrize("header", ("request-id", "Request-Id", "x-request-id", "llm_provider-request-id"))
+def test_native_clients_receive_the_provider_request_id(header: str) -> None:
+    result = get_response_headers({header: "req_test", "unrelated": "value"})
+    assert result["request-id"] == "req_test"
+    assert get_provider_request_id(result) == "req_test"
+    assert result["llm_provider-unrelated"] == "value"
+    assert "unrelated" not in result
+
+
+@pytest.mark.parametrize("headers", (None, {}, {"request-id": ""}, {"request-id": 42}))
+def test_invalid_provider_request_ids_remain_absent(headers: object) -> None:
+    assert get_provider_request_id(headers) is None

tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py

@@ -163,6 +163,7 @@ def test_success_row_mapping():
     assert row["request_id"] == "chatcmpl-abc123"
     assert row["response_id"] == "chatcmpl-abc123"
     assert row["litellm_call_id"] == "gateway-call"
+    assert row["provider_request_id"] == ""
     assert row["spend"] == 0.00042
     assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30)
     assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7)
@@ -186,6 +187,18 @@ def test_success_row_mapping():
     assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key"
 
 
+@pytest.mark.parametrize("header_source", ("response_headers", "additional_headers"))
+def test_provider_request_id_stays_separate_from_message_and_gateway_ids(header_source: str) -> None:
+    headers: Final = {"request-id": "req_native"}
+    hidden: Final = {"additional_headers": headers} if header_source == "additional_headers" else {}
+    kwargs: Final = {"response_cost": 0.00042, "response_headers": headers if header_source == "response_headers" else None}
+    payload: Final = cast(StandardLoggingPayload, _payload(id="msg_native", hidden_params=hidden))
+    row: Final = spend_log_row_from_payload(payload, kwargs)
+    assert (row["provider_request_id"], row["response_id"], row["litellm_call_id"]) == (
+        "req_native", "msg_native", "gateway-call"
+    )
+
+
 @pytest.mark.parametrize("status", ("success", "failure"))
 @pytest.mark.asyncio
 async def test_custom_request_metadata_is_redacted_before_clickhouse_logging(

tests/unit/caching/test_caching_handler.py

@@ -23,17 +23,23 @@ from litellm.caching.caching_handler import (
     _is_chat_completion_cached_dict,
     _should_defer_streaming_cache_hit_callbacks,
 )
+from litellm.caching import DualCache, InMemoryCache
 from litellm.caching.caching import LiteLLMCacheType
+from litellm.types.caching import CACHED_STREAM_EVENTS_KEY
 from litellm.types.utils import CallTypes
 from litellm.types.rerank import RerankResponse
 from litellm.types.utils import (
+    Delta,
     ModelResponse,
+    ModelResponseStream,
+    StreamingChoices,
     EmbeddingResponse,
     TextCompletionResponse,
     TranscriptionResponse,
     Embedding,
 )
 from litellm.types.llms.openai import ResponsesAPIResponse
+from collections.abc import Awaitable, Callable
 from datetime import timedelta, datetime
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging
 from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
@@ -2313,3 +2319,355 @@ async def test_response_cache_lookup_and_write_declare_the_llm_response_target(m
 
     assert seen == {"get": "llm_response", "set": "llm_response"}
     assert current_service_target() is None
+
+
+def _completion_logging_obj(call_type: str) -> LiteLLMLogging:
+    return LiteLLMLogging(
+        litellm_call_id=str(uuid.uuid4()),
+        call_type=call_type,
+        model="gpt-3.5-turbo",
+        messages=[],
+        function_id=str(uuid.uuid4()),
+        stream=False,
+        start_time=_FIXED_START,
+    )
+
+
+_FIXED_START = datetime(2026, 1, 1)
+
+
+def _unique_messages() -> list[dict[str, str]]:
+    return [{"role": "user", "content": f"no choices {uuid.uuid4()}"}]
+
+
+async def aanthropic_messages(**kwargs: object) -> None:
+    return None
+
+
+def _responses_api_response_without_output() -> ResponsesAPIResponse:
+    return ResponsesAPIResponse(
+        id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response", output=[]
+    )
+
+
+def _anthropic_message_without_content() -> dict[str, object]:
+    return {
+        "id": f"msg_{uuid.uuid4()}",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-sonnet-5",
+        "content": [],
+    }
+
+
+def _responses_api_response_missing_output() -> ResponsesAPIResponse:
+    return ResponsesAPIResponse.model_construct(
+        id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response"
+    )
+
+
+def _anthropic_message_with_null_content() -> dict[str, object]:
+    return {
+        "id": f"msg_{uuid.uuid4()}",
+        "type": "message",
+        "role": "assistant",
+        "model": "claude-sonnet-5",
+        "content": None,
+    }
+
+
+def _anthropic_message_missing_content() -> dict[str, object]:
+    return {"id": f"msg_{uuid.uuid4()}", "type": "message", "role": "assistant", "model": "claude-sonnet-5"}
+
+
+def _anthropic_stream_without_content_blocks() -> dict[str, object]:
+    return {
+        CACHED_STREAM_EVENTS_KEY: [
+            'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_1", "content": []}}\n\n',
+            'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}}\n\n',
+            'event: message_stop\ndata: {"type": "message_stop"}\n\n',
+        ]
+    }
+
+
+EmptyResult = ModelResponse | TextCompletionResponse | ResponsesAPIResponse | dict[str, object]
+EmptyResponseCase = tuple[EmptyResult, Callable[..., Awaitable[object]], str, dict[str, object]]
+
+
+def _empty_response_cases() -> list[EmptyResponseCase]:
+    return [
+        (litellm.ModelResponse(choices=[]), litellm.acompletion, CallTypes.acompletion.value, {"messages": _unique_messages()}),
+        (
+            litellm.TextCompletionResponse(choices=[]),
+            litellm.atext_completion,
+            CallTypes.atext_completion.value,
+            {"prompt": str(uuid.uuid4())},
+        ),
+        (
+            _responses_api_response_without_output(),
+            aresponses,
+            CallTypes.aresponses.value,
+            {"input": str(uuid.uuid4())},
+        ),
+        (
+            _responses_api_response_missing_output(),
+            aresponses,
+            CallTypes.aresponses.value,
+            {"input": str(uuid.uuid4())},
+        ),
+        (
+            _anthropic_message_without_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_message_with_null_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_message_missing_content(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16},
+        ),
+        (
+            _anthropic_stream_without_content_blocks(),
+            aanthropic_messages,
+            CallTypes.aanthropic_messages.value,
+            {"messages": _unique_messages(), "max_tokens": 16, "stream": True},
+        ),
+    ]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases())
+async def test_async_set_cache_skips_response_without_output(
+    empty_result: EmptyResult,
+    original_function: Callable[..., Awaitable[object]],
+    call_type: str,
+    kwargs: dict[str, object],
+):
+    setup_cache()
+    handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START)
+
+    await handler.async_set_cache(result=empty_result, original_function=original_function, kwargs=kwargs)
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert await litellm.cache.async_get_cache(**kwargs) is None
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=original_function,
+        logging_obj=_completion_logging_obj(call_type),
+        start_time=_FIXED_START,
+        call_type=call_type,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases())
+async def test_async_get_cache_treats_stored_response_without_output_as_miss(
+    empty_result: EmptyResult,
+    original_function: Callable[..., Awaitable[object]],
+    call_type: str,
+    kwargs: dict[str, object],
+):
+    setup_cache()
+    stored = empty_result.model_dump_json() if hasattr(empty_result, "model_dump_json") else empty_result
+    await litellm.cache.async_add_cache(stored, **kwargs)
+    assert await litellm.cache.async_get_cache(**kwargs) is not None
+
+    handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START)
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=original_function,
+        logging_obj=_completion_logging_obj(call_type),
+        start_time=_FIXED_START,
+        call_type=call_type,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+@pytest.mark.asyncio
+async def test_async_get_cache_heals_stored_completion_without_choices():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+    poisoned = litellm.ModelResponse(
+        choices=[], usage=litellm.Usage(prompt_tokens=7, completion_tokens=0, total_tokens=7)
+    )
+    await litellm.cache.async_add_cache(poisoned.model_dump_json(), **kwargs)
+    assert await litellm.cache.async_get_cache(**kwargs) is not None
+
+    async def lookup():
+        return await handler._async_get_cache(
+            model="gpt-3.5-turbo",
+            original_function=litellm.acompletion,
+            logging_obj=_completion_logging_obj(CallTypes.acompletion.value),
+            start_time=_FIXED_START,
+            call_type=CallTypes.acompletion.value,
+            kwargs=kwargs,
+        )
+
+    assert (await lookup()).cached_result is None
+
+    await handler.async_set_cache(result=chat_completion_response, original_function=litellm.acompletion, kwargs=kwargs)
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    healed = (await lookup()).cached_result
+    assert healed is not None
+    assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content
+
+
+def test_sync_set_cache_skips_response_without_output():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+
+    handler.sync_set_cache(result=litellm.ModelResponse(choices=[]), kwargs=kwargs)
+
+    assert litellm.cache.get_cache(**kwargs) is None
+    lookup = handler._sync_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=completion,
+        logging_obj=_completion_logging_obj(CallTypes.completion.value),
+        start_time=_FIXED_START,
+        call_type=CallTypes.completion.value,
+        kwargs=kwargs,
+    )
+    assert lookup.cached_result is None
+
+
+def test_sync_get_cache_heals_stored_completion_without_choices():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START)
+    kwargs = {"messages": _unique_messages()}
+    litellm.cache.add_cache(litellm.ModelResponse(choices=[]).model_dump_json(), **kwargs)
+    assert litellm.cache.get_cache(**kwargs) is not None
+
+    def lookup():
+        return handler._sync_get_cache(
+            model="gpt-3.5-turbo",
+            original_function=completion,
+            logging_obj=_completion_logging_obj(CallTypes.completion.value),
+            start_time=_FIXED_START,
+            call_type=CallTypes.completion.value,
+            kwargs=kwargs,
+        )
+
+    assert lookup().cached_result is None
+
+    handler.sync_set_cache(result=chat_completion_response, kwargs=kwargs)
+    healed = lookup().cached_result
+    assert healed is not None
+    assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content
+
+
+@pytest.mark.asyncio
+async def test_acompletion_after_a_response_without_choices_calls_the_provider_again():
+    setup_cache()
+    messages = _unique_messages()
+
+    first = await litellm.acompletion(
+        model="gpt-4o", messages=messages, mock_response=litellm.ModelResponse(choices=[]), caching=True
+    )
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+    assert first.choices == []
+
+    second = await litellm.acompletion(model="gpt-4o", messages=messages, mock_response="hi", caching=True)
+    assert second.choices[0].message.content == "hi"
+    assert second._hidden_params.get("cache_hit") is not True
+
+
+def _stream_chunk(choices: list[StreamingChoices]) -> ModelResponseStream:
+    return ModelResponseStream(
+        id="chatcmpl-stream", created=0, model="gpt-4o", object="chat.completion.chunk", choices=choices
+    )
+
+
+def _closing_chunk_without_output() -> ModelResponseStream:
+    return _stream_chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())])
+
+
+def _cached_content(cached: object) -> object:
+    assert cached is not None
+    stored = cached if isinstance(cached, dict) else json.loads(cached)
+    return stored["choices"][0]["message"]["content"]
+
+
+def _content_chunk(content: str) -> ModelResponseStream:
+    return _stream_chunk([StreamingChoices(index=0, delta=Delta(content=content, role="assistant"))])
+
+
+@pytest.mark.asyncio
+async def test_async_streamed_answer_without_output_is_never_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    await handler._add_streaming_response_to_cache(_closing_chunk_without_output())
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert litellm.cache.get_cache(**kwargs) is None
+
+
+@pytest.mark.asyncio
+async def test_async_streamed_answer_with_content_is_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    await handler._add_streaming_response_to_cache(_content_chunk("hi"))
+    await handler._add_streaming_response_to_cache(_closing_chunk_without_output())
+    await asyncio.gather(*_PENDING_CACHE_WRITES)
+
+    assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi"
+
+
+def test_sync_streamed_answer_without_output_is_never_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output())
+
+    assert litellm.cache.get_cache(**kwargs) is None
+
+
+def test_sync_streamed_answer_with_content_is_cached():
+    setup_cache()
+    kwargs = {"model": "gpt-4o", "messages": _unique_messages()}
+    handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START)
+
+    handler._sync_add_streaming_response_to_cache(_content_chunk("hi"))
+    handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output())
+
+    assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi"
+
+
+@pytest.mark.asyncio
+async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_without_output():
+    setup_cache()
+    handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START)
+    handler.dual_cache = DualCache(in_memory_cache=InMemoryCache())
+    kwargs = {"messages": _unique_messages()}
+    key = litellm.cache.get_cache_key(**kwargs)
+    poisoned = litellm.ModelResponse(choices=[]).model_dump_json()
+    await handler.dual_cache.async_set_cache(key, {"timestamp": _FIXED_START.timestamp(), "response": poisoned})
+    assert await handler.dual_cache.async_get_cache(key) is not None
+
+    lookup = await handler._async_get_cache(
+        model="gpt-3.5-turbo",
+        original_function=litellm.acompletion,
+        logging_obj=_completion_logging_obj(CallTypes.acompletion.value),
+        start_time=_FIXED_START,
+        call_type=CallTypes.acompletion.value,
+        kwargs=kwargs,
+    )
+
+    assert lookup.cached_result is None
+    assert await handler.dual_cache.async_get_cache(key) is None

tests/unit/integrations/otel/test_otel_v2_destinations.py

@@ -3081,6 +3081,126 @@ class TestBackendEndpointParity:
         assert destination.endpoint == "https://trace.wandb.ai/otel/v1/traces"
 
 
+class TestArizeOtlpProtocol:
+    @staticmethod
+    def _arize(monkeypatch, endpoint_env, **env):
+        monkeypatch.delenv("ARIZE_ENDPOINT", raising=False)
+        monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False)
+        for name, value in env.items():
+            monkeypatch.setenv(name, value)
+        return destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k", **endpoint_env})
+
+    def test_http_transport_rewrites_the_grpc_endpoint_to_the_traces_path(self, monkeypatch):
+        destination = self._arize(
+            monkeypatch,
+            {"arize_otlp_protocol": "http/protobuf"},
+            ARIZE_ENDPOINT="https://arize.internal.example/v1",
+        )
+
+        assert destination.protocol == "otlp_http"
+        assert destination.endpoint == "https://arize.internal.example/v1/traces"
+
+    def test_http_transport_prefers_the_http_endpoint(self, monkeypatch):
+        destination = self._arize(
+            monkeypatch,
+            {"arize_otlp_protocol": "http/protobuf"},
+            ARIZE_ENDPOINT="https://arize.internal.example/v1",
+            ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces",
+        )
+
+        assert destination.protocol == "otlp_http"
+        assert destination.endpoint == "https://http.arize.internal.example/v1/traces"
+
+    def test_http_transport_falls_back_to_the_arize_cloud_http_endpoint(self, monkeypatch):
+        destination = self._arize(monkeypatch, {"arize_otlp_protocol": "http/protobuf"})
+
+        assert destination.protocol == "otlp_http"
+        assert destination.endpoint == "https://otlp.arize.com/v1/traces"
+
+    def test_grpc_transport_accepts_the_http_endpoint_as_the_grpc_target(self, monkeypatch):
+        destination = self._arize(
+            monkeypatch,
+            {"arize_otlp_protocol": "grpc"},
+            ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces",
+        )
+
+        assert destination.protocol == "otlp_grpc"
+        assert destination.endpoint == "https://http.arize.internal.example/v1/traces"
+
+    def test_no_transport_var_keeps_the_grpc_env_default(self, monkeypatch):
+        destination = self._arize(monkeypatch, {}, ARIZE_ENDPOINT="https://arize.internal.example")
+
+        assert destination.protocol == "otlp_grpc"
+        assert destination.endpoint == "https://arize.internal.example"
+
+    def test_the_http_destination_builds_an_http_exporter_on_the_traces_path(self, monkeypatch):
+        from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
+            OTLPSpanExporter as HTTPSpanExporter,
+        )
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+        from litellm.integrations.otel.plumbing.providers import _destination_processor
+
+        destination = self._arize(
+            monkeypatch,
+            {"arize_otlp_protocol": "http/protobuf"},
+            ARIZE_ENDPOINT="https://arize.internal.example/v1",
+        )
+        processor = _destination_processor(destination)
+        try:
+            assert isinstance(processor, BatchSpanProcessor)
+            assert isinstance(processor.span_exporter, HTTPSpanExporter)
+            assert processor.span_exporter._endpoint == "https://arize.internal.example/v1/traces"
+        finally:
+            processor.shutdown()
+
+    def test_the_grpc_destination_builds_a_grpc_exporter(self, monkeypatch):
+        from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import (
+            OTLPSpanExporter as GRPCSpanExporter,
+        )
+        from opentelemetry.sdk.trace.export import BatchSpanProcessor
+
+        from litellm.integrations.otel.plumbing.providers import _destination_processor
+
+        destination = self._arize(
+            monkeypatch,
+            {"arize_otlp_protocol": "grpc"},
+            ARIZE_ENDPOINT="https://arize.internal.example",
+        )
+        processor = _destination_processor(destination)
+        try:
+            assert isinstance(processor, BatchSpanProcessor)
+            assert isinstance(processor.span_exporter, GRPCSpanExporter)
+        finally:
+            processor.shutdown()
+
+    def test_a_team_metadata_protocol_reaches_the_destination(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1")
+        monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False)
+        is_otel_v2_enabled.cache_clear()
+        auth = UserAPIKeyAuth(
+            team_metadata={
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "success",
+                        "callback_vars": {
+                            "arize_space_id": "s",
+                            "arize_api_key": "k",
+                            "arize_otlp_protocol": "http/protobuf",
+                        },
+                    }
+                ]
+            }
+        )
+
+        destinations = resolve_tenant_otel_destinations(auth)
+
+        assert [d.protocol for d in destinations] == ["otlp_http"]
+        assert destinations[0].endpoint == "https://arize.internal.example/v1/traces"
+
+
 class TestIncompleteCredentials:
     """Half a credential set builds a non-empty but unusable header dict. Accepting it
     would suppress the operator's exporter and send the trace where it cannot land."""

tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py

@@ -13,7 +13,13 @@ from litellm.integrations.websearch_interception.handler import (
     WebSearchInterceptionLogger,
 )
 from litellm.llms.base_llm.search.transformation import SearchResponse
-from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth
+from litellm.proxy._types import (
+    LiteLLM_ObjectPermissionTable,
+    LiteLLM_TeamTable,
+    LitellmUserRoles,
+    ProxyException,
+    UserAPIKeyAuth,
+)
 from litellm.types.utils import LlmProviders
 
 
@@ -899,3 +905,184 @@ async def test_pre_request_hook_syncs_forced_tool_choice():
         "type": "tool",
         "name": LITELLM_WEB_SEARCH_TOOL_NAME,
     }
+
+
+def _single_search_tool_router(search_tool_name):
+    router = MagicMock()
+    router.search_tools = [
+        {
+            "search_tool_name": search_tool_name,
+            "litellm_params": {"search_provider": "tavily", "api_key": "fake-ui-key"},
+        }
+    ]
+    return router
+
+
+def _virtual_key(
+    team_id: str | None = None, key_search_tools: list[str] | None = None, api_key: str = "sk-caller"
+) -> UserAPIKeyAuth:
+    token = UserAPIKeyAuth(
+        api_key=api_key,
+        user_id="user-1",
+        team_id=team_id,
+        object_permission_id=None if key_search_tools is None else "op-key",
+        object_permission=(
+            None
+            if key_search_tools is None
+            else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools)
+        ),
+    )
+    token.via_virtual_key = True
+    return token
+
+
+@pytest.mark.parametrize(
+    "general_settings, key_search_tools, team_search_tools, expect_search",
+    [
+        ({}, None, [], True),
+        ({"search_tool_deny_by_default": True}, ["team-search"], [], False),
+        ({"search_tool_deny_by_default": True}, ["team-search"], None, False),
+        ({"search_tool_deny_by_default": True}, [], ["team-search"], False),
+        ({"search_tool_deny_by_default": True}, ["team-search"], ["team-search"], True),
+    ],
+)
+@pytest.mark.asyncio
+async def test_execute_search_team_key_follows_search_tool_deny_by_default(
+    monkeypatch: pytest.MonkeyPatch,
+    general_settings: dict[str, bool],
+    key_search_tools: list[str] | None,
+    team_search_tools: list[str] | None,
+    expect_search: bool,
+):
+    import litellm
+    from litellm.proxy import proxy_server
+
+    logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="team-search")
+    mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[]))
+    team_object = LiteLLM_TeamTable(
+        team_id="team-1",
+        object_permission_id=None if team_search_tools is None else "op-team",
+        object_permission=(
+            None
+            if team_search_tools is None
+            else LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=team_search_tools)
+        ),
+    )
+
+    monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("team-search"))
+    monkeypatch.setattr(proxy_server, "general_settings", general_settings)
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(litellm, "asearch", mock_asearch)
+    monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", AsyncMock(return_value=team_object))
+    kwargs = {"metadata": {"user_api_key_auth": _virtual_key("team-1", key_search_tools)}}
+
+    if expect_search:
+        await logger._execute_search("what is litellm", kwargs=kwargs)
+        mock_asearch.assert_awaited_once()
+    else:
+        with pytest.raises(ProxyException) as exc_info:
+            await logger._execute_search("what is litellm", kwargs=kwargs)
+        assert exc_info.value.code == "403"
+        mock_asearch.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_execute_search_key_without_grant_is_denied_under_search_tool_deny_by_default(
+    monkeypatch: pytest.MonkeyPatch,
+):
+    import litellm
+    from litellm.proxy import proxy_server
+
+    logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="any-search")
+    mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[]))
+
+    monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("any-search"))
+    monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True})
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(litellm, "asearch", mock_asearch)
+
+    with pytest.raises(ProxyException) as exc_info:
+        await logger._execute_search("what is litellm", kwargs={"metadata": {"user_api_key_auth": _virtual_key()}})
+
+    assert exc_info.value.code == "403"
+    mock_asearch.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+    "general_settings, caller, expect_search",
+    [
+        ({}, _virtual_key(), True),
+        ({"search_tool_deny_by_default": False}, _virtual_key(), True),
+        ({"search_tool_deny_by_default": True}, _virtual_key(key_search_tools=["any"]), False),
+        (
+            {"search_tool_deny_by_default": True},
+            UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
+            False,
+        ),
+        ({"search_tool_deny_by_default": True}, _virtual_key(api_key="litellm_proxy_master_key"), True),
+        ({"search_tool_deny_by_default": True}, None, True),
+    ],
+    ids=["omitted", "false", "virtual-key", "proxy-admin-not-exempt", "master-key-exempt", "sdk-without-proxy-auth"],
+)
+@pytest.mark.asyncio
+async def test_execute_search_unregistered_fallback_follows_search_tool_deny_by_default(
+    monkeypatch: pytest.MonkeyPatch,
+    general_settings: dict[str, bool],
+    caller: UserAPIKeyAuth | None,
+    expect_search: bool,
+):
+    import litellm
+    from litellm.proxy import proxy_server
+
+    logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"])
+    router = MagicMock()
+    router.search_tools = []
+    mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[]))
+
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "general_settings", general_settings)
+    monkeypatch.setattr(litellm, "asearch", mock_asearch)
+    kwargs = {"metadata": {} if caller is None else {"user_api_key_auth": caller}}
+
+    if expect_search:
+        await logger._execute_search("what is litellm", kwargs=kwargs)
+        assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity"
+    else:
+        with pytest.raises(ProxyException) as exc_info:
+            await logger._execute_search("what is litellm", kwargs=kwargs)
+        assert exc_info.value.code == "403"
+        mock_asearch.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+    "general_settings, expect_search",
+    [({}, True), ({"search_tool_deny_by_default": True}, False)],
+)
+@pytest.mark.asyncio
+async def test_execute_search_registered_tool_without_provider_follows_unregistered_fallback_policy(
+    monkeypatch: pytest.MonkeyPatch,
+    general_settings: dict[str, bool],
+    expect_search: bool,
+):
+    import litellm
+    from litellm.proxy import proxy_server
+
+    logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="no-provider")
+    router = MagicMock()
+    router.search_tools = [{"search_tool_name": "no-provider", "litellm_params": {}}]
+    mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[]))
+
+    monkeypatch.setattr(proxy_server, "llm_router", router)
+    monkeypatch.setattr(proxy_server, "general_settings", general_settings)
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(litellm, "asearch", mock_asearch)
+    kwargs = {"metadata": {"user_api_key_auth": _virtual_key(key_search_tools=["no-provider"])}}
+
+    if expect_search:
+        await logger._execute_search("what is litellm", kwargs=kwargs)
+        assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity"
+    else:
+        with pytest.raises(ProxyException) as exc_info:
+            await logger._execute_search("what is litellm", kwargs=kwargs)
+        assert exc_info.value.code == "403"
+        mock_asearch.assert_not_awaited()

tests/unit/litellm_core_utils/test_core_helpers.py

@@ -1,6 +1,7 @@
 """Tests for litellm_core_utils.core_helpers module."""
 
 import logging
+from typing import Final
 
 import httpx
 import pytest
@@ -16,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import (
     get_provider_response_headers_from_hidden_params,
     map_finish_reason,
     normalize_drop_params,
+    process_response_headers,
     reconstruct_model_name,
     redact_nested_match_and_regex_keys,
     set_provider_response_headers_in_hidden_params,
@@ -25,6 +27,14 @@ from litellm.proxy._types import UserAPIKeyAuth
 from litellm.types.utils import ImageResponse, TranscriptionResponse
 
 
+@pytest.mark.parametrize("header", ("request-id", "x-request-id", "llm_provider-request-id"))
+def test_native_request_id_survives_stream_header_processing(header: str) -> None:
+    processed: Final = process_response_headers(httpx.Headers({header: "req_native"}))
+
+    assert processed["request-id"] == "req_native"
+    assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native"
+
+
 class TestBudgetReservationBinding:
     """The request-end release skips a reservation a cost callback has claimed, so the claim
     must land on the one dict auth stamped, through whichever metadata field or auth object

tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py

@@ -160,6 +160,26 @@ async def test_streaming_cache_is_not_shared_with_non_streaming(local_cache, req
     assert non_streaming["content"][0]["text"] == "ALPHA"
 
 
+@pytest.mark.asyncio
+async def test_stream_without_content_blocks_is_not_cached(local_cache, request_kwargs, monkeypatch):
+    empty_events = [STREAM_EVENTS[0], STREAM_EVENTS[4], STREAM_EVENTS[5]]
+    fake_handler = _CountingHandler(
+        [_byte_stream(empty_events), _byte_stream(STREAM_EVENTS), _byte_stream([b"event: never_used\n\n"])]
+    )
+    monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler)
+
+    empty = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+    await asyncio.sleep(0)
+    refilled = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+    await asyncio.sleep(0)
+    replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True))
+
+    assert empty == empty_events
+    assert refilled == STREAM_EVENTS
+    assert replayed == STREAM_EVENTS
+    assert len(fake_handler.calls) == 2
+
+
 @pytest.mark.asyncio
 async def test_failed_stream_is_not_cached(local_cache, request_kwargs, monkeypatch):
     error_events = STREAM_EVENTS[:3] + [

tests/unit/llms/custom_httpx/test_llm_http_handler.py

@@ -895,6 +895,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp
     additional_headers = result._hidden_params["additional_headers"]
     assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123"
     assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def"
+    assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers)
 
     collected = b"".join([chunk async for chunk in result])
     assert b"message_start" in collected
@@ -1601,8 +1602,7 @@ async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeyp
     expected_response = {"id": "msg_1", "content": []}
     mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response)
 
-    ok_response = Mock()
-    ok_response.raise_for_status = Mock(return_value=None)
+    ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
     mock_client = AsyncMock(spec=AsyncHTTPHandler)
     mock_client.post = AsyncMock(return_value=ok_response)
 
@@ -2072,8 +2072,7 @@ async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks(
         captured_kwargs.update(call_kwargs)
         return sentinel_response
 
-    mock_httpx_response = Mock()
-    mock_httpx_response.status_code = 200
+    mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages"))
 
     with (
         patch.object(

tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py

@@ -4,7 +4,7 @@ import json
 import re
 import sys
 import time
-from collections.abc import Iterator, Mapping
+from collections.abc import Awaitable, Iterator, Mapping
 from types import SimpleNamespace
 from typing import TYPE_CHECKING, Final, Literal, Optional
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -61,6 +61,7 @@ from litellm.proxy.auth.auth_checks import (
     _check_agent_caller_model_access,
     _virtual_key_max_budget_check,
     _virtual_key_soft_budget_check,
+    common_checks,
     get_key_object,
     get_user_object,
     invalidate_team_member_spend_state,
@@ -74,6 +75,7 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
 from litellm.constants import (
     DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL,
     END_USER_RESTRICTED_REGISTRY_MAX_SIZE,
+    LITELLM_PROXY_MASTER_KEY_ALIAS,
     PROXY_DB_LOOKUP_MAX_CONCURRENCY,
     REGISTRY_ERROR_NEGATIVE_CACHE_TTL,
     TAG_REGISTRY_MAX_SIZE,
@@ -90,6 +92,7 @@ from litellm.proxy.common_utils.user_api_key_cache import (
     UserApiKeyCache,
     end_user_cache_key,
     end_user_restricted_registry_cache_key,
+    object_permission_cache_key,
     tag_cache_key,
     tag_registry_cache_key,
 )
@@ -1646,8 +1649,9 @@ async def test_vector_store_access_check_with_permissions():
     )
 
     mock_prisma_client = MagicMock()
-    mock_permissions = MagicMock()
-    mock_permissions.vector_stores = ["store-1", "store-2"]
+    mock_permissions = LiteLLM_ObjectPermissionTable(
+        object_permission_id="perm-123", vector_stores=["store-1", "store-2"]
+    )
     mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=mock_permissions)
 
     mock_vector_store_registry = MagicMock()
@@ -1688,12 +1692,12 @@ async def test_vector_store_access_check_with_team_permissions():
     request_body = {}
     valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None)
 
-    team_object = MagicMock()
-    team_object.object_permission_id = "team-permission"
+    team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission")
 
     mock_prisma_client = MagicMock()
-    team_permissions = MagicMock()
-    team_permissions.vector_stores = ["team-store-allowed"]
+    team_permissions = LiteLLM_ObjectPermissionTable(
+        object_permission_id="team-permission", vector_stores=["team-store-allowed"]
+    )
     mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions)
 
     mock_vector_store_registry = MagicMock()
@@ -1755,12 +1759,12 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query(
     }
     valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None)
 
-    team_object = MagicMock()
-    team_object.object_permission_id = "team-permission"
+    team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission")
 
     mock_prisma_client = MagicMock()
-    team_permissions = MagicMock()
-    team_permissions.vector_stores = ["KBALLOWED123"]
+    team_permissions = LiteLLM_ObjectPermissionTable(
+        object_permission_id="team-permission", vector_stores=["KBALLOWED123"]
+    )
     mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions)
 
     with (
@@ -1786,6 +1790,592 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query(
     assert exc_info.value.type == expected_error_type
 
 
+_KEY_DENIED: Final = ProxyErrorTypes.key_vector_store_access_denied
+_TEAM_DENIED: Final = ProxyErrorTypes.team_vector_store_access_denied
+_USER_DENIED: Final = ProxyErrorTypes.user_vector_store_access_denied
+
+
+def _virtual_key(
+    object_permission_id: str | None = None,
+    api_key: str = "sk-standalone",
+    user_role: LitellmUserRoles | None = None,
+    team_id: str | None = None,
+) -> UserAPIKeyAuth:
+    key: Final = UserAPIKeyAuth(
+        api_key=api_key,
+        user_id="key-owner",
+        user_role=user_role,
+        team_id=team_id,
+        object_permission_id=object_permission_id,
+    )
+    key.via_virtual_key = True
+    return key
+
+
+def _permission_row(
+    object_permission_id: str, permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None
+) -> LiteLLM_ObjectPermissionTable | None:
+    if isinstance(permission, SimpleNamespace):
+        return LiteLLM_ObjectPermissionTable(
+            object_permission_id=object_permission_id, vector_stores=permission.vector_stores
+        )
+    return permission
+
+
+async def _common_checks_for_rag_query(
+    general_settings: Mapping[str, object],
+    valid_token: UserAPIKeyAuth,
+    key_permission: SimpleNamespace | None,
+    team_object: LiteLLM_TeamTable | None = None,
+    team_permission: SimpleNamespace | None = None,
+    user_object: LiteLLM_UserTable | None = None,
+    user_permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None = None,
+    request_body: Mapping[str, object] | None = None,
+    user_api_key_cache: UserApiKeyCache | None = None,
+    vector_store_registry: VectorStoreRegistry | None = None,
+) -> bool:
+    permissions: Final = {
+        "key-permission": _permission_row("key-permission", key_permission),
+        "team-permission": _permission_row("team-permission", team_permission),
+        "user-permission": _permission_row("user-permission", user_permission),
+    }
+    mock_prisma_client: Final = MagicMock()
+    mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(
+        side_effect=lambda where: permissions.get(where["object_permission_id"])
+    )
+    mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None)
+    body: Final = (
+        dict(request_body)
+        if request_body is not None
+        else {
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "what is in this KB?"}],
+            "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"},
+        }
+    )
+    with (
+        patch(  # test-quality-ok: production auth reads these module globals; no dependency injection seam exists
+            "litellm.proxy.proxy_server.prisma_client", mock_prisma_client
+        ),
+        patch(  # test-quality-ok: production auth reads this module global; no dependency injection seam exists
+            "litellm.vector_store_registry", vector_store_registry
+        ),
+        patch(  # test-quality-ok: production auth reads this module global; no dependency injection seam exists
+            "litellm.proxy.proxy_server.user_api_key_cache",
+            UserApiKeyCache() if user_api_key_cache is None else user_api_key_cache,
+        ),
+    ):
+        return await common_checks(
+            request_body=body,
+            team_object=team_object,
+            user_object=user_object,
+            end_user_object=None,
+            global_proxy_spend=None,
+            general_settings=dict(general_settings),
+            route="/v1/rag/query",
+            llm_router=None,
+            proxy_logging_obj=MagicMock(),
+            valid_token=valid_token,
+            request=MagicMock(spec=Request),
+        )
+
+
+async def _assert_rag_query_outcome(request: Awaitable[bool], denied_by: ProxyErrorTypes | None) -> None:
+    if denied_by is None:
+        assert await request is True
+        return
+    with pytest.raises(ProxyException) as exc_info:
+        await request
+    assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (denied_by, "vector_store", "401")
+
+
+async def _team_key_rag_query(
+    general_settings: Mapping[str, object],
+    key_vector_stores: list[str] | None,
+    team_vector_stores: list[str] | None,
+    user_object: LiteLLM_UserTable | None = None,
+) -> bool:
+    return await _common_checks_for_rag_query(
+        general_settings,
+        _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission", team_id="team-1"),
+        None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores),
+        LiteLLM_TeamTable(
+            team_id="team-1", object_permission_id=None if team_vector_stores is None else "team-permission"
+        ),
+        None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores),
+        user_object,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("general_settings", "denied_by"),
+    [
+        ({}, None),
+        ({"vector_store_deny_by_default": False}, None),
+        ({"vector_store_deny_by_default": True}, _KEY_DENIED),
+    ],
+    ids=["flag-omitted", "flag-false", "flag-true"],
+)
+async def test_standalone_key_without_vector_store_permission_follows_deny_by_default(
+    general_settings: Mapping[str, object], denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(_common_checks_for_rag_query(general_settings, _virtual_key(), None), denied_by)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("deny_by_default", "vector_stores", "denied_by"),
+    [
+        (True, [], _KEY_DENIED),
+        (True, ["KBSTOREA"], None),
+        (True, ["KBSTOREB"], _KEY_DENIED),
+        (True, None, _KEY_DENIED),
+        (False, ["KBSTOREB"], _KEY_DENIED),
+    ],
+    ids=["enabled-empty", "enabled-contains", "enabled-excludes", "enabled-null", "disabled-excludes"],
+)
+async def test_standalone_key_vector_store_permission_record_under_deny_by_default(
+    deny_by_default: bool, vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": deny_by_default},
+            _virtual_key(object_permission_id="key-permission"),
+            SimpleNamespace(vector_stores=vector_stores),
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deny_by_default", [False, True], ids=["flag-false", "flag-true"])
+async def test_master_key_rag_query_is_unchanged_by_deny_by_default(deny_by_default: bool):
+    master_key: Final = _virtual_key(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN)
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query({"vector_store_deny_by_default": deny_by_default}, master_key, None), None
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("general_settings", "vector_stores", "denied_by"),
+    [
+        ({}, None, None),
+        ({"vector_store_deny_by_default": False}, None, None),
+        ({"vector_store_deny_by_default": False}, ["KBSTOREB"], _KEY_DENIED),
+    ],
+    ids=["flag-omitted-no-record", "flag-false-no-record", "flag-false-excludes"],
+)
+async def test_proxy_admin_virtual_key_keeps_flag_off_vector_store_behavior(
+    general_settings: Mapping[str, object], vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    admin_key: Final = _virtual_key(
+        object_permission_id=None if vector_stores is None else "key-permission", user_role=LitellmUserRoles.PROXY_ADMIN
+    )
+    key_permission: Final = None if vector_stores is None else SimpleNamespace(vector_stores=vector_stores)
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(general_settings, admin_key, key_permission), denied_by
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("key_vector_stores", "team_vector_stores", "denied_by"),
+    [
+        (["KBSTOREA"], ["KBSTOREA"], None),
+        (None, ["KBSTOREA"], _KEY_DENIED),
+        ([], ["KBSTOREA"], _KEY_DENIED),
+        (["KBSTOREA"], None, _TEAM_DENIED),
+        (["KBSTOREA"], [], _TEAM_DENIED),
+        (["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED),
+        (["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED),
+    ],
+    ids=["both-grant", "key-no-record", "key-empty", "team-no-record", "team-empty", "key-excludes", "team-excludes"],
+)
+async def test_team_key_requires_key_and_team_grant_under_deny_by_default(
+    key_vector_stores: list[str] | None, team_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _team_key_rag_query({"vector_store_deny_by_default": True}, key_vector_stores, team_vector_stores), denied_by
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("general_settings", "key_vector_stores", "team_vector_stores", "denied_by"),
+    [
+        ({}, [], ["KBSTOREA"], None),
+        ({"vector_store_deny_by_default": False}, [], ["KBSTOREA"], None),
+        ({"vector_store_deny_by_default": False}, None, None, None),
+        ({"vector_store_deny_by_default": False}, ["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED),
+        ({"vector_store_deny_by_default": False}, ["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED),
+    ],
+    ids=["omitted-key-empty", "false-key-empty", "false-no-records", "false-key-excludes", "false-team-excludes"],
+)
+async def test_team_key_keeps_legacy_vector_store_behavior_when_flag_off(
+    general_settings: Mapping[str, object],
+    key_vector_stores: list[str] | None,
+    team_vector_stores: list[str] | None,
+    denied_by: ProxyErrorTypes | None,
+):
+    await _assert_rag_query_outcome(
+        _team_key_rag_query(general_settings, key_vector_stores, team_vector_stores), denied_by
+    )
+
+
+@pytest.mark.asyncio
+async def test_user_owned_team_key_without_key_grant_is_denied_despite_team_membership():
+    team_member: Final = LiteLLM_UserTable(
+        user_id="key-owner", teams=["team-1"], user_role=LitellmUserRoles.INTERNAL_USER
+    )
+
+    await _assert_rag_query_outcome(
+        _team_key_rag_query({"vector_store_deny_by_default": True}, [], ["KBSTOREA"], team_member), _KEY_DENIED
+    )
+
+
+def _user_row(user_vector_stores: list[str] | None, teams: tuple[str, ...] = ()) -> LiteLLM_UserTable:
+    return LiteLLM_UserTable(
+        user_id="user-1",
+        teams=list(teams),
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        object_permission_id=None if user_vector_stores is None else "user-permission",
+    )
+
+
+async def _keyless_rag_query(
+    general_settings: Mapping[str, object],
+    user_vector_stores: list[str] | None,
+    team_vector_stores: list[str] | None = None,
+    team_id: str | None = None,
+    user_permission: LiteLLM_ObjectPermissionTable | None = None,
+) -> bool:
+    return await _common_checks_for_rag_query(
+        general_settings,
+        UserAPIKeyAuth(user_id="user-1", team_id=team_id, user_role=LitellmUserRoles.INTERNAL_USER),
+        None,
+        None
+        if team_id is None
+        else LiteLLM_TeamTable(
+            team_id=team_id, object_permission_id=None if team_vector_stores is None else "team-permission"
+        ),
+        None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores),
+        _user_row(user_vector_stores, teams=() if team_id is None else (team_id,)),
+        user_permission or (None if user_vector_stores is None else SimpleNamespace(vector_stores=user_vector_stores)),
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_vector_stores", "denied_by"),
+    [(["KBSTOREA"], None), (["KBSTOREB"], _USER_DENIED), ([], _USER_DENIED), (None, _USER_DENIED)],
+    ids=["user-grants", "user-excludes", "user-empty", "user-no-record"],
+)
+async def test_keyless_user_without_team_needs_user_grant_under_deny_by_default(
+    user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores), denied_by
+    )
+
+
+@pytest.mark.asyncio
+async def test_keyless_user_null_vector_store_list_grants_nothing_under_deny_by_default():
+    null_list: Final = LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=None)
+
+    await _assert_rag_query_outcome(
+        _keyless_rag_query({"vector_store_deny_by_default": True}, [], user_permission=null_list), _USER_DENIED
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("general_settings", "user_vector_stores"),
+    [({}, ["KBSTOREB"]), ({"vector_store_deny_by_default": False}, ["KBSTOREB"]), ({}, None)],
+    ids=["omitted-user-excludes", "false-user-excludes", "omitted-user-no-record"],
+)
+async def test_keyless_user_keeps_legacy_vector_store_behavior_when_flag_off(
+    general_settings: Mapping[str, object], user_vector_stores: list[str] | None
+):
+    await _assert_rag_query_outcome(_keyless_rag_query(general_settings, user_vector_stores), None)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("team_vector_stores", "user_vector_stores", "denied_by"),
+    [
+        (["KBSTOREA"], [], None),
+        (["KBSTOREB"], ["KBSTOREA"], _TEAM_DENIED),
+        ([], ["KBSTOREA"], _TEAM_DENIED),
+        (None, ["KBSTOREA"], _TEAM_DENIED),
+    ],
+    ids=["team-grants-user-empty", "team-excludes-user-grants", "team-empty-user-grants", "team-no-record-user-grants"],
+)
+async def test_keyless_team_member_needs_only_team_grant_under_deny_by_default(
+    team_vector_stores: list[str] | None, user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores, team_vector_stores, "team-1"),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("team_id", "denied_by"),
+    [("team-1", None), (None, None)],
+    ids=["session-on-team-uses-team-grant", "session-without-team-uses-user-grant"],
+)
+async def test_session_token_is_not_a_virtual_key_under_deny_by_default(
+    team_id: str | None, denied_by: ProxyErrorTypes | None
+):
+    session: Final = UserAPIKeyAuth(
+        api_key="hashed-session-token",
+        user_id="user-1",
+        team_id=team_id,
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        is_session_token=True,
+    )
+    session.via_virtual_key = True
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            session,
+            None,
+            None if team_id is None else LiteLLM_TeamTable(team_id=team_id, object_permission_id="team-permission"),
+            SimpleNamespace(vector_stores=["KBSTOREA"]),
+            _user_row(["KBSTOREA"] if team_id is None else []),
+            SimpleNamespace(vector_stores=["KBSTOREA"] if team_id is None else []),
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+async def test_user_owned_standalone_key_cannot_use_owner_grants_under_deny_by_default():
+    owner: Final = LiteLLM_UserTable(
+        user_id="key-owner", user_role=LitellmUserRoles.INTERNAL_USER, object_permission_id="user-permission"
+    )
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            _virtual_key(),
+            None,
+            user_object=owner,
+            user_permission=SimpleNamespace(vector_stores=["KBSTOREA"]),
+        ),
+        _KEY_DENIED,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("deny_by_default", "key_vector_stores", "denied_by"),
+    [
+        (True, ["KBSTOREA", "KBSTOREB"], None),
+        (True, ["KBSTOREA"], _KEY_DENIED),
+        (True, ["KBSTOREB"], _KEY_DENIED),
+        (False, ["KBSTOREA"], _KEY_DENIED),
+    ],
+    ids=["enabled-grants-both", "enabled-missing-second", "enabled-missing-first", "disabled-missing-second"],
+)
+async def test_every_requested_vector_store_needs_a_grant(
+    deny_by_default: bool, key_vector_stores: list[str], denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": deny_by_default},
+            _virtual_key(object_permission_id="key-permission"),
+            SimpleNamespace(vector_stores=key_vector_stores),
+            request_body={
+                "model": "gpt-4o-mini",
+                "messages": [{"role": "user", "content": "what is in these KBs?"}],
+                "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREB"]}],
+                "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"},
+            },
+            vector_store_registry=VectorStoreRegistry(),
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "valid_token",
+    [_virtual_key(), UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER)],
+    ids=["standalone-key", "keyless-user"],
+)
+async def test_request_without_vector_stores_is_unaffected_by_deny_by_default(valid_token: UserAPIKeyAuth):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            valid_token,
+            None,
+            user_object=_user_row(None),
+            request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]},
+            vector_store_registry=VectorStoreRegistry(),
+        ),
+        None,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "request_body",
+    [
+        {"query": "what is in this KB?", "vector_store_id": "KBSTOREA", "vector_store_ids": ["KBSTOREA"]},
+        {
+            "model": "gpt-4o-mini",
+            "input": "what is in this KB?",
+            "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREA"]}],
+        },
+    ],
+    ids=["vector-store-search-route", "responses-file-search"],
+)
+@pytest.mark.parametrize(
+    ("deny_by_default", "key_vector_stores", "denied_by"),
+    [(True, None, _KEY_DENIED), (True, ["KBSTOREA"], None), (False, None, None)],
+    ids=["enabled-no-grant", "enabled-grant", "disabled-no-grant"],
+)
+async def test_deny_by_default_reads_requested_vector_stores_without_a_registry(
+    request_body: Mapping[str, object],
+    deny_by_default: bool,
+    key_vector_stores: list[str] | None,
+    denied_by: ProxyErrorTypes | None,
+):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": deny_by_default},
+            _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"),
+            None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores),
+            request_body=request_body,
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("request_body", "key_vector_stores", "denied_by"),
+    [
+        ({"tools": [1]}, None, None),
+        ({"tools": 1}, None, None),
+        ({"tools": [{"type": "file_search", "vector_store_ids": None}]}, None, None),
+        ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREA"], None),
+        ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREB"], _KEY_DENIED),
+    ],
+    ids=["int-tool", "non-list-tools", "null-tool-ids", "int-tool-beside-granted", "int-tool-beside-ungranted"],
+)
+async def test_deny_by_default_ignores_tools_that_name_no_vector_store(
+    request_body: Mapping[str, object], key_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"),
+            None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores),
+            request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body},
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "request_body",
+    [
+        {"tools": [{"type": "file_search", "vector_store_ids": "KBSTOREA"}]},
+        {"tools": [{"type": "file_search", "vector_store_ids": [1]}]},
+        {"vector_store_ids": "KBSTOREA"},
+    ],
+    ids=["string-tool-ids", "int-tool-id", "string-top-level-ids"],
+)
+async def test_deny_by_default_rejects_malformed_vector_store_ids_as_bad_request(request_body: Mapping[str, object]):
+    with pytest.raises(ProxyException) as exc_info:
+        await _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            _virtual_key(object_permission_id="key-permission"),
+            SimpleNamespace(vector_stores=["KBSTOREA"]),
+            request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body},
+        )
+    assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
+        "invalid_request_error",
+        "vector_store_ids",
+        "400",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("request_body", "denied_by"),
+    [
+        ({"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, None),
+        (None, _KEY_DENIED),
+    ],
+    ids=["no-vector-store", "ungranted-vector-store"],
+)
+@pytest.mark.parametrize("flag_value", [None, "enabled"], ids=["null", "string"])
+async def test_invalid_deny_by_default_value_only_denies_vector_store_requests(
+    flag_value: object, request_body: Mapping[str, object] | None, denied_by: ProxyErrorTypes | None
+):
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": flag_value}, _virtual_key(), None, request_body=request_body
+        ),
+        denied_by,
+    )
+
+
+@pytest.mark.asyncio
+async def test_keyless_user_grant_is_read_through_the_object_permission_cache():
+    cache: Final = UserApiKeyCache(default_in_memory_ttl=60)
+    await cache.async_set_cache(
+        key=object_permission_cache_key("user-permission"),
+        value=LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=["KBSTOREA"]),
+        model_type=LiteLLM_ObjectPermissionTable,
+    )
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": True},
+            UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER),
+            None,
+            user_object=_user_row(["KBSTOREA"]),
+            user_api_key_cache=cache,
+        ),
+        None,
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("deny_by_default", [True, False], ids=["strict", "default"])
+async def test_key_and_team_grants_are_read_through_the_object_permission_cache(deny_by_default: bool):
+    cache: Final = UserApiKeyCache(default_in_memory_ttl=60)
+    for object_permission_id, vector_stores in (("key-permission", ["KBSTOREA"]), ("team-permission", ["KBSTOREB"])):
+        await cache.async_set_cache(
+            key=object_permission_cache_key(object_permission_id),
+            value=LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, vector_stores=vector_stores),
+            model_type=LiteLLM_ObjectPermissionTable,
+        )
+
+    await _assert_rag_query_outcome(
+        _common_checks_for_rag_query(
+            {"vector_store_deny_by_default": deny_by_default},
+            _virtual_key(object_permission_id="key-permission", team_id="team-1"),
+            None,
+            LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission"),
+            None,
+            user_api_key_cache=cache,
+        ),
+        _TEAM_DENIED,
+    )
+
+
 def test_can_object_call_model_with_alias():
     """Test that can_object_call_model works with model aliases"""
     from litellm import Router
@@ -10346,7 +10936,9 @@ async def test_authoritative_access_group_reads_writer_despite_stale_allow_cache
     from litellm.proxy._types import LiteLLM_AccessGroupTable
     from litellm.proxy.auth.auth_checks import get_access_object
 
-    stale: Final = LiteLLM_AccessGroupTable(access_group_id="group", access_group_name="Policy", access_model_names=["old"])
+    stale: Final = LiteLLM_AccessGroupTable(
+        access_group_id="group", access_group_name="Policy", access_model_names=["old"]
+    )
     current: Final = stale.model_copy(update={"access_model_names": ["new"] if allowed else []})
     client: Final = MagicMock()
     client.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(return_value=current)
@@ -10498,9 +11090,12 @@ async def test_authoritative_key_cannot_keep_grants_when_permission_is_unavailab
     from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 
     database: Final = MagicMock()
-    database.get_data = AsyncMock(return_value=UserAPIKeyAuth(
-        object_permission_id="grant", object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"])
-    ))
+    database.get_data = AsyncMock(
+        return_value=UserAPIKeyAuth(
+            object_permission_id="grant",
+            object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"]),
+        )
+    )
     database.writer_db.litellm_objectpermissiontable.find_unique = AsyncMock(
         return_value=None, side_effect=None if missing else RuntimeError("writer unavailable")
     )
@@ -10523,7 +11118,9 @@ async def test_authoritative_group_grants_propagate_policy_outages(
 
     database: Final = MagicMock()
     database.db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable"))
-    database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable"))
+    database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(
+        side_effect=RuntimeError("database unavailable")
+    )
     monkeypatch.setattr(proxy_server, "prisma_client", database)
     monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache())
     if strict:

tests/unit/proxy/auth/test_fallback_model_access.py

@@ -1,12 +1,13 @@
 import pytest
 
 from litellm import Router
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth
 from litellm.proxy.auth.fallback_model_access import (
     RouterFallbackAccessCheck,
     is_model_authorized_for_token,
     router_fallback_access_check,
 )
+from litellm.search import asearch
 
 
 def _router() -> Router:
@@ -105,3 +106,69 @@ async def test_proxy_check_reads_enforce_fallback_model_access_from_general_sett
         )
         is expected
     )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("check", [ENFORCED, NOT_ENFORCED], ids=["enforced", "not-enforced"])
+async def test_search_tool_fallback_target_follows_the_key_search_tool_grant(
+    monkeypatch: pytest.MonkeyPatch, check: RouterFallbackAccessCheck
+):
+    monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {})
+    router = Router(
+        model_list=[],
+        search_tools=[
+            {"search_tool_name": name, "litellm_params": {"search_provider": "tavily", "api_key": "k"}}
+            for name in ("search-a", "search-b")
+        ],
+    )
+    request_kwargs = {
+        "original_generic_function": asearch,
+        "litellm_metadata": {
+            "user_api_key_auth": UserAPIKeyAuth(
+                api_key="hashed",
+                object_permission_id="op-key",
+                object_permission=LiteLLM_ObjectPermissionTable(
+                    object_permission_id="op-key", search_tools=["search-a"]
+                ),
+            )
+        },
+    }
+
+    assert await check(model="search-a", request_kwargs=request_kwargs, llm_router=router)
+    assert not await check(model="search-b", request_kwargs=request_kwargs, llm_router=router)
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "request_kwargs, expected",
+    [
+        ({"original_generic_function": asearch}, True),
+        ({}, False),
+    ],
+    ids=["search-request", "completion-request"],
+)
+async def test_fallback_named_like_both_a_model_and_a_search_tool_follows_the_request_kind(
+    request_kwargs: dict, expected: bool
+):
+    router = Router(
+        model_list=[
+            {"model_name": "shared-name", "litellm_params": {"model": "openai/secret", "api_key": "k"}},
+        ],
+        search_tools=[
+            {"search_tool_name": "shared-name", "litellm_params": {"search_provider": "tavily", "api_key": "k"}},
+        ],
+    )
+    key = UserAPIKeyAuth(
+        api_key="hashed",
+        models=["open-model"],
+        object_permission_id="op-key",
+        object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=["shared-name"]),
+    )
+
+    allowed = await ENFORCED(
+        model="shared-name",
+        request_kwargs={**request_kwargs, "metadata": {"user_api_key_auth": key}},
+        llm_router=router,
+    )
+
+    assert allowed is expected

tests/unit/proxy/auth/test_search_tool_deny_by_default.py

@@ -0,0 +1,270 @@
+from collections.abc import Mapping
+from typing import Final, TypedDict
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS
+from litellm.proxy import proxy_server
+from litellm.proxy._types import (
+    UI_TEAM_ID,
+    LiteLLM_ObjectPermissionTable,
+    LiteLLM_TeamTable,
+    LiteLLM_UserTable,
+    LitellmUserRoles,
+    ProxyErrorTypes,
+    ProxyException,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.auth_checks import (
+    TeamObjectLoader,
+    can_caller_call_search_tool,
+    check_unregistered_search_fallback,
+)
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key
+
+_DENY_ON: Final = {"search_tool_deny_by_default": True}
+_KEY: Final = ProxyErrorTypes.key_search_tool_access_denied
+_TEAM: Final = ProxyErrorTypes.team_search_tool_access_denied
+_USER: Final = ProxyErrorTypes.user_search_tool_access_denied
+_LEGACY: Final = ProxyErrorTypes.key_model_access_denied
+_TEAM_LOAD_FAILS: Final = "team-load-fails"
+_NO_ROW: Final = "no-row"
+
+
+class CallerFields(TypedDict, total=False):
+    virtual_key: bool
+    key_tools: list[str] | None | str
+    team_id: str | None
+    user_id: str | None
+    user_role: LitellmUserRoles | None
+    api_key: str
+
+
+def _grant(search_tools: list[str] | None, object_permission_id: str = "op") -> LiteLLM_ObjectPermissionTable:
+    return LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, search_tools=search_tools)
+
+
+def _caller(
+    virtual_key: bool = True,
+    key_tools: list[str] | None | str = _NO_ROW,
+    team_id: str | None = None,
+    user_id: str | None = "user-1",
+    user_role: LitellmUserRoles | None = LitellmUserRoles.INTERNAL_USER,
+    api_key: str = "sk-caller",
+) -> UserAPIKeyAuth:
+    token: Final = UserAPIKeyAuth(
+        api_key=api_key,
+        user_id=user_id,
+        user_role=user_role,
+        team_id=team_id,
+        object_permission_id=None if key_tools == _NO_ROW else "op-key",
+        object_permission=None if key_tools == _NO_ROW else _grant(key_tools, "op-key"),
+    )
+    token.via_virtual_key = virtual_key
+    return token
+
+
+def _team_loader(team_tools: list[str] | None | str) -> TeamObjectLoader:
+    async def load() -> LiteLLM_TeamTable | None:
+        if team_tools == _TEAM_LOAD_FAILS:
+            raise ProxyException(message="team lookup failed", type="auth_error", param="team_id", code=404)
+        if team_tools == _NO_ROW:
+            return LiteLLM_TeamTable(team_id="team-1")
+        return LiteLLM_TeamTable(
+            team_id="team-1", object_permission_id="op-team", object_permission=_grant(team_tools, "op-team")
+        )
+
+    return load
+
+
+async def _no_team() -> None:
+    return None
+
+
+@pytest.fixture
+def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache:
+    user_api_key_cache: Final = UserApiKeyCache()
+    monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache)
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    return user_api_key_cache
+
+
+def _cache_user(cache: UserApiKeyCache, user_tools: list[str] | None) -> None:
+    cache.set_cache(key="user-1", value=LiteLLM_UserTable(user_id="user-1", object_permission_id="op-user"))
+    cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant(user_tools, "op-user"))
+
+
+async def _denied_by(
+    general_settings: Mapping[str, object], caller: UserAPIKeyAuth, load_team: TeamObjectLoader = _no_team
+) -> ProxyErrorTypes | None:
+    try:
+        await can_caller_call_search_tool("search-a", caller, general_settings, load_team)
+    except ProxyException as e:
+        denial = e
+    else:
+        return None
+    assert (denial.code, denial.param) == ("403", "search_tool_name")
+    return ProxyErrorTypes(denial.type)
+
+
+@pytest.mark.parametrize(
+    "general_settings",
+    [{}, {"search_tool_deny_by_default": False}],
+    ids=["omitted", "false"],
+)
+@pytest.mark.parametrize(
+    "caller, team_tools, expected",
+    [
+        pytest.param(CallerFields(), None, None, id="no grants anywhere"),
+        pytest.param(CallerFields(key_tools=[]), None, None, id="empty key list"),
+        pytest.param(CallerFields(team_id="team-1"), [], None, id="empty team list"),
+        pytest.param(CallerFields(key_tools=["search-b"]), None, _LEGACY, id="key allowlist excludes"),
+        pytest.param(CallerFields(team_id="team-1"), ["search-b"], _LEGACY, id="team allowlist excludes"),
+        pytest.param(CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, id="both allow"),
+    ],
+)
+@pytest.mark.asyncio
+async def test_search_tool_access_is_unchanged_without_search_tool_deny_by_default(
+    cache: UserApiKeyCache,
+    general_settings: dict[str, bool],
+    caller: CallerFields,
+    team_tools: list[str] | None,
+    expected: ProxyErrorTypes | None,
+):
+    _cache_user(cache, ["search-b"])
+    load_team: Final = _no_team if team_tools is None else _team_loader(team_tools)
+    assert await _denied_by(general_settings, _caller(**caller), load_team) == expected
+
+
+@pytest.mark.parametrize(
+    "caller, team_tools, user_tools, expected",
+    [
+        pytest.param(CallerFields(), None, ["search-a"], _KEY, id="standalone key, no permission row"),
+        pytest.param(CallerFields(key_tools=None), None, None, _KEY, id="standalone key, search_tools null"),
+        pytest.param(CallerFields(key_tools=[]), None, None, _KEY, id="standalone key, empty grant"),
+        pytest.param(CallerFields(key_tools=["search-b"]), None, None, _KEY, id="standalone key, grants another tool"),
+        pytest.param(CallerFields(key_tools=["search-a"]), None, [], None, id="standalone key, key grant is enough"),
+        pytest.param(CallerFields(key_tools=[]), None, ["search-a"], _KEY, id="standalone key, user cannot stand in"),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, None, id="team key, both"
+        ),
+        pytest.param(CallerFields(key_tools=[], team_id="team-1"), ["search-a"], None, _KEY, id="team key, empty key"),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"), [], None, _TEAM, id="team key, empty team"
+        ),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"), None, None, _TEAM, id="team key, team null"
+        ),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"), _NO_ROW, None, _TEAM, id="team key, team has no row"
+        ),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"),
+            _TEAM_LOAD_FAILS,
+            None,
+            _TEAM,
+            id="team key, team fails to load",
+        ),
+        pytest.param(
+            CallerFields(key_tools=["search-a"], team_id="team-1"),
+            ["search-b"],
+            ["search-a"],
+            _TEAM,
+            id="team key, user cannot stand in for team",
+        ),
+        pytest.param(
+            CallerFields(virtual_key=False, team_id="team-1"), ["search-a"], [], None, id="keyless team member"
+        ),
+        pytest.param(
+            CallerFields(virtual_key=False, team_id="team-1"), [], ["search-a"], _TEAM, id="keyless, empty team"
+        ),
+        pytest.param(CallerFields(virtual_key=False), None, ["search-a"], None, id="keyless user, user grants"),
+        pytest.param(CallerFields(virtual_key=False), None, None, _USER, id="keyless user, search_tools null"),
+        pytest.param(CallerFields(virtual_key=False), None, [], _USER, id="keyless user, empty grant"),
+        pytest.param(
+            CallerFields(virtual_key=False, user_id="user-2"), None, None, _USER, id="keyless user fails to load"
+        ),
+        pytest.param(CallerFields(virtual_key=False, user_id=None), None, None, _USER, id="keyless caller, no user"),
+        pytest.param(
+            CallerFields(user_role=LitellmUserRoles.PROXY_ADMIN),
+            None,
+            None,
+            _KEY,
+            id="proxy admin virtual key not exempt",
+        ),
+        pytest.param(CallerFields(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, None, None, id="master key exempt"),
+        pytest.param(
+            CallerFields(team_id=UI_TEAM_ID, virtual_key=False), None, None, None, id="dashboard session exempt"
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_search_tool_deny_by_default_requires_every_owning_identity_to_grant(
+    cache: UserApiKeyCache,
+    caller: CallerFields,
+    team_tools: list[str] | str | None,
+    user_tools: list[str] | None,
+    expected: ProxyErrorTypes | None,
+):
+    if user_tools is not None:
+        _cache_user(cache, user_tools)
+    elif caller.get("user_id", "user-1") == "user-1":
+        _cache_user(cache, None)
+    load_team: Final = _no_team if team_tools is None and "team_id" not in caller else _team_loader(team_tools)
+    assert await _denied_by(_DENY_ON, _caller(**caller), load_team) == expected
+
+
+@pytest.mark.parametrize("value", ["true", "enabled", 1], ids=["string-true", "string", "int"])
+@pytest.mark.asyncio
+async def test_non_boolean_search_tool_deny_by_default_enables_the_policy(cache: UserApiKeyCache, value: object):
+    assert await _denied_by({"search_tool_deny_by_default": value}, _caller(key_tools=[])) == _KEY
+
+
+@pytest.mark.asyncio
+async def test_search_tool_deny_by_default_denies_when_no_database_is_connected(
+    cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch
+):
+    monkeypatch.setattr(proxy_server, "prisma_client", None)
+    assert await _denied_by(_DENY_ON, _caller(key_tools=["search-a"])) == _KEY
+
+
+@pytest.mark.asyncio
+async def test_search_tool_deny_by_default_reads_the_user_grant_on_every_call(cache: UserApiKeyCache):
+    caller: Final = _caller(virtual_key=False)
+    _cache_user(cache, ["search-a"])
+    assert await _denied_by(_DENY_ON, caller) is None
+    cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant([], "op-user"))
+    assert await _denied_by(_DENY_ON, caller) == _USER
+
+
+@pytest.mark.asyncio
+async def test_search_tool_deny_by_default_does_not_load_the_user_when_off(
+    cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch
+):
+    get_user_object: Final = AsyncMock()
+    monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_user_object", get_user_object)
+    assert await _denied_by({}, _caller(virtual_key=False)) is None
+    get_user_object.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+    "general_settings, caller, expected",
+    [
+        pytest.param({}, _caller(), None, id="flag off"),
+        pytest.param(_DENY_ON, _caller(), _KEY, id="virtual key"),
+        pytest.param(_DENY_ON, _caller(virtual_key=False, team_id="team-1"), _TEAM, id="keyless team member"),
+        pytest.param(_DENY_ON, _caller(virtual_key=False), _USER, id="keyless user"),
+        pytest.param(_DENY_ON, _caller(user_role=LitellmUserRoles.PROXY_ADMIN), _KEY, id="proxy admin key"),
+        pytest.param(_DENY_ON, _caller(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, id="master key"),
+    ],
+)
+def test_unregistered_search_fallback_follows_search_tool_deny_by_default(
+    general_settings: Mapping[str, object], caller: UserAPIKeyAuth, expected: ProxyErrorTypes | None
+):
+    if expected is None:
+        assert check_unregistered_search_fallback(caller, general_settings) is True
+        return
+    with pytest.raises(ProxyException) as exc_info:
+        check_unregistered_search_fallback(caller, general_settings)
+    assert (exc_info.value.type, exc_info.value.code) == (expected, "403")

tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py

@@ -12,6 +12,7 @@ from functools import partial
 from pathlib import Path
 from textwrap import dedent
 from types import SimpleNamespace
+from typing import Final
 from unittest.mock import ANY, AsyncMock, MagicMock, patch
 
 
@@ -21,11 +22,13 @@ from fastapi import HTTPException, status
 import litellm
 import litellm.proxy.proxy_server
 from litellm.caching.dual_cache import DualCache
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy._types import (
     LiteLLMRoutes,
     LiteLLM_JWTAuth,
     LiteLLM_BudgetTable,
     LiteLLM_EndUserTable,
+    LiteLLM_ObjectPermissionTable,
     LiteLLM_OrganizationTable,
     LiteLLM_TeamTableCachedObj,
     LiteLLM_UserTable,
@@ -5304,6 +5307,222 @@ async def test_centralized_common_checks_tolerates_db_errors_when_fetching_conte
             setattr(_proxy_server_mod, k, v)
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("deny_by_default", "team_lookup_error", "denied"),
+    [
+        (False, RuntimeError("team cache unavailable"), False),
+        (True, RuntimeError("team cache unavailable"), True),
+        (True, HTTPException(status_code=404, detail="team read failed"), True),
+    ],
+    ids=["flag-off-lookup-swallowed", "flag-on-lookup-swallowed", "flag-on-team-rebuilt-from-token"],
+)
+async def test_team_key_vector_store_access_when_team_cannot_be_resolved(
+    monkeypatch: pytest.MonkeyPatch, deny_by_default: bool, team_lookup_error: Exception, denied: bool
+):
+    from fastapi import Request
+    from starlette.datastructures import URL
+
+    token: Final = UserAPIKeyAuth(
+        api_key="sk-team-key", team_id="team-1", team_models=["gpt-4o-mini"], object_permission_id="key-permission"
+    )
+    token.via_virtual_key = True
+    request: Final = Request(scope={"type": "http"})
+    request._url = URL(url="/v1/rag/query")
+    database: Final = MagicMock()
+    database.db.litellm_objectpermissiontable.find_unique = AsyncMock(
+        side_effect=lambda where: LiteLLM_ObjectPermissionTable(
+            object_permission_id=where["object_permission_id"], vector_stores=["KBSTOREA"]
+        )
+        if where["object_permission_id"] == "key-permission"
+        else None
+    )
+    attrs: Final = {
+        **_proxy_attrs_for_centralized_checks(),
+        "prisma_client": database,
+        "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())),
+        "general_settings": {"vector_store_deny_by_default": deny_by_default},
+    }
+    for name, value in attrs.items():
+        monkeypatch.setattr(litellm.proxy.proxy_server, name, value)
+    monkeypatch.setattr(litellm, "vector_store_registry", None)
+    monkeypatch.setattr(
+        "litellm.proxy.auth.user_api_key_auth.get_team_object", AsyncMock(side_effect=team_lookup_error)
+    )
+
+    checks: Final = _run_centralized_common_checks(
+        user_api_key_auth_obj=token,
+        request=request,
+        request_data={
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "what is in this KB?"}],
+            "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"},
+        },
+        route="/v1/rag/query",
+    )
+    if not denied:
+        await checks
+        return
+    with pytest.raises(ProxyException) as exc_info:
+        await checks
+    assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
+        ProxyErrorTypes.team_vector_store_access_denied,
+        "vector_store",
+        "401",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("team_lookup", "denied"),
+    [
+        ({"team-1": "team-1-grants-a"}, False),
+        (RuntimeError("team cache unavailable"), True),
+        ({"team-1": "team-1-grants-none", "team-2": "team-2-grants-a"}, True),
+    ],
+    ids=["resolved-team-grants", "team-lookup-swallowed-no-personal-fallback", "other-member-team-grants-ignored"],
+)
+async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_team(
+    monkeypatch: pytest.MonkeyPatch, team_lookup: dict[str, str] | Exception, denied: bool
+):
+    from fastapi import Request
+    from starlette.datastructures import URL
+
+    token: Final = UserAPIKeyAuth(
+        user_id="user-1",
+        team_id="team-1",
+        team_models=["gpt-4o-mini"],
+        user_role=LitellmUserRoles.INTERNAL_USER,
+    )
+    request: Final = Request(scope={"type": "http"})
+    request._url = URL(url="/v1/rag/query")
+    grants: Final = {
+        "team-1-grants-a": ["KBSTOREA"],
+        "team-1-grants-none": [],
+        "team-2-grants-a": ["KBSTOREA"],
+        "user-grants-a": ["KBSTOREA"],
+    }
+    database: Final = MagicMock()
+    database.db.litellm_objectpermissiontable.find_unique = AsyncMock(
+        side_effect=lambda where: LiteLLM_ObjectPermissionTable(
+            object_permission_id=where["object_permission_id"], vector_stores=grants[where["object_permission_id"]]
+        )
+    )
+    attrs: Final = {
+        **_proxy_attrs_for_centralized_checks(),
+        "prisma_client": database,
+        "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())),
+        "general_settings": {"vector_store_deny_by_default": True},
+    }
+    for name, value in attrs.items():
+        monkeypatch.setattr(litellm.proxy.proxy_server, name, value)
+    monkeypatch.setattr(litellm, "vector_store_registry", None)
+
+    async def get_team(team_id: str, **_: object) -> LiteLLM_TeamTableCachedObj:
+        if isinstance(team_lookup, Exception):
+            raise team_lookup
+        return LiteLLM_TeamTableCachedObj(team_id=team_id, models=["gpt-4o-mini"], object_permission_id=team_lookup[team_id])
+
+    monkeypatch.setattr("litellm.proxy.auth.user_api_key_auth.get_team_object", get_team)
+    monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_membership", AsyncMock(return_value=None))
+    monkeypatch.setattr(
+        "litellm.proxy.auth.user_api_key_auth.get_user_object",
+        AsyncMock(
+            return_value=LiteLLM_UserTable(
+                user_id="user-1", teams=["team-1", "team-2"], object_permission_id="user-grants-a"
+            )
+        ),
+    )
+
+    checks: Final = _run_centralized_common_checks(
+        user_api_key_auth_obj=token,
+        request=request,
+        request_data={
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "what is in this KB?"}],
+            "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"},
+        },
+        route="/v1/rag/query",
+    )
+    if not denied:
+        await checks
+        return
+    with pytest.raises(ProxyException) as exc_info:
+        await checks
+    assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
+        ProxyErrorTypes.team_vector_store_access_denied,
+        "vector_store",
+        "401",
+    )
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    ("user_permission_id", "denied"),
+    [("admin-grants-a", False), (None, True)],
+    ids=["admin-personal-grant", "admin-without-grant"],
+)
+async def test_keyless_proxy_admin_keeps_personal_vector_store_grants_under_deny_by_default(
+    monkeypatch: pytest.MonkeyPatch, user_permission_id: str | None, denied: bool
+):
+    from fastapi import Request
+    from starlette.datastructures import URL
+
+    token: Final = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN)
+    request: Final = Request(scope={"type": "http"})
+    request._url = URL(url="/v1/rag/query")
+    database: Final = MagicMock()
+    database.db.litellm_objectpermissiontable.find_unique = AsyncMock(
+        side_effect=lambda where: SimpleNamespace(
+            dict=lambda: {"object_permission_id": where["object_permission_id"], "vector_stores": ["KBSTOREA"]},
+            vector_stores=["KBSTOREA"],
+        )
+    )
+    attrs: Final = {
+        **_proxy_attrs_for_centralized_checks(),
+        "prisma_client": database,
+        "general_settings": {"vector_store_deny_by_default": True},
+        "user_api_key_cache": UserApiKeyCache(),
+        "proxy_logging_obj": MagicMock(
+            service_logging_obj=MagicMock(
+                async_service_success_hook=AsyncMock(), async_service_failure_hook=AsyncMock()
+            )
+        ),
+    }
+    for name, value in attrs.items():
+        monkeypatch.setattr(litellm.proxy.proxy_server, name, value)
+    monkeypatch.setattr(litellm, "vector_store_registry", None)
+    monkeypatch.setattr(
+        "litellm.proxy.auth.user_api_key_auth.get_user_object",
+        AsyncMock(
+            return_value=LiteLLM_UserTable(
+                user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN, object_permission_id=user_permission_id
+            )
+        ),
+    )
+
+    checks: Final = _run_centralized_common_checks(
+        user_api_key_auth_obj=token,
+        request=request,
+        request_data={
+            "model": "gpt-4o-mini",
+            "messages": [{"role": "user", "content": "what is in this KB?"}],
+            "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"},
+        },
+        route="/v1/rag/query",
+    )
+    if not denied:
+        await checks
+        return
+    with pytest.raises(ProxyException) as exc_info:
+        await checks
+    assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (
+        ProxyErrorTypes.user_vector_store_access_denied,
+        "vector_store",
+        "401",
+    )
+
+
 @pytest.mark.asyncio
 async def test_centralized_common_checks_propagates_end_user_budget_error():
     """Regression: ``get_end_user_object`` raises ``litellm.BudgetExceededError``

tests/unit/proxy/common_utils/test_callback_config_validation.py

@@ -121,6 +121,137 @@ def test_arize_sampling_rate_rejected_on_non_arize_callback():
     assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None
 
 
+class TestArizeOtlpProtocol:
+    def test_protocol_on_a_non_arize_callback_is_rejected(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            for callback_name in ["langfuse", "langfuse_otel", "datadog", None]:
+                error = callback_config_error(callback_name, {"arize_otlp_protocol": "grpc"})
+                assert error is not None and "applies to the arize callback only" in error
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_unknown_protocols_are_rejected(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            for bad in ["otlp_http", "HTTP/PROTOBUF", "http_json", "", "None"]:
+                error = callback_config_error("arize", {"arize_otlp_protocol": bad})
+                assert error is not None and "arize_otlp_protocol" in error and "http/protobuf" in error
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_protocol_is_rejected_while_otel_v2_is_off(self, monkeypatch):
+        monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"})
+            assert error is not None and "LITELLM_OTEL_V2" in error
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_both_protocols_are_accepted_with_otel_v2_on(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            assert callback_config_error("arize", {"arize_otlp_protocol": "grpc"}) is None
+            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_protocol_is_not_a_family_credential(self):
+        stored = [{"arize_api_key": "k1"}]
+        assert cross_entry_family_error({"arize_otlp_protocol": "grpc"}, stored) is None
+        assert cross_entry_family_error({"arize_otlp_protocol": "http/protobuf"}, stored) is None
+
+    def test_protocol_on_a_key_logging_entry_is_validated(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            invalid_metadata = {
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "success",
+                        "callback_vars": {"arize_otlp_protocol": "carrier-pigeon"},
+                    }
+                ]
+            }
+            error = logging_metadata_config_error(invalid_metadata)
+            assert error is not None and "arize_otlp_protocol" in error
+
+            valid_metadata = {
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "success",
+                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
+                    }
+                ]
+            }
+            assert logging_metadata_config_error(valid_metadata) is None
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_failure_only_arize_callbacks_reject_the_protocol(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "failure")
+            assert error is not None and "arize_otlp_protocol" in error and "failure" in error
+            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success") is None
+            assert (
+                callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success_and_failure") is None
+            )
+            assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_failure_only_key_logging_entry_rejects_the_protocol(self, monkeypatch):
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+
+        is_otel_v2_enabled.cache_clear()
+        try:
+            failure_metadata = {
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "failure",
+                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
+                    }
+                ]
+            }
+            error = logging_metadata_config_error(failure_metadata)
+            assert error is not None and "arize_otlp_protocol" in error
+
+            success_metadata = {
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "success",
+                        "callback_vars": {"arize_otlp_protocol": "http/protobuf"},
+                    }
+                ]
+            }
+            assert logging_metadata_config_error(success_metadata) is None
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+
 def test_arize_sampling_rates_are_not_family_credentials():
     """The rates choose what the Arize family exports, not where it sends, so an
     entry that repeats or adds a rate next to a stored Arize entry is not the

tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py

@@ -1,14 +1,16 @@
 import contextlib
 import json
-from types import SimpleNamespace
+from collections.abc import Callable
 from datetime import datetime
+from types import SimpleNamespace
 from unittest.mock import AsyncMock, MagicMock, patch
 
 import pytest
 from fastapi import HTTPException
 from fastapi.testclient import TestClient
 
-
+# Import proxy_server module first to ensure it's initialized
+import litellm.proxy.proxy_server as ps
 from litellm.proxy._types import (
     LiteLLM_ObjectPermissionTable,
     LiteLLM_TeamTable,
@@ -16,9 +18,6 @@ from litellm.proxy._types import (
     UserAPIKeyAuth,
 )
 
-# Import proxy_server module first to ensure it's initialized
-import litellm.proxy.proxy_server as ps
-
 # Now we can safely import app
 from litellm.proxy.proxy_server import app
 from litellm.types.search import SearchToolInfoResponse
@@ -116,9 +115,7 @@ async def test_list_search_tools_config_only(monkeypatch):
         with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma):
             # Mock proxy_config
             mock_proxy_config = MagicMock()
-            mock_proxy_config.get_config = AsyncMock(
-                return_value={"search_tools": config_tools}
-            )
+            mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools})
             mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools)
             with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config):
                 # Mock auth
@@ -192,9 +189,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch):
         with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma):
             # Mock proxy_config
             mock_proxy_config = MagicMock()
-            mock_proxy_config.get_config = AsyncMock(
-                return_value={"search_tools": config_tools}
-            )
+            mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools})
             mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools)
             with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config):
                 # Mock auth
@@ -215,11 +210,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch):
 
                     # Verify DB tool is present
                     db_tool = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "existing-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "existing-tool"),
                         None,
                     )
                     assert db_tool is not None
@@ -232,11 +223,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch):
 
                     # Verify unique config tool is present
                     config_tool = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "unique-config-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "unique-config-tool"),
                         None,
                     )
                     assert config_tool is not None
@@ -247,8 +234,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch):
                         (
                             t
                             for t in data["search_tools"]
-                            if t["search_tool_name"] == "existing-tool"
-                            and t["is_from_config"] is True
+                            if t["search_tool_name"] == "existing-tool" and t["is_from_config"] is True
                         ),
                         None,
                     )
@@ -323,11 +309,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch):
 
                     # Test datetime conversion for tool 1
                     tool1 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "datetime-test-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "datetime-test-tool"),
                         None,
                     )
                     assert tool1 is not None
@@ -341,11 +323,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch):
 
                     # Test None handling for tool 2
                     tool2 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "null-datetime-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "null-datetime-tool"),
                         None,
                     )
                     assert tool2 is not None
@@ -357,11 +335,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch):
 
                     # Test string passthrough for tool 3
                     tool3 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "string-datetime-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "string-datetime-tool"),
                         None,
                     )
                     assert tool3 is not None
@@ -401,9 +375,7 @@ async def test_list_search_tools_config_error_handling(monkeypatch):
         with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma):
             # Mock proxy_config to raise an error
             mock_proxy_config = MagicMock()
-            mock_proxy_config.get_config = AsyncMock(
-                side_effect=Exception("Config error")
-            )
+            mock_proxy_config.get_config = AsyncMock(side_effect=Exception("Config error"))
             with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config):
                 # Mock auth
                 from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
@@ -422,13 +394,8 @@ async def test_list_search_tools_config_error_handling(monkeypatch):
                     assert len(data["search_tools"]) == 1
                     assert data["search_tools"][0]["search_tool_name"] == "db-tool-1"
                     # Verify masking of sensitive values
-                    assert (
-                        data["search_tools"][0]["litellm_params"]["api_key"]
-                        != "sk-test"
-                    )
-                    assert (
-                        "****" in data["search_tools"][0]["litellm_params"]["api_key"]
-                    )
+                    assert data["search_tools"][0]["litellm_params"]["api_key"] != "sk-test"
+                    assert "****" in data["search_tools"][0]["litellm_params"]["api_key"]
                 finally:
                     app.dependency_overrides.pop(user_api_key_auth, None)
 
@@ -543,31 +510,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch):
 
                     # Test tool 1: api_key should be masked
                     tool1 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "perplexity-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "perplexity-tool"),
                         None,
                     )
                     assert tool1 is not None
-                    assert (
-                        tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef"
-                    )
+                    assert tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef"
                     assert "****" in tool1["litellm_params"]["api_key"]
                     assert tool1["litellm_params"]["search_provider"] == "perplexity"
-                    assert (
-                        tool1["litellm_params"]["api_base"]
-                        == "https://api.perplexity.ai"
-                    )
+                    assert tool1["litellm_params"]["api_base"] == "https://api.perplexity.ai"
 
                     # Test tool 2: api_key should be masked
                     tool2 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "tavily-tool"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "tavily-tool"),
                         None,
                     )
                     assert tool2 is not None
@@ -577,29 +531,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch):
 
                     # Test tool 3: access_token and secret_key should be masked
                     tool3 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "tool-with-token"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-token"),
                         None,
                     )
                     assert tool3 is not None
-                    assert (
-                        tool3["litellm_params"]["access_token"]
-                        != "token-abcdefghijklmnop"
-                    )
+                    assert tool3["litellm_params"]["access_token"] != "token-abcdefghijklmnop"
                     assert "****" in tool3["litellm_params"]["access_token"]
                     assert tool3["litellm_params"]["secret_key"] != "secret-xyz123"
                     assert "****" in tool3["litellm_params"]["secret_key"]
 
                     # Test tool 4: non-sensitive fields should remain unmasked
                     tool4 = next(
-                        (
-                            t
-                            for t in data["search_tools"]
-                            if t["search_tool_name"] == "tool-with-non-sensitive"
-                        ),
+                        (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-non-sensitive"),
                         None,
                     )
                     assert tool4 is not None
@@ -615,6 +558,7 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error():
     """`SearchToolRegistry.get_all_search_tools_from_db` self-heals across one
     ClientNotConnectedError via call_with_db_reconnect_retry."""
     import prisma
+
     from litellm.proxy.search_endpoints.search_tool_registry import (
         SearchToolRegistry,
     )
@@ -628,25 +572,18 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error():
         return []
 
     mock_prisma_client = MagicMock()
-    mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock(
-        side_effect=_flaky_find_many
-    )
+    mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock(side_effect=_flaky_find_many)
     mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=True)
     mock_prisma_client._db_auth_reconnect_timeout_seconds = 2.0
     mock_prisma_client._db_auth_reconnect_lock_timeout_seconds = 0.1
 
-    result = await SearchToolRegistry.get_all_search_tools_from_db(
-        prisma_client=mock_prisma_client
-    )
+    result = await SearchToolRegistry.get_all_search_tools_from_db(prisma_client=mock_prisma_client)
 
     assert result == []
     assert len(invocations) == 2
     mock_prisma_client.attempt_db_reconnect.assert_awaited_once()
     reconnect_kwargs = mock_prisma_client.attempt_db_reconnect.await_args.kwargs
-    assert (
-        reconnect_kwargs["reason"]
-        == "get_all_search_tools_from_db_lookup_failure"
-    )
+    assert reconnect_kwargs["reason"] == "get_all_search_tools_from_db_lookup_failure"
 
 
 @contextlib.contextmanager
@@ -749,9 +686,7 @@ async def test_list_search_tools_scoped_to_key_object_permission():
 @pytest.mark.asyncio
 async def test_list_search_tools_unrestricted_internal_user_sees_all():
     """An internal user with no search_tools allowlist is unrestricted and sees every tool."""
-    unrestricted_user = UserAPIKeyAuth(
-        user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user"
-    )
+    unrestricted_user = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user")
 
     with (
         _mock_search_tool_backend(_scoping_db_tools()),
@@ -791,9 +726,7 @@ async def test_list_search_tools_scoped_to_team_object_permission():
         response = TestClient(app).get("/search_tools/list")
 
     assert response.status_code == 200
-    assert [t["search_tool_name"] for t in response.json()["search_tools"]] == [
-        "db-tool-2"
-    ]
+    assert [t["search_tool_name"] for t in response.json()["search_tools"]] == ["db-tool-2"]
 
 
 @pytest.mark.asyncio
@@ -1061,9 +994,15 @@ def _live_router_and_db(db_rows: list):
     fake_router.search_tools = list(db_rows)
 
     with contextlib.ExitStack() as stack:
-        stack.enter_context(patch("litellm.proxy.proxy_server.prisma_client", MagicMock()))  # test-quality-ok: proxy globals are the only seam; see the module note above
-        stack.enter_context(patch("litellm.proxy.proxy_server.proxy_config", proxy_config))  # test-quality-ok: proxy globals are the only seam; see the module note above
-        stack.enter_context(patch("litellm.proxy.proxy_server.llm_router", fake_router))  # test-quality-ok: proxy globals are the only seam; see the module note above
+        stack.enter_context(
+            patch("litellm.proxy.proxy_server.prisma_client", MagicMock())
+        )  # test-quality-ok: proxy globals are the only seam; see the module note above
+        stack.enter_context(
+            patch("litellm.proxy.proxy_server.proxy_config", proxy_config)
+        )  # test-quality-ok: proxy globals are the only seam; see the module note above
+        stack.enter_context(
+            patch("litellm.proxy.proxy_server.llm_router", fake_router)
+        )  # test-quality-ok: proxy globals are the only seam; see the module note above
         stack.enter_context(
             patch(  # test-quality-ok: proxy globals are the only seam; see the module note above
                 "litellm.proxy.search_endpoints.search_tool_management.SEARCH_TOOL_REGISTRY",
@@ -1152,6 +1091,171 @@ async def test_create_search_tool_survives_a_failing_router_refresh():
     assert response.json()["search_tool_name"] == "tavily-search"
 
 
+def _search_caller(
+    virtual_key: bool,
+    key_search_tools: list[str] | None = None,
+    team_id: str | None = None,
+    user_role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER,
+    api_key: str = "sk-caller",
+) -> UserAPIKeyAuth:
+    caller = UserAPIKeyAuth(
+        api_key=api_key,
+        user_role=user_role,
+        user_id="internal_user",
+        team_id=team_id,
+        object_permission_id=None if key_search_tools is None else "op-key",
+        object_permission=(
+            None
+            if key_search_tools is None
+            else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools)
+        ),
+    )
+    caller.via_virtual_key = virtual_key
+    return caller
+
+
+@pytest.fixture
+def search_permission_cache(monkeypatch: pytest.MonkeyPatch) -> Callable[[list[str] | None], None]:
+    from litellm.proxy._types import LiteLLM_UserTable
+    from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key
+
+    cache = UserApiKeyCache()
+    monkeypatch.setattr(ps, "user_api_key_cache", cache)
+    monkeypatch.setattr(ps, "prisma_client", MagicMock())
+
+    def grant_user(search_tools: list[str] | None) -> None:
+        cache.set_cache(
+            key="internal_user", value=LiteLLM_UserTable(user_id="internal_user", object_permission_id="op-user")
+        )
+        cache.set_cache(
+            key=object_permission_cache_key("op-user"),
+            value=LiteLLM_ObjectPermissionTable(object_permission_id="op-user", search_tools=search_tools),
+        )
+
+    return grant_user
+
+
+@pytest.mark.parametrize(
+    "general_settings, user_search_tools, expected",
+    [
+        ({}, None, ["db-tool-1", "db-tool-2"]),
+        ({"search_tool_deny_by_default": False}, [], ["db-tool-1", "db-tool-2"]),
+        ({"search_tool_deny_by_default": True}, None, []),
+        ({"search_tool_deny_by_default": True}, [], []),
+        ({"search_tool_deny_by_default": True}, ["db-tool-2"], ["db-tool-2"]),
+    ],
+)
+@pytest.mark.asyncio
+async def test_filter_visible_search_tools_keyless_user_follows_search_tool_deny_by_default(
+    search_permission_cache: Callable[[list[str] | None], None],
+    general_settings: dict[str, bool],
+    user_search_tools: list[str] | None,
+    expected: list[str],
+):
+    from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools
+
+    search_permission_cache(user_search_tools)
+    team_lookup = AsyncMock()
+
+    visible = await _filter_visible_search_tools(
+        _search_tool_responses("db-tool-1", "db-tool-2"),
+        _search_caller(virtual_key=False),
+        team_lookup,
+        general_settings,
+    )
+
+    assert [t["search_tool_name"] for t in visible] == expected
+    team_lookup.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+    "key_search_tools, expected",
+    [(None, []), ([], []), (["db-tool-1"], ["db-tool-1"])],
+)
+@pytest.mark.asyncio
+async def test_filter_visible_search_tools_standalone_key_needs_its_own_grant(
+    search_permission_cache: Callable[[list[str] | None], None], key_search_tools: list[str] | None, expected: list[str]
+):
+    from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools
+
+    search_permission_cache(["db-tool-1", "db-tool-2"])
+
+    visible = await _filter_visible_search_tools(
+        _search_tool_responses("db-tool-1", "db-tool-2"),
+        _search_caller(virtual_key=True, key_search_tools=key_search_tools),
+        AsyncMock(),
+        {"search_tool_deny_by_default": True},
+    )
+
+    assert [t["search_tool_name"] for t in visible] == expected
+
+
+@pytest.mark.parametrize(
+    "key_search_tools, team_search_tools, expected",
+    [
+        (["db-tool-1"], [], []),
+        ([], ["db-tool-1"], []),
+        (["db-tool-1", "db-tool-2"], ["db-tool-1"], ["db-tool-1"]),
+    ],
+)
+@pytest.mark.asyncio
+async def test_filter_visible_search_tools_team_key_needs_key_and_team_grants(
+    search_permission_cache: Callable[[list[str] | None], None],
+    key_search_tools: list[str],
+    team_search_tools: list[str],
+    expected: list[str],
+):
+    from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools
+
+    team_lookup = AsyncMock(
+        return_value=LiteLLM_TeamTable(
+            team_id="team-1",
+            object_permission_id="op-team",
+            object_permission=LiteLLM_ObjectPermissionTable(
+                object_permission_id="op-team", search_tools=team_search_tools
+            ),
+        )
+    )
+
+    visible = await _filter_visible_search_tools(
+        _search_tool_responses("db-tool-1", "db-tool-2"),
+        _search_caller(virtual_key=True, key_search_tools=key_search_tools, team_id="team-1"),
+        team_lookup,
+        {"search_tool_deny_by_default": True},
+    )
+
+    assert [t["search_tool_name"] for t in visible] == expected
+
+
+@pytest.mark.parametrize(
+    "caller, expected",
+    [
+        (
+            _search_caller(
+                virtual_key=False, user_role=LitellmUserRoles.PROXY_ADMIN, api_key="litellm_proxy_master_key"
+            ),
+            ["db-tool-1", "db-tool-2"],
+        ),
+        (_search_caller(virtual_key=True, user_role=LitellmUserRoles.PROXY_ADMIN), []),
+    ],
+    ids=["master-key-sees-every-tool", "admin-virtual-key-is-filtered"],
+)
+@pytest.mark.asyncio
+async def test_filter_visible_search_tools_admin_under_search_tool_deny_by_default(
+    search_permission_cache: Callable[[list[str] | None], None], caller: UserAPIKeyAuth, expected: list[str]
+):
+    from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools
+
+    visible = await _filter_visible_search_tools(
+        _search_tool_responses("db-tool-1", "db-tool-2"),
+        caller,
+        AsyncMock(),
+        {"search_tool_deny_by_default": True},
+    )
+
+    assert [t["search_tool_name"] for t in visible] == expected
+
+
 class _StoredSearchToolRow(SimpleNamespace):
     def __iter__(self):
         return iter(self.__dict__.items())

tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py

@@ -361,6 +361,70 @@ class TestNewRelicTeamCallbackValidation:
             is_otel_v2_enabled.cache_clear()
 
 
+class TestArizeOtlpProtocolValidation:
+    def test_valid_protocols_are_accepted(self):
+        from litellm.proxy._types import AddTeamCallback
+
+        for protocol in ("grpc", "http/protobuf"):
+            data = AddTeamCallback(
+                callback_name="arize",
+                callback_type="success",
+                callback_vars={"arize_api_key": "k", "arize_space_id": "s", "arize_otlp_protocol": protocol},
+            )
+            assert data.callback_vars["arize_otlp_protocol"] == protocol
+
+    def test_an_invalid_protocol_is_rejected_at_model_validation(self):
+        from pydantic import ValidationError
+
+        from litellm.proxy._types import AddTeamCallback
+
+        with pytest.raises(ValidationError, match="arize_otlp_protocol"):
+            AddTeamCallback(
+                callback_name="arize",
+                callback_type="success",
+                callback_vars={"arize_otlp_protocol": "otlp_grpc"},
+            )
+
+    def test_a_failure_only_team_callback_rejects_the_protocol(self, monkeypatch):
+        from fastapi import HTTPException
+
+        from litellm.integrations.otel.model.config import is_otel_v2_enabled
+        from litellm.proxy._types import AddTeamCallback
+        from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback
+
+        monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+        is_otel_v2_enabled.cache_clear()
+        try:
+            with pytest.raises(HTTPException) as exc:
+                _validate_team_callback(
+                    AddTeamCallback(
+                        callback_name="arize",
+                        callback_type="failure",
+                        callback_vars={
+                            "arize_api_key": "k",
+                            "arize_space_id": "s",
+                            "arize_otlp_protocol": "http/protobuf",
+                        },
+                    )
+                )
+            assert exc.value.status_code == 400
+            assert "arize_otlp_protocol" in str(exc.value.detail)
+        finally:
+            is_otel_v2_enabled.cache_clear()
+
+    def test_the_raw_otlp_env_var_name_stays_rejected(self):
+        from pydantic import ValidationError
+
+        from litellm.proxy._types import AddTeamCallback
+
+        with pytest.raises(ValidationError, match="Invalid callback variable"):
+            AddTeamCallback(
+                callback_name="arize",
+                callback_type="success",
+                callback_vars={"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"},
+            )
+
+
 class TestNewRelicKeyLoggingValidation:
     """Key-level logging is written through key metadata, not /team/callback."""
 

tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py

@@ -4066,6 +4066,34 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker):
     }
 
 
+@pytest.mark.asyncio
+async def test_user_update_broadcasts_the_entitlement_invalidation_to_other_workers(mocker: MockerFixture):
+    from litellm.proxy.management_endpoints.internal_user_endpoints import (
+        _update_single_user_helper,
+    )
+
+    _object_permission_mocks(mocker)
+    cache: Final = mocker.MagicMock()
+    cache.async_delete_cache = mocker.AsyncMock()
+    mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache)  # test-quality-ok: substitute the cache dependency
+    broadcast: Final = mocker.patch(  # test-quality-ok: observe the Redis publication boundary
+        "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation",
+        new_callable=mocker.AsyncMock,
+    )
+
+    await _update_single_user_helper(
+        user_request=UpdateUserRequest(user_id="target-user", object_permission={"vector_stores": []}),
+        user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN),
+    )
+
+    broadcast_keys: Final = {call.kwargs["cache_key"] for call in broadcast.await_args_list}
+    assert broadcast_keys == {
+        "object_permission_id:perm-new",
+        "user_object_permission_id:target-user",
+        "target-user",
+    }
+
+
 @pytest.mark.asyncio
 async def test_admin_can_clear_a_users_mcp_entitlement(mocker):
     """An explicit empty object_permission means "no object permission", so it must unlink.

tests/unit/proxy/management_endpoints/test_team_endpoints.py

@@ -8417,6 +8417,7 @@ async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit(
         # Mock team update
         mock_updated_team = MagicMock(spec=LiteLLM_TeamTable)
         mock_updated_team.team_id = "org-team-update-bypass-123"
+        mock_updated_team.object_permission_id = None
         mock_updated_team.tpm_limit = 10000
         mock_updated_team.rpm_limit = 1000
         mock_updated_team.access_group_ids = None
@@ -8570,6 +8571,7 @@ async def test_update_team_guardrails_with_org_id(
         # Mock team update
         mock_updated_team = MagicMock(spec=LiteLLM_TeamTable)
         mock_updated_team.team_id = "team-guardrails-123"
+        mock_updated_team.object_permission_id = None
         mock_updated_team.organization_id = "test-org-guardrails"
         mock_updated_team.metadata = {
             "guardrails": ["aporia-pre-call", "aporia-post-call"]

tests/unit/proxy/proxy_server/test_proxy_config.py

@@ -26,7 +26,7 @@ import pytest
 from pydantic import JsonValue, TypeAdapter, ValidationError
 
 import litellm
-from litellm.proxy._types import CommonProxyErrors
+from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings
 from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper
 from litellm.proxy.proxy_server import (
     ProxyConfig,
@@ -2329,6 +2329,40 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke
     assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else [])
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"])
+@pytest.mark.parametrize(("yaml_value", "expected"), [("true", True), ("false", False)])
+async def test_load_config_yaml_deny_by_default_is_boolean(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str, expected: bool
+):
+    config_file: Final = tmp_path / "deny_by_default.yaml"
+    config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n  {setting}: {yaml_value}\n")
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+
+    _, _, general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
+
+    assert general_settings[setting] is expected
+    assert getattr(ConfigGeneralSettings.model_validate(dict(general_settings)), setting) is expected
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"])
+@pytest.mark.parametrize("yaml_value", ["", "enabled"], ids=["null", "string"])
+async def test_load_config_rejects_non_boolean_deny_by_default(
+    tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str
+):
+    config_file: Final = tmp_path / "deny_by_default.yaml"
+    config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n  {setting}: {yaml_value}\n")
+    monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None)
+    monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False)
+    monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False)
+
+    with pytest.raises(ValidationError, match=setting):
+        await ProxyConfig().load_config(router=None, config_file_path=str(config_file))
+
+
 @pytest.mark.asyncio
 async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch):
     """Regression: router_settings.plugins dotted-path strings must be resolved to
@@ -3616,9 +3650,7 @@ def test_ProxyConfig__decrypt_and_set_db_env_variables_cannot_enable_mcp_stdio(m
     assert os.environ.get("LITELLM_ENABLE_MCP_STDIO") is None
 
 
-def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag(
-    monkeypatch, caplog
-):
+def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag(monkeypatch, caplog):
     monkeypatch.setattr(
         "litellm.proxy.proxy_server.decrypt_value_helper",
         lambda value, key, return_original_value=False: value,

tests/unit/proxy/search_endpoints/test_endpoints.py

@@ -1,12 +1,142 @@
+from collections.abc import Iterator
+from typing import Final
 from unittest.mock import AsyncMock, MagicMock
 
 import orjson
 import pytest
+import respx
+from fastapi import FastAPI
+from fastapi.testclient import TestClient
 
+import litellm
+from litellm import Router
 from litellm.proxy import proxy_server
-from litellm.proxy._types import UserAPIKeyAuth
+from litellm.proxy._types import (
+    LiteLLM_ObjectPermissionTable,
+    LiteLLM_TeamTableCachedObj,
+    LitellmUserRoles,
+    ProxyException,
+    UserAPIKeyAuth,
+)
+from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
+from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache
 from litellm.proxy.route_llm_request import ProxyMissingRequiredParamError
-from litellm.proxy.search_endpoints.endpoints import search
+from litellm.proxy.search_endpoints.endpoints import router, search
+
+TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search"
+TAVILY_RESULT: Final = {"title": "LiteLLM", "url": "https://docs.litellm.ai", "content": "LLM gateway"}
+
+
+def _search_router() -> Router:
+    return Router(
+        model_list=[],
+        search_tools=[
+            {"search_tool_name": "search-a", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}},
+        ],
+        num_retries=0,
+    )
+
+
+def _client(caller: UserAPIKeyAuth) -> TestClient:
+    app: Final = FastAPI()
+    app.include_router(router)
+    app.add_exception_handler(ProxyException, proxy_server.openai_exception_handler)
+    app.dependency_overrides[user_api_key_auth] = lambda: caller
+    return TestClient(app, raise_server_exceptions=False)
+
+
+def _team_key(key_search_tools: list[str]) -> UserAPIKeyAuth:
+    caller: Final = UserAPIKeyAuth(
+        api_key="sk-team-key",
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        user_id="user-1",
+        team_id="team-1",
+        object_permission_id="op-key",
+        object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools),
+    )
+    caller.via_virtual_key = True
+    return caller
+
+
+@pytest.fixture
+def tavily(monkeypatch: pytest.MonkeyPatch) -> Iterator[respx.Route]:
+    monkeypatch.setattr(  # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary.
+        litellm,
+        "disable_aiohttp_transport",
+        True,
+    )
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    with respx.mock(assert_all_called=False) as mock:
+        yield mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]})
+    litellm.in_memory_llm_clients_cache.flush_cache()
+
+
+@pytest.fixture
+def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache:
+    user_api_key_cache: Final = UserApiKeyCache()
+    monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache)
+    monkeypatch.setattr(proxy_server, "prisma_client", MagicMock())
+    monkeypatch.setattr(proxy_server, "llm_router", _search_router())
+    return user_api_key_cache
+
+
+def _cache_team(cache: UserApiKeyCache, search_tools: list[str]) -> None:
+    team: Final = LiteLLM_TeamTableCachedObj(
+        team_id="team-1",
+        object_permission_id="op-team",
+        object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=search_tools),
+    )
+    cache.set_cache(key="team_id:team-1", value=team)
+
+
+@pytest.mark.parametrize(
+    "general_settings, key_search_tools, team_search_tools, expected_status",
+    [
+        ({}, [], [], 200),
+        ({"search_tool_deny_by_default": False}, [], [], 200),
+        ({"search_tool_deny_by_default": True}, ["search-a"], [], 403),
+        ({"search_tool_deny_by_default": True}, [], ["search-a"], 403),
+        ({"search_tool_deny_by_default": True}, ["search-a"], ["search-b"], 403),
+        ({"search_tool_deny_by_default": True}, ["search-a"], ["search-a"], 200),
+    ],
+)
+@pytest.mark.parametrize("path", ["/v1/search/search-a", "/search/search-a"])
+def test_direct_search_team_key_follows_search_tool_deny_by_default(
+    monkeypatch: pytest.MonkeyPatch,
+    cache: UserApiKeyCache,
+    tavily: respx.Route,
+    path: str,
+    general_settings: dict[str, bool],
+    key_search_tools: list[str],
+    team_search_tools: list[str],
+    expected_status: int,
+):
+    monkeypatch.setattr(proxy_server, "general_settings", general_settings)
+    _cache_team(cache, team_search_tools)
+
+    response: Final = _client(_team_key(key_search_tools)).post(path, json={"query": "what is litellm"})
+
+    assert response.status_code == expected_status, response.text
+    if expected_status == 200:
+        assert response.json()["results"][0]["url"] == TAVILY_RESULT["url"]
+        assert tavily.call_count == 1
+    else:
+        assert "search-a" in response.text
+        assert tavily.call_count == 0
+
+
+def test_direct_search_body_tool_name_is_denied_under_search_tool_deny_by_default(monkeypatch, cache, tavily):
+    monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True})
+    caller: Final = UserAPIKeyAuth(api_key="sk-standalone", user_role=LitellmUserRoles.INTERNAL_USER)
+    caller.via_virtual_key = True
+
+    response: Final = _client(caller).post(
+        "/v1/search", json={"search_tool_name": "search-a", "query": "what is litellm"}
+    )
+
+    assert response.status_code == 403, response.text
+    assert response.json()["error"]["type"] == "key_search_tool_access_denied"
+    assert tavily.call_count == 0
 
 
 def _json_request(body: dict[str, object]) -> MagicMock:
@@ -52,3 +182,104 @@ async def test_search_with_only_a_query_falls_back_to_the_proxy_default_model(mo
     router.asearch.assert_awaited_once()
     assert router.asearch.await_args.kwargs["query"] == "litellm"
     assert router.asearch.await_args.kwargs["model"] == "perplexity-search"
+
+
+def _standalone_key(key_search_tools: list[str] | None) -> UserAPIKeyAuth:
+    caller: Final = UserAPIKeyAuth(
+        api_key="sk-standalone",
+        user_role=LitellmUserRoles.INTERNAL_USER,
+        object_permission_id=None if key_search_tools is None else "op-key",
+        object_permission=(
+            None
+            if key_search_tools is None
+            else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools)
+        ),
+    )
+    caller.via_virtual_key = True
+    return caller
+
+
+@pytest.mark.parametrize(
+    "general_settings, body",
+    [
+        ({"search_tool_deny_by_default": True}, {"model": "search-a", "query": "what is litellm"}),
+        (
+            {"search_tool_deny_by_default": True, "completion_model": "search-a"},
+            {"query": "what is litellm"},
+        ),
+    ],
+    ids=["body-model", "completion-model"],
+)
+@pytest.mark.parametrize("key_search_tools, expected_status", [(None, 403), (["search-a"], 200)])
+def test_direct_search_authorizes_the_tool_resolved_from_model_settings(
+    monkeypatch, cache, tavily, general_settings, body, key_search_tools, expected_status
+):
+    monkeypatch.setattr(proxy_server, "general_settings", general_settings)
+
+    response: Final = _client(_standalone_key(key_search_tools)).post("/v1/search", json=body)
+
+    assert response.status_code == expected_status, response.text
+    assert tavily.call_count == (1 if expected_status == 200 else 0)
+
+
+@pytest.mark.parametrize(
+    "key_search_tools, expected_status, fallback_calls",
+    [(["search-a"], 500, 0), (["search-a", "search-b"], 200, 1)],
+)
+def test_router_search_fallback_target_must_be_granted(
+    monkeypatch, cache, key_search_tools, expected_status, fallback_calls
+):
+    from litellm.proxy.auth.fallback_model_access import router_fallback_access_check
+
+    monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True})
+    monkeypatch.setattr(  # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary.
+        litellm,
+        "disable_aiohttp_transport",
+        True,
+    )
+    monkeypatch.setattr(
+        proxy_server,
+        "llm_router",
+        Router(
+            model_list=[],
+            search_tools=[
+                {"search_tool_name": "search-a", "litellm_params": {"search_provider": "exa_ai", "api_key": "fake"}},
+                {"search_tool_name": "search-b", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}},
+            ],
+            fallbacks=[{"search-a": ["search-b"]}],
+            fallback_access_check=router_fallback_access_check,
+            num_retries=0,
+        ),
+    )
+    litellm.in_memory_llm_clients_cache.flush_cache()
+    with respx.mock(assert_all_called=False) as mock:
+        failing_tool: Final = mock.post(url__regex=r"https://api\.exa\.ai/.*").respond(500, json={"error": "down"})
+        fallback_tool: Final = mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]})
+        response: Final = _client(_standalone_key(key_search_tools)).post(
+            "/v1/search/search-a", json={"query": "what is litellm"}
+        )
+    litellm.in_memory_llm_clients_cache.flush_cache()
+
+    assert response.status_code == expected_status, response.text
+    assert failing_tool.call_count == 1
+    assert fallback_tool.call_count == fallback_calls
+
+
+@pytest.mark.parametrize(
+    "requested_tool, completion_model, expected_status",
+    [("search-b", "search-a", 403), ("search-a", "search-b", 200)],
+    ids=["requested-tool-ungranted", "completion-model-ungranted"],
+)
+def test_direct_search_authorizes_the_requested_tool_over_completion_model(
+    monkeypatch, cache, tavily, requested_tool, completion_model, expected_status
+):
+    monkeypatch.setattr(
+        proxy_server, "general_settings", {"search_tool_deny_by_default": True, "completion_model": completion_model}
+    )
+
+    response: Final = _client(_standalone_key(["search-a"])).post(
+        f"/v1/search/{requested_tool}", json={"query": "what is litellm"}
+    )
+
+    assert response.status_code == expected_status, response.text
+    assert tavily.call_count == (1 if expected_status == 200 else 0)

tests/unit/proxy/test_credential_slot_registry.py

@@ -89,6 +89,7 @@ CALLBACK_PARAM_CLASSIFICATION: Final[Mapping[str, Classification]] = MappingProx
         "arize_space_id": NotSecret("space identifier"),
         "arize_success_sampling_rate": NotSecret("sampling rate"),
         "arize_error_sampling_rate": NotSecret("sampling rate"),
+        "arize_otlp_protocol": NotSecret("transport protocol enum"),
         "posthog_api_key": Unplanted(),
         "posthog_api_url": NotSecret("sink endpoint URL"),
         "wandb_api_key": Unplanted(),

tests/unit/proxy/test_litellm_pre_call_utils.py

@@ -8590,3 +8590,38 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback():
         team_callback_settings_obj=None,
     )
     assert under_other.callback_vars == {"langfuse_host": "https://cloud.langfuse.com"}
+
+
+def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monkeypatch):
+    from litellm.integrations.otel.model.config import is_otel_v2_enabled
+    from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations
+
+    monkeypatch.setenv("LITELLM_OTEL_V2", "true")
+    monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1")
+    monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False)
+    is_otel_v2_enabled.cache_clear()
+    try:
+        auth = UserAPIKeyAuth(
+            api_key="hashed-key",
+            metadata={
+                "logging": [
+                    {
+                        "callback_name": "arize",
+                        "callback_type": "success",
+                        "callback_vars": {
+                            "arize_space_id": "s",
+                            "arize_api_key": "k",
+                            "arize_otlp_protocol": "http/protobuf",
+                        },
+                    }
+                ]
+            },
+            team_metadata={},
+        )
+
+        destinations = resolve_tenant_otel_destinations(auth)
+
+        assert [d.protocol for d in destinations] == ["otlp_http"]
+        assert destinations[0].endpoint == "https://arize.internal.example/v1/traces"
+    finally:
+        is_otel_v2_enabled.cache_clear()

tests/unit/responses/test_streaming_iterator.py

@@ -646,7 +646,15 @@ def test_stream_cache_write_completes_when_asyncio_run_closes_the_loop(monkeypat
             status="completed",
             model="test-model",
             object="response",
-            output=[],
+            output=[
+                {
+                    "type": "message",
+                    "id": "msg_lit6184",
+                    "status": "completed",
+                    "role": "assistant",
+                    "content": [{"type": "output_text", "text": "cached", "annotations": []}],
+                }
+            ],
         ),
     )
     monkeypatch.setattr(litellm, "cache", _SlowWriteCache())
@@ -1261,6 +1269,25 @@ def test_billed_terminal_response_copies_when_estimating_and_leaves_the_original
     assert response.usage is None
 
 
+def test_persist_completed_response_to_cache_skips_a_response_without_output(monkeypatch):
+    logging_obj: Final = _logging_obj_stub()
+    caching_handler: Final = Mock()
+    caching_handler.request_kwargs = {"stream": True}
+    logging_obj._llm_caching_handler = caching_handler
+    iterator: Final = _make_iterator(sse_events=[], logging_obj=logging_obj)
+    iterator.completed_response = ResponseCompletedEvent.model_construct(
+        type="response.completed",
+        response=ResponsesAPIResponse.model_construct(id="resp_empty", output=[], usage=None),
+    )
+    cache: Final = Mock()
+    monkeypatch.setattr(litellm, "cache", cache)
+
+    iterator._persist_completed_response_to_cache(is_async=False)
+
+    cache.add_cache.assert_not_called()
+    caching_handler._should_store_result_in_cache.assert_not_called()
+
+
 def test_persist_completed_response_to_cache_survives_an_unserializable_response(monkeypatch):
     bad_response: Final = ResponsesAPIResponse.model_construct(id="r", output=[object()], usage=None)
     with pytest.raises(PydanticSerializationError):

tests/unit/rust_bridge/messages/test_route_host.py

@@ -161,4 +161,4 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl
     assert isinstance(additional, dict)
     assert additional["llm_provider-request-id"] == "req_upstream_123"
     assert additional["x-ratelimit-remaining-requests"] == "41"
-    assert "request-id" not in additional
+    assert additional["request-id"] == additional["llm_provider-request-id"]

tests/unit/rust_bridge/test_callbacks_legacy_python.py

@@ -34,6 +34,14 @@ def _supplied_logger() -> Logging:
     )
 
 
+def test_native_stream_headers_reach_spend_callbacks() -> None:
+    logger: Final = _supplied_logger()
+    legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}})
+
+    assert logger.stream is True
+    assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"}
+
+
 def test_setup_reuses_a_supplied_logger() -> None:
     supplied: Final = _supplied_logger()
     result: Final = setup(
@@ -159,9 +167,7 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation
     pending.close()
 
 
-CONTRACT_PATH: Final = (
-    Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
-)
+CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json"
 
 
 def test_the_rust_contract_matches_the_shim_signatures() -> None:

ui/litellm-dashboard/src/components/callback_info_helpers.tsx

@@ -20,6 +20,7 @@ interface CallbackConfig {
   supports_key_team_logging: boolean;
   dynamic_params: Record<string, "text" | "password" | "select" | "upload" | "number">;
   dynamic_param_options?: Record<string, readonly string[]>;
+  success_event_params?: string[];
   description: string;
 }
 
@@ -34,7 +35,12 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [
       arize_space_id: "password",
       arize_success_sampling_rate: "number",
       arize_error_sampling_rate: "number",
+      arize_otlp_protocol: "select",
     },
+    dynamic_param_options: {
+      arize_otlp_protocol: ["grpc", "http/protobuf"],
+    },
+    success_event_params: ["arize_otlp_protocol"],
     description: "Arize Logging Integration",
   },
   {

ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts

@@ -26,6 +26,8 @@ function makeTrace(scene: Scenario, index: number, now: number) {
     parent_span_id: null,
     span_id: spanId(0),
     spend: null,
+    spend_log_request_id: null,
+    spend_match: null,
     start_offset_ms: 0,
     status: "ok",
     type: "agent",
@@ -53,6 +55,7 @@ function makeTrace(scene: Scenario, index: number, now: number) {
     input_tokens: 520 + index * 41,
     output_tokens: 48 + index * 7,
     spend: 0.003 + index * 0.0002,
+    spend_match: "matched",
   };
   const trace: Trace = {
     summary: {
@@ -71,6 +74,7 @@ function makeTrace(scene: Scenario, index: number, now: number) {
       service: "demo-agents",
       span_count: toolCount + 2,
       spend: model.spend,
+      priced_calls: 1,
       start_time: iso(now - (index + 1) * 35 * 60_000),
       status: scene.failed ? "error" : "ok",
       tool_calls: toolCount,
@@ -85,6 +89,7 @@ function makeTrace(scene: Scenario, index: number, now: number) {
         llm_calls: 1,
         tool_calls: toolCount,
         spend: model.spend,
+        priced_calls: 1,
       },
     ],
     spans: [base, ...tools, model],

ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts

@@ -85,6 +85,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) {
     input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0),
     output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0),
     spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0),
+    priced_calls: models.length,
   };
   const finalDetail = run.details.at(-1)!;
   const history = caseDetails.filter((_, index) => index % 3 === 2);
@@ -102,6 +103,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) {
           llm_calls: caseCount,
           tool_calls: caseCount,
           spend: summary.spend - (final.spend ?? 0),
+          priced_calls: caseCount,
         },
       ],
     },