diff --git a/litellm-rust/crates/callbacks-legacy-python/python_contract.json b/litellm-rust/crates/callbacks-legacy-python/python_contract.json index 9ed13ae5ed5..f65516265c5 100644 --- a/litellm-rust/crates/callbacks-legacy-python/python_contract.json +++ b/litellm-rust/crates/callbacks-legacy-python/python_contract.json @@ -88,7 +88,8 @@ "call_type" ], "stream_opened": [ - "logger" + "logger", + "head" ], "stream_success": [ "logger", diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 21f563d9f3b..70b7ecb45fd 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -536,7 +536,7 @@ impl LegacyLogging { head.bind(py).set_item("cache_key", key)?; head.bind(py).set_item("cache_hit", true)?; } - Streaming::Opened.call(py, (self.logger()?.object(py),))?; + Streaming::Opened.call(py, (self.logger()?.object(py), head.bind(py)))?; self.stream = Some(DeliveredStream { chunks: PyList::empty(py).unbind(), first_chunk: None, @@ -1768,13 +1768,13 @@ assert logger.calls[1][1] is response fn stream_bindings_deliver_collected_chunks_in_order_without_success_fan_out() { Python::initialize(); Python::attach(|py| { - let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None"); + let locals = namespace(py, c"first = b'first'\nlast = b'last'\nresponse = None\nhead = {'additional_headers': {'request-id': 'req_native'}}"); let mut logging = LegacyLogging { operation: crate::LoggingOperation::Messages, ..logged(py, &locals, true) }; logging - .on_stream_open(py, &pyo3::types::PyDict::new(py).into_any().unbind()) + .on_stream_open(py, &local(&locals, "head").unbind()) .unwrap(); logging .on_stream_chunk(py, &local(&locals, "first").unbind()) @@ -1791,6 +1791,7 @@ assert logger.calls[1][1] is response &locals, c" assert logger.names() == ['stream_opened', 'stream_success'], logger.calls +assert logger.calls[0][1] is head chunks = logger.calls[1][1] assert len(chunks) == 2 assert chunks[0] is first diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index 46c93369100..f252a45b562 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -84,7 +84,7 @@ FAKES = { 'success', response, call_type ), 'after_deployment_failure': lambda kwargs, error, call_type: kwargs['logger'].hook('failure', error, call_type), - 'stream_opened': lambda logger: logger.record('stream_opened', None), + 'stream_opened': lambda logger, head: logger.record('stream_opened', head), 'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk: logger.record( 'stream_success', list(chunks) ), diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index f308714146e..a5526a39963 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -298,6 +298,7 @@ mod tests { fn row(span_id: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: String::new(), name: "run".into(), diff --git a/litellm-rust/crates/traces-cache/src/spend.rs b/litellm-rust/crates/traces-cache/src/spend.rs index 1cfcde841d0..6ea2cc79257 100644 --- a/litellm-rust/crates/traces-cache/src/spend.rs +++ b/litellm-rust/crates/traces-cache/src/spend.rs @@ -48,6 +48,7 @@ pub(super) async fn spend( let params = SpendByResponseIdsParams { access: access.clone(), response_ids: lookup.response_ids, + provider_request_ids: lookup.provider_request_ids, request_ids: lookup.request_ids, trace_ids: lookup.trace_ids, start_ms: window.start, diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index 27dc612f3e6..02f11fdd1b7 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -270,6 +270,7 @@ fn access() -> ReadAccessParams { fn span(index: usize) -> TraceSpansRow { TraceSpansRow { trace_id: "trace".into(), + original_trace_id: String::new(), span_id: format!("span-{index}"), parent_span_id: if index == 0 { String::new() @@ -533,6 +534,7 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { litellm_call_id: String::new(), response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), team_id: "team".into(), diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index 42085d3e95b..0a206004080 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -17,6 +17,7 @@ const TTL: Duration = Duration::from_secs(120); fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), diff --git a/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql new file mode 100644 index 00000000000..443ad0688e6 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/migrations/0016_spend_provider_request_id.sql @@ -0,0 +1,4 @@ +ALTER TABLE {database}.spend_logs + ADD COLUMN IF NOT EXISTS provider_request_id String DEFAULT '' AFTER response_id, + ADD INDEX IF NOT EXISTS idx_provider_request_id provider_request_id + TYPE bloom_filter(0.001) GRANULARITY 1 diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql index 3918286a61f..55f8fe88a88 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -15,7 +15,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql index da64dafbc39..e6402fff6cf 100644 --- a/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql +++ b/litellm-rust/crates/traces-clickhouse/query/spend_by_response_ids.sql @@ -1,4 +1,4 @@ -SELECT request_id, litellm_call_id, response_id, upstream_response_id, trace_id, span_id, team_id, api_key, user, spend, +SELECT request_id, litellm_call_id, response_id, upstream_response_id, provider_request_id, trace_id, span_id, team_id, api_key, user, spend, toUnixTimestamp64Milli(start_time) AS start_ms FROM ( SELECT *, @@ -14,7 +14,8 @@ FROM ( OR ({user_id:String} != '' AND user = {user_id:String}) OR has({team_ids:Array(String)}, team_id)) ) -WHERE response_id IN {response_ids:Array(String)} +WHERE provider_request_id IN {provider_request_ids:Array(String)} + OR response_id IN {response_ids:Array(String)} OR upstream_response_id IN {response_ids:Array(String)} OR litellm_call_id IN {request_ids:Array(String)} OR (litellm_call_id = '' AND request_id IN {request_ids:Array(String)}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql index 679edfdec2e..d6ab416dfd3 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql index b81bba61e7c..27a8e2ac0da 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql index 967ef2fcf1f..a084325feaa 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql @@ -1,5 +1,5 @@ SELECT * FROM ( -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index 2e0ac4f6dfb..306692e9709 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -1,4 +1,4 @@ -SELECT o.TraceId AS trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, +SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS original_trace_id, o.SpanId AS span_id, o.ParentSpanId AS parent_span_id, o.SpanName AS name, o.ObservationType AS type, toUInt8(o.WrapperCandidate) AS wrapper_candidate, o.AgentName AS agent, o.Framework AS framework, o.StatusCode AS status, substringUTF8(o.StatusMessage, 1, 128) AS status_message, diff --git a/litellm-rust/crates/traces-clickhouse/src/query/named.rs b/litellm-rust/crates/traces-clickhouse/src/query/named.rs index 84973b8b756..ff64259ec33 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/named.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/named.rs @@ -82,6 +82,8 @@ pub use contracts::TraceSpansParams; struct TraceSpansRowEncoding { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -181,6 +183,7 @@ struct SpendByResponseIdsParamsEncoding { #[serde(flatten)] pub access: contracts::ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, #[serde(deserialize_with = "super::number::deserialize")] @@ -207,6 +210,8 @@ struct SpendByResponseIdsRowEncoding { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, @@ -359,7 +364,7 @@ mod tests { quoted, ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), quoted, ); round_trip::( @@ -371,7 +376,7 @@ mod tests { quoted, ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), quoted, ); } @@ -389,7 +394,7 @@ mod tests { quoted, ); round_trip::( - json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"all_teams": 0, "user_id": "user", "team_ids": ["team-a", "team-b"], "response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), quoted, ); } diff --git a/litellm-rust/crates/traces-clickhouse/src/span_row.rs b/litellm-rust/crates/traces-clickhouse/src/span_row.rs index 0189dbff2c8..ad2863d6c5d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/span_row.rs +++ b/litellm-rust/crates/traces-clickhouse/src/span_row.rs @@ -211,7 +211,10 @@ fn request_id(evidence: &CallEvidence) -> &str { .flatten() .find_map(|key| match key { CallKey::ProviderResponse(id) => Some(id.as_str()), - CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, + CallKey::LiteLlmRequest(_) + | CallKey::ProviderRequest(_) + | CallKey::Transport + | CallKey::GatewayAttempt => None, }) .unwrap_or_default() } diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index d90c366118b..e51d9083c59 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -211,6 +211,10 @@ async fn schema_supports_span_rollups_and_spend_joins( Parameter::Strings(vec!["response-1".into()]), ), ("request_ids".into(), Parameter::Strings(Vec::new())), + ( + "provider_request_ids".into(), + Parameter::Strings(Vec::new()), + ), ("trace_ids".into(), Parameter::Strings(Vec::new())), ("all_teams".into(), Parameter::Integer(0)), ("user_id".into(), Parameter::Text(String::new())), @@ -2223,6 +2227,7 @@ async fn named_and_sql_readers_share_request_log_visibility( "api_key_hash": legacy_key.unwrap_or_default(), }))?, response_ids: vec!["shared-response".into()], + provider_request_ids: Vec::new(), request_ids: Vec::new(), trace_ids: Vec::new(), start_ms: timestamp / 1_000_000 - 1, diff --git a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs index 371424c63d6..7532d9aecc9 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/queries/support.rs @@ -158,7 +158,8 @@ fn span_row(span: &DecodedSpan, team: &str, key: &str) -> BTreeMap Some(id.as_str()), - litellm_traces::CallKey::Transport + litellm_traces::CallKey::ProviderRequest(_) + | litellm_traces::CallKey::Transport | litellm_traces::CallKey::GatewayAttempt => None, }) .unwrap_or_default() diff --git a/litellm-rust/crates/traces-clickhouse/tests/reads.rs b/litellm-rust/crates/traces-clickhouse/tests/reads.rs index aef6f956231..48978fc3611 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/reads.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/reads.rs @@ -601,7 +601,7 @@ async fn an_oversized_span_keeps_the_run_list_available_with_partial_totals( #[rstest] #[tokio::test] -async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback( +async fn assigned_call_ids_require_shared_ownership_through_detail_and_batch_reads( #[future(awt)] migrated_database: TestResult, ) -> TestResult { let fixture = migrated_database?; @@ -610,31 +610,41 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let start_ms = 1_790_000_000_000_i64; let cases = [ ( - "gateway", - "provider-request", - "gateway", + "same-key", + "provider_response:same-key", "team-a", "key-a", Some(0.25), ), - ("legacy", "legacy", "", "team-a", "key-a", Some(0.25)), - ("conflict", "conflict", "different", "team-a", "key-a", None), + ( + "other-key", + "provider_response:other-key", + "team-a", + "key-b", + None, + ), ( "foreign-team", - "request", - "foreign-team", + "provider_response:foreign-team", "team-b", "key-a", None, ), ( - "foreign-key", - "request", - "foreign-key", + "call-id-other-key", + "litellm_request:call-id-other-key", "team-a", "key-b", None, ), + ( + "call-id-foreign-team", + "litellm_request:call-id-foreign-team", + "team-b", + "key-a", + None, + ), + ("transport-only", "transport:", "team-a", "key-a", None), ]; insert_rows( client, @@ -643,7 +653,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::OtelTraces, cases .iter() - .map(|(id, _, _, _, _, _)| { + .map(|(id, key, _, _, _)| { BTreeMap::from([ ("Timestamp".into(), json!(start_ms * 1_000_000)), ("Duration".into(), json!(1_000_000)), @@ -652,7 +662,7 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback ("ObservationType".into(), json!("llm")), ("TeamId".into(), json!("team-a")), ("ApiKeyHash".into(), json!("key-a")), - ("CallKeys".into(), json!([format!("litellm_request:{id}")])), + ("CallKeys".into(), json!([key])), ("CallEvidence".into(), json!("complete")), ]) }) @@ -666,11 +676,11 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback InsertTable::SpendLogs, cases .iter() - .map(|(_, request, call_id, team, key, _)| { + .map(|(id, _, team, key, _)| { BTreeMap::from([ - ("request_id".into(), json!(request)), - ("response_id".into(), json!("provider-response")), - ("litellm_call_id".into(), json!(call_id)), + ("request_id".into(), json!(format!("request-{id}"))), + ("response_id".into(), json!(id)), + ("litellm_call_id".into(), json!(id)), ("team_id".into(), json!(team)), ("api_key".into(), json!(key)), ("start_time".into(), json!(start_ms)), @@ -689,13 +699,13 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback let access = ReadAccessParams { all_teams: false, user_id: String::new(), - team_ids: vec!["team-a".into()], + team_ids: vec!["team-a".into(), "team-b".into()], }; let page = reader .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) .await?; assert_eq!(page.data.len(), cases.len()); - for (id, _, _, _, _, expected) in cases { + for (id, _, _, _, expected) in cases { let summary = page .data .iter() @@ -707,10 +717,126 @@ async fn gateway_ids_resolve_through_detail_and_batch_reads_with_legacy_fallback .ok_or("missing trace")?; assert_eq!(detail.summary.spend, expected, "{id}"); assert_eq!(summary.spend, expected, "{id}"); + assert_eq!(summary.priced_calls, u64::from(expected.is_some()), "{id}"); } Ok(()) } +#[rstest] +#[case::provider_request(true)] +#[case::transport(false)] +#[tokio::test] +async fn native_cost_correlation_survives_session_grouping_and_excludes_other_owners( + #[future(awt)] migrated_database: TestResult, + #[case] response_header: bool, + #[values(false, true)] grouped: bool, +) -> TestResult { + let fixture = migrated_database?; + let client = &fixture.database.client; + let writer = Connection::writer(&fixture.database.url)?; + let trace_id = if grouped { + "grouped-trace" + } else { + "original-trace" + }; + let start_ms = 1_790_000_000_000_i64; + let keys = if response_header { + vec!["provider_response:req_native"] + } else { + Vec::new() + }; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::OtelTraces, + vec![BTreeMap::from([ + ("Timestamp".into(), json!(start_ms * 1_000_000)), + ("TraceId".into(), json!(trace_id)), + ("SpanId".into(), json!("native-call")), + ("SpanName".into(), json!("claude_code.llm_request")), + ("ObservationType".into(), json!("llm")), + ("Framework".into(), json!("claude-code")), + ("TeamId".into(), json!("team-a")), + ("ApiKeyHash".into(), json!("key-a")), + ("CallKeys".into(), json!(keys)), + ( + "CallEvidence".into(), + json!(if response_header { + "complete" + } else { + "unknown" + }), + ), + ( + "SpanAttributes".into(), + json!({"lens.original_trace_id": if grouped { "original-trace" } else { "" }}), + ), + ])], + ) + .await?; + insert_rows( + client, + &writer, + DATABASE, + InsertTable::SpendLogs, + ["key-a", "key-b"] + .into_iter() + .map(|key| { + BTreeMap::from([ + ("request_id".into(), json!(format!("log-{key}"))), + ("response_id".into(), json!("msg_native")), + ("provider_request_id".into(), json!("req_native")), + ( + "trace_id".into(), + json!(if response_header { + "" + } else { + "original-trace" + }), + ), + ( + "span_id".into(), + json!(if response_header { "" } else { "native-call" }), + ), + ("team_id".into(), json!("team-a")), + ("api_key".into(), json!(key)), + ("start_time".into(), json!(start_ms)), + ("end_time".into(), json!(start_ms + 1)), + ("spend".into(), json!(0.25)), + ]) + }) + .collect(), + ) + .await?; + let connection = fixture + .readers + .connection(client, &QueryScope::All, "fixture-secret") + .await?; + let (reader, store) = make_reader(client, connection); + let access = ReadAccessParams { + all_teams: true, + user_id: String::new(), + team_ids: Vec::new(), + }; + let page = reader + .list_traces(&store, &access, 0, 2_000_000_000_000, None, 50) + .await?; + assert_eq!(page.data.len(), 1); + let summary = &page.data[0]; + let detail = reader + .get_trace(&store, &access, trace_id, &summary.trace_ref) + .await? + .ok_or("missing trace")?; + assert_eq!((summary.spend, summary.priced_calls), (Some(0.25), 1)); + assert_eq!(detail.summary.spend, summary.spend); + assert_eq!( + detail.spans[0].spend_log_request_id.as_deref(), + Some("log-key-a") + ); + Ok(()) +} + #[rstest] #[case::recorded(false)] #[case::conflicting(true)] diff --git a/litellm-rust/crates/traces/src/lib.rs b/litellm-rust/crates/traces/src/lib.rs index f79fa59bdba..b51701638e8 100644 --- a/litellm-rust/crates/traces/src/lib.rs +++ b/litellm-rust/crates/traces/src/lib.rs @@ -44,5 +44,6 @@ pub use tenant::Tenant; pub use truncate::{truncate_messages, truncate_value}; pub use ui::{ChatRole, UiContent, UiField, UiMessage, UiToolCall, to_ui_content}; pub use view::{ - AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, Trace, TracePage, TraceSummary, + AgentNode, Span, SpanDetail, SpanErrorPage, SpanStatus, SpendMatch, Trace, TracePage, + TraceSummary, }; diff --git a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs index 18d7d74dc44..d24b9e6d8d2 100644 --- a/litellm-rust/crates/traces/src/normalize/format/claude_code.rs +++ b/litellm-rust/crates/traces/src/normalize/format/claude_code.rs @@ -6,7 +6,7 @@ use super::{Extraction, Format, SpanFacts}; use crate::{ Error, normalize::{ - CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, CallKey, + CLAUDE_CODE_AGENT, CLAUDE_CODE_EVENTS_SCOPE, CLAUDE_CODE_SCOPE, CallEvidence, ObservationType, RoleEvidence, SpanContext, attr, present, tokens, }, otlp::DecodedEvent, @@ -265,7 +265,7 @@ impl Format for ClaudeCode { output: llm_output(attributes), calls: present(attributes, &["gen_ai.response.id", "request_id"]) .map_or(CallEvidence::Unknown, |id| { - CallEvidence::complete(CallKey::ProviderResponse(id)) + CallEvidence::complete(crate::normalize::claude_call_key(id)) }), ..base }, diff --git a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs index 550afae77ca..1003834de79 100644 --- a/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/instrumentation/mod.rs @@ -216,7 +216,16 @@ impl Instrumentation { fn with_call_ids(context: &SpanContext<'_>, facts: SpanFacts) -> SpanFacts { let calls = [ - present(context.attributes, &["gen_ai.response.id"]).map(CallKey::ProviderResponse), + present(context.attributes, &["gen_ai.response.id"]).map(|id| { + if matches!( + context.scope, + crate::normalize::CLAUDE_CODE_SCOPE | crate::normalize::CLAUDE_CODE_EVENTS_SCOPE + ) { + crate::normalize::claude_call_key(id) + } else { + CallKey::ProviderResponse(id) + } + }), present(context.attributes, &["litellm.call_id"]).map(CallKey::LiteLlmRequest), ] .into_iter() diff --git a/litellm-rust/crates/traces/src/normalize/mod.rs b/litellm-rust/crates/traces/src/normalize/mod.rs index f2c5376af10..53d5e521799 100644 --- a/litellm-rust/crates/traces/src/normalize/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/mod.rs @@ -65,16 +65,26 @@ pub enum CallKey { LiteLlmRequest(String), /// The provider response id returned to the caller (`spend_logs.response_id`). ProviderResponse(String), + ProviderRequest(String), /// The span is the HTTP request itself; LiteLLM logs its `traceparent` span id. Transport, GatewayAttempt, } +pub(crate) fn claude_call_key(id: String) -> CallKey { + if id.starts_with("msg_") { + CallKey::ProviderResponse(id) + } else { + CallKey::ProviderRequest(id) + } +} + impl fmt::Display for CallKey { fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::LiteLlmRequest(id) => write!(formatter, "litellm_request:{id}"), Self::ProviderResponse(id) => write!(formatter, "provider_response:{id}"), + Self::ProviderRequest(id) => write!(formatter, "provider_request:{id}"), Self::Transport => formatter.write_str("transport:"), Self::GatewayAttempt => formatter.write_str("gateway_attempt:"), } @@ -89,6 +99,9 @@ impl FromStr for CallKey { Some(("provider_response", id)) if !id.is_empty() => { Ok(Self::ProviderResponse(id.to_owned())) } + Some(("provider_request", id)) if !id.is_empty() => { + Ok(Self::ProviderRequest(id.to_owned())) + } Some(("litellm_request", id)) if !id.is_empty() => { Ok(Self::LiteLlmRequest(id.to_owned())) } @@ -127,24 +140,45 @@ pub enum CallEvidence { impl CallEvidence { pub(crate) fn row_keys(row: &crate::query::named::TraceSpansRow) -> BTreeSet { - if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { + let native = Self::native_request(row); + let keys = if row.call_keys.is_empty() && !row.litellm_request_id.is_empty() { BTreeSet::from([CallKey::ProviderResponse(row.litellm_request_id.clone())]) } else { row.call_keys.iter().cloned().collect() + }; + if !native { + return keys; } + if keys.is_empty() { + return BTreeSet::from([CallKey::Transport]); + } + keys.into_iter() + .map(|key| match key { + CallKey::ProviderResponse(id) => claude_call_key(id), + key => key, + }) + .collect() + } + + fn native_request(row: &crate::query::named::TraceSpansRow) -> bool { + matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk") + && row.name == "claude_code.llm_request" } pub(crate) fn from_row(row: &crate::query::named::TraceSpansRow) -> Self { - let kind = row - .call_evidence - .unwrap_or(if Self::row_keys(row).is_empty() { + let keys = Self::row_keys(row); + let kind = if Self::native_request(row) { + CallEvidenceKind::Complete + } else { + row.call_evidence.unwrap_or(if keys.is_empty() { CallEvidenceKind::Unknown } else { CallEvidenceKind::Complete - }); + }) + }; match kind { - CallEvidenceKind::Complete => Self::Complete(Self::row_keys(row)), - CallEvidenceKind::Partial => Self::Partial(Self::row_keys(row)), + CallEvidenceKind::Complete => Self::Complete(keys), + CallEvidenceKind::Partial => Self::Partial(keys), CallEvidenceKind::Unknown => Self::Unknown, } } diff --git a/litellm-rust/crates/traces/src/otlp/span.rs b/litellm-rust/crates/traces/src/otlp/span.rs index 4e63d78d286..58ccb9ea3b4 100644 --- a/litellm-rust/crates/traces/src/otlp/span.rs +++ b/litellm-rust/crates/traces/src/otlp/span.rs @@ -193,9 +193,9 @@ fn decoded_span( .into_iter() .flatten() .map(|key| match key { - crate::CallKey::LiteLlmRequest(id) | crate::CallKey::ProviderResponse(id) => { - id.len() + size_of::() - } + crate::CallKey::LiteLlmRequest(id) + | crate::CallKey::ProviderResponse(id) + | crate::CallKey::ProviderRequest(id) => id.len() + size_of::(), crate::CallKey::Transport | crate::CallKey::GatewayAttempt => { size_of::() } diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index 86b128f4cf7..9af02ecfad1 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -68,6 +68,8 @@ pub struct TraceSpansParams { pub struct TraceSpansRow { #[serde(default)] pub trace_id: String, + #[serde(default)] + pub original_trace_id: String, pub span_id: String, pub parent_span_id: String, pub name: String, @@ -113,6 +115,16 @@ pub struct TraceSpansRow { pub user_id: String, } +impl TraceSpansRow { + pub(crate) fn transport_trace_id(&self) -> &str { + if self.original_trace_id.is_empty() { + &self.trace_id + } else { + &self.original_trace_id + } + } +} + #[derive(Clone, Debug, Deserialize, Serialize)] pub struct TracePageSpansParams { #[serde(flatten)] @@ -172,6 +184,7 @@ pub struct SpendByResponseIdsParams { #[serde(flatten)] pub access: ReadAccessParams, pub response_ids: Vec, + pub provider_request_ids: Vec, pub request_ids: Vec, pub trace_ids: Vec, pub start_ms: i64, @@ -184,6 +197,8 @@ pub struct SpendByResponseIdsRow { pub litellm_call_id: String, pub response_id: String, pub upstream_response_id: String, + #[serde(default)] + pub provider_request_id: String, pub trace_id: String, pub span_id: String, pub team_id: String, diff --git a/litellm-rust/crates/traces/src/resolve/resolution.rs b/litellm-rust/crates/traces/src/resolve/resolution.rs index 9b47755a16f..e19a98a1ede 100644 --- a/litellm-rust/crates/traces/src/resolve/resolution.rs +++ b/litellm-rust/crates/traces/src/resolve/resolution.rs @@ -3,6 +3,7 @@ use std::collections::HashMap; use indexmap::IndexMap; use crate::{ + SpendMatch, normalize::{CallKey, ObservationType}, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -27,8 +28,11 @@ pub(super) struct Resolution<'a> { types: HashMap<&'a str, ObservationType>, tool_failures: HashMap<&'a str, &'a TraceSpansRow>, pub(super) model_calls: Vec, + call_matches: HashMap>, } +pub(super) type CallMatch<'a> = (Option>, SpendMatch); + impl<'a> Resolution<'a> { pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self { let graph = Graph::new(rows); @@ -36,7 +40,7 @@ impl<'a> Resolution<'a> { let types: HashMap<&str, ObservationType> = (0..rows.len()) .map(|index| (graph.id(index), resolved_type(&graph, index, named_agents))) .collect(); - let model_calls = (0..rows.len()) + let model_calls: Vec = (0..rows.len()) .filter(|index| { types[graph.id(*index)] == ObservationType::Llm && !graph @@ -45,7 +49,7 @@ impl<'a> Resolution<'a> { .any(|descendant| types[graph.id(descendant)] == ObservationType::Llm) }) .collect(); - Self { + let resolution = Self { ownership: Ownership { team_id: &rows[0].team_id, api_key_hash: &rows[0].api_key_hash, @@ -65,6 +69,16 @@ impl<'a> Resolution<'a> { .map(|row| (row.tool_call_id.as_str(), row)) .collect(), model_calls, + call_matches: HashMap::new(), + }; + let call_matches = resolution + .model_calls + .iter() + .map(|call| (*call, resolution.resolve_call_match(*call))) + .collect(); + Self { + call_matches, + ..resolution } } @@ -118,7 +132,24 @@ impl<'a> Resolution<'a> { spend::requests(self.row(index), &self.ownership, self.spend) } + pub(super) fn call_match(&self, call: usize) -> Option<&CallMatch<'a>> { + self.call_matches.get(&call) + } + pub(super) fn call_requests(&self, call: usize) -> Option> { + self.call_match(call) + .and_then(|(requests, _)| requests.clone()) + } + + fn resolve_call_match(&self, call: usize) -> CallMatch<'a> { + if let Some(requests) = self.resolve_call_requests(call) { + return (Some(requests), SpendMatch::Matched); + } + let evidence = self.requests(call); + (None, evidence.unmatched_reason()) + } + + fn resolve_call_requests(&self, call: usize) -> Option> { let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| { self.kind(*ancestor) == ObservationType::Llm && self diff --git a/litellm-rust/crates/traces/src/resolve/spend.rs b/litellm-rust/crates/traces/src/resolve/spend.rs index ce23362b7f8..a6b2fc0dbc5 100644 --- a/litellm-rust/crates/traces/src/resolve/spend.rs +++ b/litellm-rust/crates/traces/src/resolve/spend.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use indexmap::IndexMap; use crate::{ - CallEvidence, CallEvidenceKind, CallKey, + CallEvidence, CallEvidenceKind, CallKey, SpendMatch, query::named::{SpendByResponseIdsRow as SpendRow, TraceSpansRow}, }; @@ -12,6 +12,7 @@ use crate::{ pub struct SpendLookup { pub response_ids: Vec, pub request_ids: Vec, + pub provider_request_ids: Vec, /// Traces whose transport spans LiteLLM logged by `traceparent`. pub trace_ids: Vec, } @@ -45,13 +46,21 @@ impl SpendLookup { }) .collect(), ), + provider_request_ids: sorted( + keys() + .filter_map(|(_, key)| match key { + CallKey::ProviderRequest(id) => Some(id.clone()), + _ => None, + }) + .collect(), + ), trace_ids: sorted( keys() .filter_map(|(row, key)| match key { CallKey::Transport | CallKey::GatewayAttempt - if !row.trace_id.is_empty() => + if !row.transport_trace_id().is_empty() => { - Some(row.trace_id.clone()) + Some(row.transport_trace_id().to_owned()) } _ => None, }) @@ -61,7 +70,10 @@ impl SpendLookup { } pub fn is_empty(&self) -> bool { - self.response_ids.is_empty() && self.request_ids.is_empty() && self.trace_ids.is_empty() + self.response_ids.is_empty() + && self.request_ids.is_empty() + && self.provider_request_ids.is_empty() + && self.trace_ids.is_empty() } } @@ -86,6 +98,7 @@ pub(super) type Requests<'a> = Vec<&'a SpendRow>; enum KeyFamily { GatewayCall, ProviderResponse, + ProviderRequest, Transport, } @@ -93,12 +106,14 @@ fn key_family(key: &CallKey) -> KeyFamily { match key { CallKey::LiteLlmRequest(_) => KeyFamily::GatewayCall, CallKey::ProviderResponse(_) => KeyFamily::ProviderResponse, + CallKey::ProviderRequest(_) => KeyFamily::ProviderRequest, CallKey::Transport | CallKey::GatewayAttempt => KeyFamily::Transport, } } pub(super) enum KeyMatch<'a> { Missing, + Conflicting, Unique(&'a SpendRow), Ambiguous(Requests<'a>), } @@ -115,13 +130,13 @@ impl<'a> KeyMatch<'a> { fn unique(&self) -> Option<&'a SpendRow> { match self { Self::Unique(request) => Some(request), - Self::Missing | Self::Ambiguous(_) => None, + Self::Missing | Self::Conflicting | Self::Ambiguous(_) => None, } } fn agrees_with(&self, selected: &[&SpendRow]) -> bool { match self { - Self::Missing => false, + Self::Missing | Self::Conflicting => false, Self::Unique(request) => selected .iter() .any(|row| row.identity() == request.identity()), @@ -147,6 +162,29 @@ pub(super) enum SpendEvidence<'a> { } impl<'a> SpendEvidence<'a> { + pub(super) fn unmatched_reason(&self) -> SpendMatch { + match self { + Self::Unknown => SpendMatch::NoCallId, + Self::Partial(_) => SpendMatch::IncompleteEvidence, + Self::Complete(matches) if matches.is_empty() => SpendMatch::NoCallId, + Self::Complete(matches) + if matches.iter().any(|evidence| { + matches!(evidence, KeyMatch::Conflicting | KeyMatch::Ambiguous(_)) + }) => + { + SpendMatch::Ambiguous + } + Self::Complete(matches) + if matches + .iter() + .any(|evidence| matches!(evidence, KeyMatch::Missing)) => + { + SpendMatch::NoSpendLog + } + Self::Complete(_) => SpendMatch::Ambiguous, + } + } + pub(super) fn complete_requests(&self) -> Option> { match self { Self::Complete(matches) if !matches.is_empty() => { @@ -186,15 +224,16 @@ fn matches<'a>( CallKey::ProviderResponse(id) => { !id.is_empty() && (spend.response_id == *id || spend.upstream_response_id == *id) } + CallKey::ProviderRequest(id) => !id.is_empty() && spend.provider_request_id == *id, CallKey::LiteLlmRequest(id) => { !id.is_empty() && (spend.litellm_call_id == *id || (spend.litellm_call_id.is_empty() && spend.request_id == *id)) } CallKey::Transport | CallKey::GatewayAttempt => { - !row.trace_id.is_empty() + !row.transport_trace_id().is_empty() && !row.span_id.is_empty() - && spend.trace_id == row.trace_id + && spend.trace_id == row.transport_trace_id() && spend.span_id == row.span_id } }; @@ -253,7 +292,8 @@ pub(super) fn requests<'a>( let matches = aliases .into_iter() .map(|(_, requests)| { - KeyMatch::new( + let had_candidates = !requests.is_empty(); + let matched = KeyMatch::new( requests .into_iter() .filter(|request| { @@ -262,7 +302,12 @@ pub(super) fn requests<'a>( .all(|family| family.contains(&request.identity())) }) .collect(), - ) + ); + if had_candidates && matches!(matched, KeyMatch::Missing) { + KeyMatch::Conflicting + } else { + matched + } }) .collect(); match evidence.kind() { @@ -280,18 +325,35 @@ pub(super) fn request_cost(requests: &[&SpendRow]) -> Option { }) } -pub(super) fn total(calls: &[Option>]) -> Option { - if calls.is_empty() { - return None; - } - let requests: Option> = calls - .iter() - .map(|requests| requests.as_ref()) - .collect::>>() - .map(|calls| calls.into_iter().flatten().copied().collect()); - let unique: IndexMap<(&str, i64, &str), &SpendRow> = requests? +pub(super) fn unique<'a>(requests: impl IntoIterator) -> Requests<'a> { + requests .into_iter() .map(|request| (request.identity(), request)) - .collect(); - request_cost(&unique.into_values().collect::>()) + .collect::>() + .into_values() + .collect() +} + +pub(super) struct Priced { + pub(super) spend: Option, + pub(super) priced_calls: u64, +} + +pub(super) fn total(calls: &[Option>]) -> Priced { + let priced: Vec<&Requests<'_>> = calls + .iter() + .flatten() + .filter(|requests| request_cost(requests).is_some()) + .collect(); + let spend = if priced.is_empty() { + None + } else { + request_cost(&unique( + priced.iter().flat_map(|requests| requests.iter().copied()), + )) + }; + Priced { + spend, + priced_calls: priced.len() as u64, + } } diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 87688d9b36e..9e9edd51bf4 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -6,7 +6,7 @@ use time::OffsetDateTime; use crate::{ normalize::ObservationType, query::named::{ListTracesRow, SpendByResponseIdsRow as SpendRow, TraceSpansRow}, - view::{AgentNode, Span, SpanStatus, Trace, TraceSummary}, + view::{AgentNode, Span, SpanStatus, SpendMatch, Trace, TraceSummary}, }; use super::{ @@ -23,7 +23,18 @@ fn optional(value: &str) -> Option { fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span { let row = resolution.row(index); let status = resolution.status_source(index); - let requests = resolution.requests(index).complete_requests(); + let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) { + (requests.clone(), Some(*matched)) + } else { + (resolution.requests(index).complete_requests(), None) + }; + let spend = requests + .as_ref() + .and_then(|requests| request_cost(requests)); + let spend_log_request_id = match (spend_match, requests.as_deref()) { + (Some(SpendMatch::Matched), Some([request])) => Some(request.request_id.clone()), + _ => None, + }; Span { span_id: row.span_id.clone(), parent_span_id: optional(&row.parent_span_id), @@ -42,9 +53,9 @@ fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span input_tokens: row.input_tokens, output_tokens: row.output_tokens, litellm_request_id: optional(&row.litellm_request_id), - spend: requests - .as_ref() - .and_then(|requests| request_cost(requests)), + spend, + spend_log_request_id, + spend_match, } } @@ -87,6 +98,7 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .filter(|(owner, _)| *owner == name) .map(|(_, requests)| requests.clone()) .collect(); + let priced = total(&owned_calls); AgentNode { name: name.to_owned(), parent_agent, @@ -101,7 +113,8 @@ fn agents(resolution: &Resolution<'_>) -> Vec { .map(|span| graph.rows[*span].duration_ns) .sum::() as f64 / NANOS_PER_MS, - spend: total(&owned_calls), + spend: priced.spend, + priced_calls: priced.priced_calls, } }) .collect() @@ -160,6 +173,12 @@ pub fn resolve_trace( } else { calls.iter().map(|call| &rows[*call]).collect() }; + let priced = total( + &calls + .iter() + .map(|call| resolution.call_requests(*call)) + .collect::>(), + ); let first_input = spans .iter() .zip(rows) @@ -208,12 +227,8 @@ pub fn resolve_trace( input_tokens: counted.iter().map(|row| u64::from(row.input_tokens)).sum(), output_tokens: counted.iter().map(|row| u64::from(row.output_tokens)).sum(), models: sorted_unique(calls.iter().map(|call| rows[*call].model.as_str())), - spend: total( - &calls - .iter() - .map(|call| resolution.call_requests(*call)) - .collect::>(), - ), + spend: priced.spend, + priced_calls: priced.priced_calls, }; Some(Trace { summary, @@ -250,5 +265,6 @@ pub fn listed_summary(row: &ListTracesRow) -> TraceSummary { output_tokens: row.output_tokens, models: row.models.clone(), spend: None, + priced_calls: 0, } } diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index a864c740526..5a214d33da7 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -37,6 +37,19 @@ pub struct Span { pub output_tokens: u32, pub litellm_request_id: Option, pub spend: Option, + pub spend_log_request_id: Option, + pub spend_match: Option, +} + +#[macro_rules_attribute::apply(response_type)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +pub enum SpendMatch { + Matched, + NoCallId, + NoSpendLog, + Ambiguous, + IncompleteEvidence, } /// One distinct agent in a trace: 200 invocations of `researcher` are one node. @@ -50,6 +63,7 @@ pub struct AgentNode { pub tool_calls: u64, pub duration_ms: f64, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] @@ -80,6 +94,7 @@ pub struct TraceSummary { pub output_tokens: u64, pub models: Vec, pub spend: Option, + pub priced_calls: u64, } #[macro_rules_attribute::apply(response_type)] diff --git a/litellm-rust/crates/traces/tests/captures.rs b/litellm-rust/crates/traces/tests/captures.rs index ca498fc914a..ff20db617f6 100644 --- a/litellm-rust/crates/traces/tests/captures.rs +++ b/litellm-rust/crates/traces/tests/captures.rs @@ -117,6 +117,7 @@ fn captured_spend_rows(spend_logs: &str) -> (FixtureCapture, Vec TraceSpansRow { .into_iter() .flatten() .find_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => Some(id.clone()), CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => None, }) .unwrap_or_default(); TraceSpansRow { trace_id: span.trace_id, + original_trace_id: String::new(), span_id: span.span_id, parent_span_id: span.parent_span_id, name: span.name, @@ -267,6 +269,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow { .expect("valid unrelated transport timestamp"); TraceSpansRow { trace_id: call.trace_id.clone(), + original_trace_id: call.original_trace_id.clone(), span_id: format!("unrelated-transport-{}", call.span_id), parent_span_id: call.parent_span_id.clone(), name: "unrelated-http".into(), @@ -405,7 +408,9 @@ fn redundant_genai_response_id_keeps_call_evidence( let response_ids: Vec<_> = keys .iter() .filter_map(|key| match key { - CallKey::ProviderResponse(id) => Some(id.clone()), + CallKey::ProviderResponse(id) | CallKey::ProviderRequest(id) => { + Some(id.clone()) + } CallKey::LiteLlmRequest(_) | CallKey::Transport | CallKey::GatewayAttempt => { None } diff --git a/litellm-rust/crates/traces/tests/normalization_formats.rs b/litellm-rust/crates/traces/tests/normalization_formats.rs index 124f03593c8..e8b11ece64a 100644 --- a/litellm-rust/crates/traces/tests/normalization_formats.rs +++ b/litellm-rust/crates/traces/tests/normalization_formats.rs @@ -928,3 +928,28 @@ fn convention_markers_keep_genai_call_evidence( let marked = decode(span, scope, &marked_attributes, vec![]).unwrap(); assert_eq!(marked.normalized.calls, plain.normalized.calls); } + +#[rstest] +#[case::request("req_native", CallKey::ProviderRequest("req_native".into()))] +#[case::legacy_message("msg_legacy", CallKey::ProviderResponse("msg_legacy".into()))] +fn native_claude_preserves_the_provider_id_family( + span: Span, + #[case] id: &str, + #[case] key: CallKey, +) { + let native = Span { + name: "claude_code.llm_request".into(), + ..span + }; + let decoded = decode( + native, + "com.anthropic.claude_code.tracing", + &[("gen_ai.response.id", id)], + Vec::new(), + ) + .unwrap(); + assert_eq!( + decoded.normalized.calls, + CallEvidence::Complete(std::collections::BTreeSet::from([key])) + ); +} diff --git a/litellm-rust/crates/traces/tests/normalize.rs b/litellm-rust/crates/traces/tests/normalize.rs index 923e2006088..bfff82fcdd7 100644 --- a/litellm-rust/crates/traces/tests/normalize.rs +++ b/litellm-rust/crates/traces/tests/normalize.rs @@ -284,6 +284,7 @@ fn llamaindex_wrapped_responses_keep_provider_call_keys(#[case] body: &[u8]) { #[rstest] #[case::request(litellm_traces::CallKey::LiteLlmRequest("request:with:colons".to_owned()))] #[case::response(litellm_traces::CallKey::ProviderResponse("response:with:colons".to_owned()))] +#[case::provider_request(litellm_traces::CallKey::ProviderRequest("req_native".into()))] #[case::transport(litellm_traces::CallKey::Transport)] #[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt)] fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { @@ -305,6 +306,7 @@ fn call_keys_round_trip_through_storage(#[case] key: litellm_traces::CallKey) { #[rstest] #[case::missing_separator("provider_response")] #[case::missing_response("provider_response:")] +#[case::missing_provider_request("provider_request:")] #[case::missing_request("litellm_request:")] #[case::transport_id("transport:unexpected")] #[case::gateway_attempt_separator("gateway_attempt")] diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 6b853ebf151..3ac9668e42a 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -43,7 +43,7 @@ fn named_requests_preserve_all_access_cases( json!({"trace_id": "trace", "trace_ref": "ref", "span_id": "span", "error_offset": u64::MAX, "error_version": "version"}), )); round_trip::(request( - json!({"response_ids": ["response"], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), + json!({"response_ids": ["response"], "provider_request_ids": [], "request_ids": ["request"], "trace_ids": ["trace"], "start_ms": -1, "end_ms": 10}), )); } @@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() { json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"trace_id": "trace", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), @@ -62,6 +62,6 @@ fn result_contracts_preserve_public_field_names() { json!({"span_id": "span", "message": "error", "total_chars": u64::MAX, "version": "version"}), ); round_trip::( - json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), + json!({"request_id": "request", "litellm_call_id": "gateway", "response_id": "response", "upstream_response_id": "upstream", "provider_request_id": "req_provider", "trace_id": "trace", "span_id": "span", "team_id": "team", "api_key": "key", "user": "user", "spend": 0.125, "start_ms": -1}), ); } diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index 641d15db5b9..8d65adb0c0f 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -1,5 +1,5 @@ use litellm_traces::{ - AgentNode, SpanStatus, iso_time, listed_summary, + AgentNode, SpanStatus, SpendMatch, iso_time, listed_summary, query::named::{ListTracesRow, SpendByResponseIdsRow, TraceSpansRow}, resolve_trace, }; @@ -11,6 +11,7 @@ const MS: i64 = 1_000_000; fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> TraceSpansRow { TraceSpansRow { trace_id: String::new(), + original_trace_id: String::new(), span_id: span_id.into(), parent_span_id: parent.into(), name: name.into(), @@ -32,8 +33,8 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac call_keys: Vec::new(), call_evidence: None, tool_call_id: String::new(), - team_id: String::new(), - api_key_hash: String::new(), + team_id: "team".into(), + api_key_hash: "key".into(), user_id: String::new(), } } @@ -55,31 +56,18 @@ fn llm(span_id: &str, parent: &str, agent: &str, response_id: &str) -> TraceSpan } } -fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { - span.team_id = team.into(); - span.user_id = user.into(); - span.api_key_hash = key.into(); - span -} - -fn spend( - request_id: &str, - response_id: &str, - team: &str, - user: &str, - key: &str, - cost: f64, -) -> SpendByResponseIdsRow { +fn spend(request_id: &str, response_id: &str, cost: f64) -> SpendByResponseIdsRow { SpendByResponseIdsRow { request_id: request_id.into(), - response_id: response_id.into(), litellm_call_id: String::new(), + response_id: response_id.into(), upstream_response_id: String::new(), + provider_request_id: String::new(), trace_id: String::new(), span_id: String::new(), - team_id: team.into(), - api_key: key.into(), - user: user.into(), + api_key: "key".into(), + user: String::new(), + team_id: "team".into(), spend: Some(cost), start_ms: T0 / MS, } @@ -221,6 +209,7 @@ fn repeated_subagent_invocations_aggregate_into_one_node() { tool_calls: 1, duration_ms: 1000.0, spend: None, + priced_calls: 0, } ); let researcher = &trace.agents[1]; @@ -384,38 +373,15 @@ fn names_and_frameworks_are_sorted_and_distinct() { } #[rstest] -fn repeated_response_counts_once_and_other_owners_are_ignored() { +fn repeated_response_id_counts_once() { let rows = [ - owned( - row("root", "", "agent", "agent", "agent"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-1", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), - owned( - llm("llm-2", "root", "agent", "response-1"), - "team-a", - "", - "key-a", - ), + row("root", "", "agent", "agent", "agent"), + llm("llm-1", "root", "agent", "response-1"), + llm("llm-2", "root", "agent", "response-1"), ]; let spend = [ - spend("request-other", "response-1", "team-b", "", "key-b", 99.0), - spend("request-1", "response-1", "team-a", "", "key-a", 0.25), - spend( - "request-other-key", - "unrelated-response", - "team-a", - "", - "key-c", - 50.0, - ), + spend("request-1", "response-1", 0.25), + spend("request-other", "unrelated-response", 50.0), ]; let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); assert_eq!(trace.summary.spend, Some(0.25)); @@ -431,352 +397,363 @@ fn repeated_response_counts_once_and_other_owners_are_ignored() { } #[rstest] -fn ambiguous_response_id_keeps_cost_unknown() { - let rows = [owned( - llm("llm-1", "", "agent", "response-1"), - "", - "user", - "key-a", - )]; - let spend = [ - spend("response-1", "response-1", "", "user", "key-a", 0.25), - spend( - "response-1_cache_hit123", - "response-1", - "", - "user", - "key-a", - 0.0, - ), - ]; - let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); - assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); -} - -#[rstest] -#[case::key_differs("team", "", "export", "team", "", "request", false)] -#[case::shared_key("team", "", "export", "team", "", "export", true)] -#[case::shared_user("", "user", "export", "", "user", "request", true)] -#[case::teamless_key("", "", "key", "", "", "key", true)] -#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] -#[case::other_user("", "user", "export", "", "other-user", "request", false)] -#[case::no_shared_identity("", "", "export", "", "", "request", false)] -#[case::no_identity("", "", "", "", "", "", false)] -#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] -fn cost_requires_shared_ownership( - #[case] trace_team: &str, - #[case] trace_user: &str, - #[case] trace_key: &str, - #[case] spend_team: &str, - #[case] spend_user: &str, - #[case] spend_key: &str, - #[case] known: bool, -) { +fn model_calls_link_the_spend_log_they_were_priced_from() { let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - trace_team, - trace_user, - trace_key, - ), - owned( - llm("llm", "agent", "agent", "response"), - trace_team, - trace_user, - trace_key, - ), + row("agent", "", "agent", "agent", "agent"), + llm("matched", "agent", "agent", "matched"), + llm("no-log", "agent", "agent", "missing"), + llm("no-id", "agent", "agent", ""), + llm("ambiguous", "agent", "agent", "cached"), ]; - let spend = [spend( - "request", "response", spend_team, spend_user, spend_key, 0.25, - )]; - let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); - let expected = known.then_some(0.25); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[1].spend, expected); -} - -#[rstest] -#[case::missing_id("missing_id")] -#[case::missing_spend("missing_spend")] -#[case::duplicate_spend("duplicate_spend")] -fn incomplete_call_cost_never_becomes_a_partial_total(#[case] failure: &str) { - let second_id = if failure == "missing_id" { - "" - } else { - "second" - }; - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "export", - ), - owned( - llm("first", "agent", "agent", "first"), - "team", - "", - "export", - ), - owned( - llm("second", "agent", "agent", second_id), - "team", - "", - "export", - ), + let logs = [ + spend("request-matched", "matched", 0.25), + spend("request-cached-a", "cached", 0.25), + spend("request-cached-b", "cached", 0.0), ]; - let first = spend("first", "first", "team", "", "export", 0.25); - let second = spend("second", "second", "team", "", "export", 0.25); - let duplicate = spend("duplicate", "second", "team", "", "export", 0.25); - let spend = if failure == "duplicate_spend" { - vec![first, second, duplicate] - } else { - vec![first] - }; - let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); - assert_eq!(trace.spans[1].spend, Some(0.25)); - assert_eq!(trace.spans[2].spend, None); - assert_eq!(trace.summary.spend, None); - assert_eq!(trace.agents[0].spend, None); -} - -#[rstest] -fn transport_spans_complete_a_call_without_its_own_id() { - let mut transport = row("http", "llm", "POST", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec!["transport:".parse().unwrap()]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, Some(0.5)); -} - -#[rstest] -#[case::lone_call(1, Some(0.5))] -#[case::two_calls(2, None)] -fn sibling_transports_belong_to_the_only_model_call_under_their_parent( - #[case] calls: usize, - #[case] expected: Option, -) { - let mut transport = at( - row("http", "step", "gateway.request", "framework", ""), - 2, - 10, + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + let links: Vec<_> = trace + .spans + .iter() + .map(|span| (span.spend_log_request_id.as_deref(), span.spend_match)) + .collect(); + assert_eq!( + links, + [ + (None, None), + (Some("request-matched"), Some(SpendMatch::Matched)), + (None, Some(SpendMatch::NoSpendLog)), + (None, Some(SpendMatch::NoCallId)), + (None, Some(SpendMatch::Ambiguous)), + ] ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut rows = vec![ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - owned(transport, "team", "", "key"), - ]; - for index in 0..calls { - let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); - call.call_evidence = None; - rows.push(owned(call, "team", "", "key")); - } - let mut logged = spend("request", "", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] -#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] -fn sibling_transport_does_not_lose_model_call_spend( - #[case] transport_timing: Option<(i64, u64)>, - #[case] unrelated_spend: bool, +fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_a_wrapper() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", ""), + ]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-request", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!( + ( + trace.spans[1].spend, + trace.spans[1].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn unpriced_call_leaves_a_partial_total_of_the_priced_calls() { + let rows = [ + row("agent", "", "agent", "agent", "agent"), + llm("first", "agent", "agent", "first"), + llm("second", "agent", "agent", "second"), + llm("third", "agent", "agent", ""), + ]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("first", "first", 0.25)]).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.summary.priced_calls, + trace.summary.llm_calls + ), + (Some(0.25), 1, 3) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), None, None] + ); +} + +#[rstest] +fn no_priced_call_leaves_cost_unknown() { + let rows = [llm("call", "", "agent", "response")]; + let trace = resolve_trace("trace", "ref", &rows, &[spend("other", "other", 0.25)]).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +fn gateway_logged(request_id: &str, call_id: &str, cost: f64) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..spend(request_id, &format!("chatcmpl-{request_id}"), cost) + } +} + +#[rstest] +#[case::response_id(litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), Some(0.25))] +#[case::gateway_call_id(litellm_traces::CallKey::LiteLlmRequest("gateway".into()), Some(0.25))] +#[case::other_call_id(litellm_traces::CallKey::LiteLlmRequest("other".into()), None)] +#[case::request_id_is_not_a_call_id(litellm_traces::CallKey::LiteLlmRequest("request".into()), None)] +#[case::transport(litellm_traces::CallKey::Transport, None)] +#[case::gateway_attempt(litellm_traces::CallKey::GatewayAttempt, None)] +fn only_ids_litellm_assigned_join_spend( #[case] key: litellm_traces::CallKey, #[case] expected: Option, ) { - let call = owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![litellm_traces::CallKey::ProviderResponse( - "chatcmpl-1".into(), - )], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("chat", "step", "agent", "chatcmpl-1") - }, - "team", - "", - "key", + let rows = [TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![key], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "trace".into(), + ..gateway_logged("request", "gateway", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (expected, u64::from(expected.is_some())) ); - let base_rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), - call, - ]; - let rows: Vec<_> = base_rows - .into_iter() - .chain(transport_timing.map(|(start, duration)| { - let mut transport = at( - row("tool-http", "step", "GET", "framework", ""), - start, - duration, - ); - transport.trace_id = "trace".into(); - transport.call_keys = vec![key]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - owned(transport, "team", "", "key") - })) - .collect(); - let logs: Vec<_> = std::iter::once(spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5)) - .chain(unrelated_spend.then(|| SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "tool-http".into(), - ..spend("unrelated", "unrelated", "team", "", "key", 0.75) - })) - .collect(); - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); } #[rstest] -#[case::agreeing_ids( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] -#[case::conflicting_response_id( - litellm_traces::CallKey::Transport, - "call-a", - Some("response-b"), - None -)] -#[case::conflicting_gateway_and_response( - litellm_traces::CallKey::Transport, - "call-b", - Some("response-b"), - None -)] -#[case::agreeing_gateway_attempt( - litellm_traces::CallKey::GatewayAttempt, - "call-a", - Some("response-a"), - Some(0.25) -)] -#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] -fn gateway_attempt_identifiers_must_match_one_spend_row( - #[case] transport: litellm_traces::CallKey, - #[case] call_id: &str, - #[case] response_id: Option<&str>, +#[case::same_team("team", Some(0.25), SpendMatch::Matched)] +#[case::other_team("other-team", None, SpendMatch::NoSpendLog)] +fn spend_logs_price_only_calls_in_their_own_team( + #[case] logged_team: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let keys = [ - transport, - litellm_traces::CallKey::LiteLlmRequest(call_id.into()), - ] - .into_iter() - .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) - .collect(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", - ), - owned(llm("call", "agent", "agent", ""), "team", "", "key"), - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: keys, - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..row("attempt", "call", "gateway.request", "framework", "") - }, - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - litellm_call_id: "call-a".into(), - trace_id: "trace".into(), - span_id: "attempt".into(), - ..spend("request-a", "response-a", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - litellm_call_id: "call-b".into(), - trace_id: "trace".into(), - span_id: "other-attempt".into(), - ..spend("request-b", "response-b", "team", "", "key", 0.5) - }, - ]; + let rows = [TraceSpansRow { + team_id: "team".into(), + ..llm("call", "", "agent", "response") + }]; + let logs = [SpendByResponseIdsRow { + team_id: logged_team.into(), + ..spend("request", "response", 0.25) + }]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); - assert_eq!(trace.spans[2].spend, expected); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); } #[rstest] -#[case::legacy_row("", Some(0.5))] -#[case::other_call("other-call", None)] -fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( +#[case::legacy_row_without_call_id("", Some(0.25), SpendMatch::Matched)] +#[case::call_id_names_nothing("missing", None, SpendMatch::Ambiguous)] +fn recorded_gateway_id_must_agree_unless_spend_predates_gateway_ids( #[case] logged_call_id: &str, #[case] expected: Option, + #[case] matched: SpendMatch, ) { - let mut transport = row("http", "llm", "gateway.request", "framework", ""); - transport.trace_id = "trace".into(); - transport.call_keys = vec![ - "transport:".parse().unwrap(), - "litellm_request:gateway-call".parse().unwrap(), - ]; - transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); - let mut call = llm("llm", "agent", "agent", ""); - call.trace_id = "trace".into(); - let rows = [ - owned( - row("agent", "", "agent", "agent", "agent"), - "team", - "", - "key", + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [gateway_logged("request", logged_call_id, 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.spans[0].spend, trace.spans[0].spend_match), + (expected, Some(matched)) + ); +} + +#[rstest] +#[case::missing_cost(None, SpendMatch::Matched)] +#[case::finite_cost(Some(0.25), SpendMatch::Matched)] +fn a_single_matched_row_is_matched_whatever_its_cost( + #[case] cost: Option, + #[case] matched: SpendMatch, +) { + let rows = [llm("call", "", "agent", "response")]; + let logs = [SpendByResponseIdsRow { + spend: cost, + ..spend("request", "response", 0.0) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.spans[0].spend_match, + trace.spans[0].spend_log_request_id.as_deref() ), - owned(call, "team", "", "key"), - owned(transport, "team", "", "key"), + (cost, Some(matched), Some("request")) + ); + assert_eq!(trace.summary.priced_calls, u64::from(cost.is_some())); +} + +#[rstest] +#[case::same_request("gateway", Some(0.25), SpendMatch::Matched)] +#[case::conflicting_requests("other", None, SpendMatch::Ambiguous)] +fn response_id_and_call_id_must_name_the_same_request( + #[case] call_id: &str, + #[case] expected: Option, + #[case] matched: SpendMatch, +) { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-request".into()), + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + gateway_logged("request", "gateway", 0.25), + gateway_logged("other-request", "other", 0.5), ]; - let mut logged = spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); - logged.trace_id = "trace".into(); - logged.span_id = "http".into(); - logged.litellm_call_id = logged_call_id.into(); - let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); - assert_eq!(trace.summary.spend, expected); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.spans[0].spend, + trace.summary.spend, + trace.spans[0].spend_match + ), + (expected, expected, Some(matched)) + ); +} + +#[rstest] +fn call_id_picks_the_call_when_its_response_id_has_a_cache_hit_twin() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-shared".into()), + litellm_traces::CallKey::LiteLlmRequest("gateway".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "gateway".into(), + ..spend("request", "chatcmpl-shared", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "cache-hit".into(), + ..spend("request_cache_hit", "chatcmpl-shared", 0.0) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + ( + trace.summary.spend, + trace.spans[0].spend_log_request_id.as_deref() + ), + (Some(0.25), Some("request")) + ); +} + +#[rstest] +fn wrapper_recording_a_retry_and_its_final_attempt_prices_both() { + let rows = [TraceSpansRow { + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("chatcmpl-retry".into()), + litellm_traces::CallKey::ProviderResponse("chatcmpl-final".into()), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [ + spend("retry", "chatcmpl-retry", 0.25), + spend("final", "chatcmpl-final", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!( + (trace.summary.spend, trace.spans[0].spend_match), + (Some(0.75), Some(SpendMatch::Matched)) + ); +} + +#[rstest] +fn call_id_prices_a_legacy_spend_log_without_litellm_call_id() { + let rows = [TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("legacy".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "") + }]; + let logs = [spend("legacy", "chatcmpl-legacy", 0.25)]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn spend_lookup_collects_only_assigned_ids() { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..row("span", "", "operation", "llm", "") + }; + assert_eq!( + litellm_traces::SpendLookup::new(&[recorded]), + litellm_traces::SpendLookup { + response_ids: vec!["response".into()], + provider_request_ids: Vec::new(), + request_ids: vec!["request".into()], + trace_ids: vec!["trace".into()], + } + ); +} + +#[rstest] +fn upstream_response_id_inside_a_managed_id_joins_spend() { + let rows = [llm("call", "", "agent", "chatcmpl-upstream")]; + let logs = [SpendByResponseIdsRow { + upstream_response_id: "chatcmpl-upstream".into(), + ..spend("request", "resp_managed", 0.25) + }]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); +} + +#[rstest] +fn conflicting_complete_wrapper_cannot_price_an_unrelated_call() { + let rows = [ + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "retry-response".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + llm("call", "wrapper", "agent", "response"), + ]; + let logs = [ + spend("retry", "retry-response", 0.25), + spend("final", "response", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!((trace.summary.spend, trace.summary.priced_calls), (None, 0)); +} + +#[rstest] +fn ambiguous_response_id_keeps_cost_unknown() { + let rows = [llm("llm-1", "", "agent", "response-1")]; + let spend = [ + spend("response-1", "response-1", 0.25), + spend("response-1_cache_hit123", "response-1", 0.0), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!((trace.summary.spend, trace.spans[0].spend), (None, None)); } #[rstest] @@ -827,37 +804,6 @@ fn iso_time_matches_python_isoformat(#[case] ms: i64, #[case] expected: &str) { assert_eq!(iso_time(ms), expected); } -#[rstest] -#[case::narrows_ambiguity("request-a", Some(0.25))] -#[case::conflicting_exact_request("request-c", None)] -fn complete_wrapper_reconciles_ambiguous_response( - #[case] exact_id: &str, - #[case] expected: Option, -) { - let wrapper = TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") - }; - let rows = [ - wrapper, - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("request-a", "response", "team", "", "key", 0.25), - spend("request-b", "response", "team", "", "key", 0.5), - spend("request-c", "other-response", "team", "", "key", 0.75), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - #[rstest] #[case::missing(None, None)] #[case::free(Some(0.0), Some(0.0))] @@ -868,15 +814,10 @@ fn complete_correlation_requires_known_finite_cost( #[case] cost: Option, #[case] expected: Option, ) { - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let logged = SpendByResponseIdsRow { spend: cost, - ..spend("request", "response", "team", "", "key", 0.25) + ..spend("request", "response", 0.25) }; let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); assert_eq!(trace.summary.spend, expected); @@ -884,160 +825,39 @@ fn complete_correlation_requires_known_finite_cost( assert_eq!(trace.spans[0].spend, expected); } -#[rstest] -#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] -#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] -#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] -#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] -fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( - #[case] retry_logged: bool, - #[case] retry_evidence: litellm_traces::CallEvidenceKind, - #[case] expected: Option, -) { - let transport = |id: &str| { - owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec!["transport:".parse().unwrap()], - call_evidence: Some(if id == "first" { - retry_evidence - } else { - litellm_traces::CallEvidenceKind::Complete - }), - ..row(id, "call", "POST", "framework", "") - }, - "team", - "", - "key", - ) - }; - let rows = [ - owned( - llm("call", "", "agent", "final-response"), - "team", - "", - "key", - ), - transport("first"), - transport("second"), - ]; - let logs = [ - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "first".into(), - ..spend("retry", "retry-response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - trace_id: "trace".into(), - span_id: "second".into(), - ..spend("final", "final-response", "team", "", "key", 0.5) - }, - spend("cached", "final-response", "team", "", "key", 0.0), - ]; - let available = if retry_logged { &logs[..] } else { &logs[1..] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, expected); - assert_eq!(trace.agents[0].spend, expected); -} - -#[rstest] -#[case::same_request(false)] -#[case::ambiguous_response(true)] -fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![ - "provider_response:response".parse().unwrap(), - "litellm_request:request".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - spend("request", "response", "team", "", "key", 0.25), - spend("cached", "response", "team", "", "key", 0.5), - ]; - let available = if cached_row { &logs[..] } else { &logs[..1] }; - let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.agents[0].spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::finite(0.25, Some(0.5))] #[case::overflow(f64::MAX, None)] fn trace_cost_requires_a_finite_total(#[case] cost: f64, #[case] expected: Option) { let rows = [ - owned(llm("first", "", "agent", "response-a"), "team", "", "key"), - owned(llm("second", "", "agent", "response-b"), "team", "", "key"), + llm("first", "", "agent", "response-a"), + llm("second", "", "agent", "response-b"), ]; let logs = [ - spend("request-a", "response-a", "team", "", "key", cost), - spend("request-b", "response-b", "team", "", "key", cost), + spend("request-a", "response-a", cost), + spend("request-b", "response-b", cost), ]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.agents[0].spend, expected); } -#[rstest] -fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![ - "litellm_request:retry".parse().unwrap(), - "litellm_request:final".parse().unwrap(), - ], - call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - spend("retry", "retry-response", "team", "", "key", 0.25), - spend("final", "response", "team", "", "key", 0.5), - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.75)); - assert_eq!(trace.agents[0].spend, Some(0.75)); -} - #[rstest] #[case::legacy(None, Some(0.25))] #[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] #[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] #[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] -fn legacy_request_id_fallback_respects_recorded_evidence( +fn stored_response_id_requires_complete_call_evidence( #[case] evidence: Option, #[case] expected: Option, ) { - let span = owned( - TraceSpansRow { - call_evidence: evidence, - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - ); + let span = TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }; let stored = serde_json::to_value(span).unwrap(); let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); - let logs = [spend("request", "response", "team", "", "key", 0.25)]; + let logs = [spend("request", "response", 0.25)]; let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.spans[0].spend, expected); @@ -1058,28 +878,6 @@ fn malformed_stored_span_fields_are_rejected( assert!(serde_json::from_value::(encoded).is_err()); } -#[rstest] -#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] -#[case::complete(litellm_traces::CallEvidenceKind::Complete)] -fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( - #[case] evidence: litellm_traces::CallEvidenceKind, -) { - let recorded = TraceSpansRow { - trace_id: "trace".to_owned(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".to_owned()), - litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), - litellm_traces::CallKey::Transport, - ], - call_evidence: Some(evidence), - ..row("span", "", "operation", "llm", "") - }; - let lookup = litellm_traces::SpendLookup::new(&[recorded]); - assert_eq!(lookup.response_ids, ["response"]); - assert_eq!(lookup.request_ids, ["request"]); - assert_eq!(lookup.trace_ids, ["trace"]); -} - #[rstest] #[case::parent_first(false)] #[case::child_first(true)] @@ -1150,68 +948,23 @@ fn empty_root_preview_uses_the_earliest_agent_or_model_input() { fn repeated_request_ids_preserve_storage_identity(#[case] reverse: bool) { let first = SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) + ..spend("same", "response", 0.25) }; let second = SpendByResponseIdsRow { start_ms: 200, - ..spend("same", "response", "team", "", "key", 0.5) + ..spend("same", "response", 0.5) }; let logs = if reverse { [second, first] } else { [first, second] }; - let rows = [owned( - llm("call", "", "agent", "response"), - "team", - "", - "key", - )]; + let rows = [llm("call", "", "agent", "response")]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, None); assert_eq!(trace.spans[0].spend, None); } -#[rstest] -#[case::gateway(false)] -#[case::transport(true)] -fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { - let rows = [owned( - TraceSpansRow { - trace_id: "trace".into(), - call_keys: vec![ - litellm_traces::CallKey::ProviderResponse("response".into()), - if transport { - litellm_traces::CallKey::Transport - } else { - litellm_traces::CallKey::LiteLlmRequest("gateway".into()) - }, - ], - ..llm("call", "", "agent", "response") - }, - "team", - "", - "key", - )]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - litellm_call_id: "gateway".into(), - trace_id: "trace".into(), - span_id: "call".into(), - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "other".into(), - ..spend("same", "response", "team", "", "key", 0.5) - }, - ]; - let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); - assert_eq!(trace.summary.spend, Some(0.25)); - assert_eq!(trace.spans[0].spend, Some(0.25)); -} - #[rstest] #[case::same_row(true, Some(0.25))] #[case::distinct_rows(false, Some(0.75))] @@ -1220,27 +973,19 @@ fn totals_deduplicate_only_equal_storage_identities( #[case] expected: Option, ) { let rows = [ - owned(llm("first", "", "agent", "a"), "team", "", "key"), - owned( - llm("second", "", "agent", if duplicate { "a" } else { "b" }), - "team", - "", - "key", - ), + llm("first", "", "agent", "a"), + llm("second", "", "agent", if duplicate { "a" } else { "b" }), ]; let logs = [ SpendByResponseIdsRow { start_ms: 100, - ..spend("same", "a", "team", "", "key", 0.25) + ..spend("same", "a", 0.25) }, SpendByResponseIdsRow { start_ms: if duplicate { 100 } else { 200 }, ..spend( "same", if duplicate { "a" } else { "b" }, - "team", - "", - "key", if duplicate { 0.25 } else { 0.5 }, ) }, @@ -1254,80 +999,6 @@ fn totals_deduplicate_only_equal_storage_identities( ); } -#[rstest] -fn conflicting_keys_cannot_agree_on_request_id_alone() { - let rows = [ - owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("wrapper", "", "agent", "") - }, - "team", - "", - "key", - ), - owned( - llm("call", "wrapper", "agent", "response"), - "team", - "", - "key", - ), - ]; - let logs = [ - SpendByResponseIdsRow { - start_ms: 100, - ..spend("same", "response", "team", "", "key", 0.25) - }, - SpendByResponseIdsRow { - start_ms: 200, - litellm_call_id: "gateway".into(), - ..spend("same", "other", "team", "", "key", 0.5) - }, - ]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - None - ); -} - -#[rstest] -#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] -#[case::legacy("", "gateway", "team", "key", Some(0.25))] -#[case::conflict("other", "gateway", "team", "key", None)] -#[case::other_team("gateway", "provider-id", "other-team", "key", None)] -#[case::other_key("gateway", "provider-id", "team", "other-key", None)] -fn gateway_lookup_respects_legacy_fallback_and_ownership( - #[case] call_id: &str, - #[case] request_id: &str, - #[case] team: &str, - #[case] key: &str, - #[case] expected: Option, -) { - let rows = [owned( - TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], - ..llm("call", "", "agent", "") - }, - "team", - "", - "key", - )]; - let logs = [SpendByResponseIdsRow { - litellm_call_id: call_id.into(), - ..spend(request_id, "provider", team, "", key, 0.25) - }]; - assert_eq!( - resolve_trace("trace", "ref", &rows, &logs) - .unwrap() - .summary - .spend, - expected - ); -} - #[rstest] #[case::matching("call-one", "claude_code.tool.execution", SpanStatus::Error)] #[case::other_tool("other-call", "claude_code.tool.execution", SpanStatus::Ok)] @@ -1389,3 +1060,790 @@ fn native_tool_failure_log_matches_by_call_id_without_double_counting( assert_eq!(trace.spans[0].status, expected); assert_eq!(trace.summary.error_count, 1); } + +fn owned_spend( + request_id: &str, + response_id: &str, + team: &str, + user: &str, + key: &str, + cost: f64, +) -> SpendByResponseIdsRow { + SpendByResponseIdsRow { + request_id: request_id.into(), + response_id: response_id.into(), + litellm_call_id: String::new(), + upstream_response_id: String::new(), + provider_request_id: String::new(), + trace_id: String::new(), + span_id: String::new(), + team_id: team.into(), + api_key: key.into(), + user: user.into(), + spend: Some(cost), + start_ms: T0 / MS, + } +} + +fn owned(mut span: TraceSpansRow, team: &str, user: &str, key: &str) -> TraceSpansRow { + span.team_id = team.into(); + span.user_id = user.into(); + span.api_key_hash = key.into(); + span +} + +#[rstest] +fn repeated_response_counts_once_and_other_owners_are_ignored() { + let rows = [ + owned( + row("root", "", "agent", "agent", "agent"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-1", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + owned( + llm("llm-2", "root", "agent", "response-1"), + "team-a", + "", + "key-a", + ), + ]; + let spend = [ + owned_spend("request-other", "response-1", "team-b", "", "key-b", 99.0), + owned_spend("request-1", "response-1", "team-a", "", "key-a", 0.25), + owned_spend( + "request-other-key", + "unrelated-response", + "team-a", + "", + "key-c", + 50.0, + ), + ]; + let trace = resolve_trace("trace-1", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!( + trace + .spans + .iter() + .map(|span| span.spend) + .collect::>(), + [None, Some(0.25), Some(0.25)] + ); +} + +#[rstest] +#[case::key_differs("team", "", "export", "team", "", "request", false)] +#[case::shared_key("team", "", "export", "team", "", "export", true)] +#[case::shared_user("", "user", "export", "", "user", "request", true)] +#[case::teamless_key("", "", "key", "", "", "key", true)] +#[case::other_team("team", "user", "key", "other-team", "user", "key", false)] +#[case::other_user("", "user", "export", "", "other-user", "request", false)] +#[case::no_shared_identity("", "", "export", "", "", "request", false)] +#[case::no_identity("", "", "", "", "", "", false)] +#[case::master_key_without_spend_key("", "", "master", "", "", "", false)] +fn cost_requires_shared_ownership( + #[case] trace_team: &str, + #[case] trace_user: &str, + #[case] trace_key: &str, + #[case] spend_team: &str, + #[case] spend_user: &str, + #[case] spend_key: &str, + #[case] known: bool, +) { + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + trace_team, + trace_user, + trace_key, + ), + owned( + llm("llm", "agent", "agent", "response"), + trace_team, + trace_user, + trace_key, + ), + ]; + let spend = [owned_spend( + "request", "response", spend_team, spend_user, spend_key, 0.25, + )]; + let trace = resolve_trace("trace", "visible-reference", &rows, &spend).unwrap(); + let expected = known.then_some(0.25); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[1].spend, expected); +} + +#[rstest] +#[case::missing_id("missing_id")] +#[case::missing_spend("missing_spend")] +#[case::duplicate_spend("duplicate_spend")] +fn partial_trace_total_counts_only_complete_calls(#[case] failure: &str) { + let second_id = if failure == "missing_id" { + "" + } else { + "second" + }; + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "export", + ), + owned( + llm("first", "agent", "agent", "first"), + "team", + "", + "export", + ), + owned( + llm("second", "agent", "agent", second_id), + "team", + "", + "export", + ), + ]; + let first = owned_spend("first", "first", "team", "", "export", 0.25); + let second = owned_spend("second", "second", "team", "", "export", 0.25); + let duplicate = owned_spend("duplicate", "second", "team", "", "export", 0.25); + let spend = if failure == "duplicate_spend" { + vec![first, second, duplicate] + } else { + vec![first] + }; + let trace = resolve_trace("trace", "ref", &rows, &spend).unwrap(); + assert_eq!(trace.spans[1].spend, Some(0.25)); + assert_eq!(trace.spans[2].spend, None); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (Some(0.25), 1) + ); + assert_eq!( + (trace.agents[0].spend, trace.agents[0].priced_calls), + (Some(0.25), 1) + ); +} + +#[rstest] +fn transport_spans_complete_a_call_without_its_own_id() { + let mut transport = row("http", "llm", "POST", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec!["transport:".parse().unwrap()]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, Some(0.5)); +} + +#[rstest] +#[case::lone_call(1, Some(0.5))] +#[case::two_calls(2, None)] +fn sibling_transports_belong_to_the_only_model_call_under_their_parent( + #[case] calls: usize, + #[case] expected: Option, +) { + let mut transport = at( + row("http", "step", "gateway.request", "framework", ""), + 2, + 10, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![litellm_traces::CallKey::GatewayAttempt]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut rows = vec![ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + for index in 0..calls { + let mut call = llm(&format!("chat-{index}"), "step", "agent", ""); + call.call_evidence = None; + rows.push(owned(call, "team", "", "key")); + } + let mut logged = owned_spend("request", "", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::without_tool_http_sibling(None, false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::after_call(Some((200, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_without_spend(Some((10, 10)), false, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::inside_call_with_unrelated_spend(Some((10, 10)), true, litellm_traces::CallKey::Transport, Some(0.5))] +#[case::missing_gateway_attempt(Some((10, 10)), false, litellm_traces::CallKey::GatewayAttempt, None)] +fn sibling_transport_does_not_lose_model_call_spend( + #[case] transport_timing: Option<(i64, u64)>, + #[case] unrelated_spend: bool, + #[case] key: litellm_traces::CallKey, + #[case] expected: Option, +) { + let call = owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![litellm_traces::CallKey::ProviderResponse( + "chatcmpl-1".into(), + )], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("chat", "step", "agent", "chatcmpl-1") + }, + "team", + "", + "key", + ); + let base_rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(row("step", "agent", "step", "chain", ""), "team", "", "key"), + call, + ]; + let rows: Vec<_> = base_rows + .into_iter() + .chain(transport_timing.map(|(start, duration)| { + let mut transport = at( + row("tool-http", "step", "GET", "framework", ""), + start, + duration, + ); + transport.trace_id = "trace".into(); + transport.call_keys = vec![key]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + owned(transport, "team", "", "key") + })) + .collect(); + let logs: Vec<_> = std::iter::once(owned_spend( + "chatcmpl-1", + "chatcmpl-1", + "team", + "", + "key", + 0.5, + )) + .chain(unrelated_spend.then(|| SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "tool-http".into(), + ..owned_spend("unrelated", "unrelated", "team", "", "key", 0.75) + })) + .collect(); + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::agreeing_ids( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_id(litellm_traces::CallKey::Transport, "call-b", None, None)] +#[case::conflicting_response_id( + litellm_traces::CallKey::Transport, + "call-a", + Some("response-b"), + None +)] +#[case::conflicting_gateway_and_response( + litellm_traces::CallKey::Transport, + "call-b", + Some("response-b"), + None +)] +#[case::agreeing_gateway_attempt( + litellm_traces::CallKey::GatewayAttempt, + "call-a", + Some("response-a"), + Some(0.25) +)] +#[case::conflicting_gateway_attempt(litellm_traces::CallKey::GatewayAttempt, "call-b", None, None)] +fn gateway_attempt_identifiers_must_match_one_spend_row( + #[case] transport: litellm_traces::CallKey, + #[case] call_id: &str, + #[case] response_id: Option<&str>, + #[case] expected: Option, +) { + let keys = [ + transport, + litellm_traces::CallKey::LiteLlmRequest(call_id.into()), + ] + .into_iter() + .chain(response_id.map(|id| litellm_traces::CallKey::ProviderResponse(id.into()))) + .collect(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(llm("call", "agent", "agent", ""), "team", "", "key"), + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: keys, + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..row("attempt", "call", "gateway.request", "framework", "") + }, + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + litellm_call_id: "call-a".into(), + trace_id: "trace".into(), + span_id: "attempt".into(), + ..owned_spend("request-a", "response-a", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + litellm_call_id: "call-b".into(), + trace_id: "trace".into(), + span_id: "other-attempt".into(), + ..owned_spend("request-b", "response-b", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); + assert_eq!(trace.spans[2].spend, expected); +} + +#[rstest] +#[case::legacy_row("", Some(0.5))] +#[case::other_call("other-call", None)] +fn gateway_id_miss_only_vetoes_rows_that_carry_a_call_id( + #[case] logged_call_id: &str, + #[case] expected: Option, +) { + let mut transport = row("http", "llm", "gateway.request", "framework", ""); + transport.trace_id = "trace".into(); + transport.call_keys = vec![ + "transport:".parse().unwrap(), + "litellm_request:gateway-call".parse().unwrap(), + ]; + transport.call_evidence = Some(litellm_traces::CallEvidenceKind::Complete); + let mut call = llm("llm", "agent", "agent", ""); + call.trace_id = "trace".into(); + let rows = [ + owned( + row("agent", "", "agent", "agent", "agent"), + "team", + "", + "key", + ), + owned(call, "team", "", "key"), + owned(transport, "team", "", "key"), + ]; + let mut logged = owned_spend("chatcmpl-1", "chatcmpl-1", "team", "", "key", 0.5); + logged.trace_id = "trace".into(); + logged.span_id = "http".into(); + logged.litellm_call_id = logged_call_id.into(); + let trace = resolve_trace("trace", "ref", &rows, &[logged]).unwrap(); + assert_eq!(trace.summary.spend, expected); +} + +#[rstest] +#[case::narrows_ambiguity("request-a", Some(0.25))] +#[case::conflicting_exact_request("request-c", None)] +fn complete_wrapper_reconciles_ambiguous_response( + #[case] exact_id: &str, + #[case] expected: Option, +) { + let wrapper = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest(exact_id.to_owned())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..owned(llm("wrapper", "", "agent", ""), "team", "", "key") + }; + let rows = [ + wrapper, + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("request-a", "response", "team", "", "key", 0.25), + owned_spend("request-b", "response", "team", "", "key", 0.5), + owned_spend("request-c", "other-response", "team", "", "key", 0.75), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::complete_retry(true, litellm_traces::CallEvidenceKind::Complete, Some(0.75))] +#[case::missing_retry(false, litellm_traces::CallEvidenceKind::Complete, None)] +#[case::unknown_retry(true, litellm_traces::CallEvidenceKind::Unknown, None)] +#[case::partial_retry(true, litellm_traces::CallEvidenceKind::Partial, None)] +fn transports_preserve_retry_spend_without_counting_unrelated_cached_rows( + #[case] retry_logged: bool, + #[case] retry_evidence: litellm_traces::CallEvidenceKind, + #[case] expected: Option, +) { + let transport = |id: &str| { + owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec!["transport:".parse().unwrap()], + call_evidence: Some(if id == "first" { + retry_evidence + } else { + litellm_traces::CallEvidenceKind::Complete + }), + ..row(id, "call", "POST", "framework", "") + }, + "team", + "", + "key", + ) + }; + let rows = [ + owned( + llm("call", "", "agent", "final-response"), + "team", + "", + "key", + ), + transport("first"), + transport("second"), + ]; + let logs = [ + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "first".into(), + ..owned_spend("retry", "retry-response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "second".into(), + ..owned_spend("final", "final-response", "team", "", "key", 0.5) + }, + owned_spend("cached", "final-response", "team", "", "key", 0.0), + ]; + let available = if retry_logged { &logs[..] } else { &logs[1..] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.agents[0].spend, expected); +} + +#[rstest] +#[case::same_request(false)] +#[case::ambiguous_response(true)] +fn multiple_identifiers_for_one_request_count_its_spend_once(#[case] cached_row: bool) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![ + "provider_response:response".parse().unwrap(), + "litellm_request:request".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + owned_spend("request", "response", "team", "", "key", 0.25), + owned_spend("cached", "response", "team", "", "key", 0.5), + ]; + let available = if cached_row { &logs[..] } else { &logs[..1] }; + let trace = resolve_trace("trace", "ref", &rows, available).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.agents[0].spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn complete_wrapper_accounts_for_retries_missing_from_the_call_span() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![ + "litellm_request:retry".parse().unwrap(), + "litellm_request:final".parse().unwrap(), + ], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + owned_spend("retry", "retry-response", "team", "", "key", 0.25), + owned_spend("final", "response", "team", "", "key", 0.5), + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.75)); + assert_eq!(trace.agents[0].spend, Some(0.75)); +} + +#[rstest] +#[case::legacy(None, Some(0.25))] +#[case::unknown(Some(litellm_traces::CallEvidenceKind::Unknown), None)] +#[case::partial(Some(litellm_traces::CallEvidenceKind::Partial), None)] +#[case::complete(Some(litellm_traces::CallEvidenceKind::Complete), Some(0.25))] +fn legacy_request_id_fallback_respects_recorded_evidence( + #[case] evidence: Option, + #[case] expected: Option, +) { + let span = owned( + TraceSpansRow { + call_evidence: evidence, + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + ); + let stored = serde_json::to_value(span).unwrap(); + let decoded: TraceSpansRow = serde_json::from_value(stored).unwrap(); + let logs = [owned_spend("request", "response", "team", "", "key", 0.25)]; + let trace = resolve_trace("trace", "ref", &[decoded], &logs).unwrap(); + assert_eq!(trace.summary.spend, expected); + assert_eq!(trace.spans[0].spend, expected); +} + +#[rstest] +#[case::unknown(litellm_traces::CallEvidenceKind::Unknown)] +#[case::complete(litellm_traces::CallEvidenceKind::Complete)] +fn spend_lookup_fetches_recorded_keys_before_resolving_completeness( + #[case] evidence: litellm_traces::CallEvidenceKind, +) { + let recorded = TraceSpansRow { + trace_id: "trace".to_owned(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".to_owned()), + litellm_traces::CallKey::LiteLlmRequest("request".to_owned()), + litellm_traces::CallKey::Transport, + ], + call_evidence: Some(evidence), + ..row("span", "", "operation", "llm", "") + }; + let lookup = litellm_traces::SpendLookup::new(&[recorded]); + assert_eq!(lookup.response_ids, ["response"]); + assert_eq!(lookup.request_ids, ["request"]); + assert_eq!(lookup.trace_ids, ["trace"]); +} + +#[rstest] +#[case::gateway(false)] +#[case::transport(true)] +fn independent_key_disambiguates_repeated_request_ids(#[case] transport: bool) { + let rows = [owned( + TraceSpansRow { + trace_id: "trace".into(), + call_keys: vec![ + litellm_traces::CallKey::ProviderResponse("response".into()), + if transport { + litellm_traces::CallKey::Transport + } else { + litellm_traces::CallKey::LiteLlmRequest("gateway".into()) + }, + ], + ..llm("call", "", "agent", "response") + }, + "team", + "", + "key", + )]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + litellm_call_id: "gateway".into(), + trace_id: "trace".into(), + span_id: "call".into(), + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "other".into(), + ..owned_spend("same", "response", "team", "", "key", 0.5) + }, + ]; + let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); + assert_eq!(trace.summary.spend, Some(0.25)); + assert_eq!(trace.spans[0].spend, Some(0.25)); +} + +#[rstest] +fn conflicting_keys_cannot_agree_on_request_id_alone() { + let rows = [ + owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("wrapper", "", "agent", "") + }, + "team", + "", + "key", + ), + owned( + llm("call", "wrapper", "agent", "response"), + "team", + "", + "key", + ), + ]; + let logs = [ + SpendByResponseIdsRow { + start_ms: 100, + ..owned_spend("same", "response", "team", "", "key", 0.25) + }, + SpendByResponseIdsRow { + start_ms: 200, + litellm_call_id: "gateway".into(), + ..owned_spend("same", "other", "team", "", "key", 0.5) + }, + ]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + None + ); +} + +#[rstest] +#[case::gateway("gateway", "provider-id", "team", "key", Some(0.25))] +#[case::legacy("", "gateway", "team", "key", Some(0.25))] +#[case::conflict("other", "gateway", "team", "key", None)] +#[case::other_team("gateway", "provider-id", "other-team", "key", None)] +#[case::other_key("gateway", "provider-id", "team", "other-key", None)] +fn gateway_lookup_respects_legacy_fallback_and_ownership( + #[case] call_id: &str, + #[case] request_id: &str, + #[case] team: &str, + #[case] key: &str, + #[case] expected: Option, +) { + let rows = [owned( + TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + ..llm("call", "", "agent", "") + }, + "team", + "", + "key", + )]; + let logs = [SpendByResponseIdsRow { + litellm_call_id: call_id.into(), + ..owned_spend(request_id, "provider", team, "", key, 0.25) + }]; + assert_eq!( + resolve_trace("trace", "ref", &rows, &logs) + .unwrap() + .summary + .spend, + expected + ); +} + +#[rstest] +#[case::native_id("provider_request:req_native", false, true)] +#[case::historical_id("provider_response:req_native", false, true)] +#[case::legacy_message_id("provider_response:msg_native", false, true)] +#[case::transport("", true, true)] +#[case::no_correlation("", false, false)] +fn native_cost_uses_typed_ids_or_the_original_transport( + #[case] encoded: &str, + #[case] traceparent: bool, + #[case] priced: bool, + #[values(false, true)] grouped: bool, +) { + let call = TraceSpansRow { + trace_id: if grouped { + "session-trace" + } else { + "original-trace" + } + .into(), + original_trace_id: if grouped { "original-trace" } else { "" }.into(), + name: "claude_code.llm_request".into(), + framework: "claude-code".into(), + call_keys: encoded.parse().into_iter().collect(), + call_evidence: Some(litellm_traces::CallEvidenceKind::Unknown), + ..llm("native-call", "", "agent", "") + }; + let log = SpendByResponseIdsRow { + provider_request_id: "req_native".into(), + trace_id: if traceparent { "original-trace" } else { "" }.into(), + span_id: if traceparent { "native-call" } else { "" }.into(), + ..spend("gateway-log", "msg_native", 0.25) + }; + let lookup = litellm_traces::SpendLookup::new(std::slice::from_ref(&call)); + let trace = resolve_trace("trace", "ref", &[call], &[log]).unwrap(); + assert_eq!( + (trace.summary.spend, trace.summary.priced_calls), + (priced.then_some(0.25), u64::from(priced)) + ); + assert_eq!(trace.spans[0].spend, priced.then_some(0.25)); + if encoded.is_empty() { + assert_eq!(lookup.trace_ids, ["original-trace"]); + } else if !encoded.ends_with("msg_native") { + assert_eq!(lookup.provider_request_ids, ["req_native"]); + assert!(lookup.response_ids.is_empty()); + } +} + +#[rstest] +fn provider_request_id_cannot_match_a_message_id_of_the_same_value() { + let call = TraceSpansRow { + call_keys: vec![litellm_traces::CallKey::ProviderRequest( + "req_native".into(), + )], + ..llm("call", "", "agent", "") + }; + let rows = [spend("unrelated", "req_native", 0.25)]; + let trace = resolve_trace("trace", "ref", &[call], &rows).unwrap(); + assert_eq!(trace.summary.spend, None); + assert_eq!(trace.spans[0].spend_match, Some(SpendMatch::NoSpendLog)); +} diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index 9d02c84f6e8..a0bae6685de 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -35,7 +35,7 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import ( from litellm.litellm_core_utils.logging_utils import ( _assemble_complete_response_from_streaming_chunks, ) -from litellm.types.caching import EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding +from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.openai import ResponsesAPIResponse @@ -43,9 +43,11 @@ from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CachingDetails, CallTypes, + Choices, Embedding, EmbeddingResponse, ModelResponse, + TextChoices, TextCompletionResponse, TranscriptionResponse, Usage, @@ -111,6 +113,43 @@ def _is_chat_completion_cached_dict(cached_result: dict) -> bool: return "choices" in cached_result +def _starts_content_block(event: object) -> bool: + text: Final = event.decode("utf-8", errors="replace") if isinstance(event, bytes) else event + return isinstance(text, str) and "event: content_block_start" in text.splitlines() + + +def is_response_without_output(result: object) -> bool: + if isinstance(result, (ModelResponse, TextCompletionResponse)): + return not result.choices + if isinstance(result, ResponsesAPIResponse): + return not getattr(result, "output", None) + if not isinstance(result, dict): + return False + cached_stream_events: Final = result.get(CACHED_STREAM_EVENTS_KEY) + if isinstance(cached_stream_events, (list, tuple)): + return not any(_starts_content_block(event) for event in cached_stream_events) + if "choices" in result: + return not result["choices"] + if result.get("object") == "response": + return not result.get("output") + if result.get("type") == "message": + return not result.get("content") + return False + + +def _choice_carries_output(choice: Choices | TextChoices) -> bool: + payload: Final[Mapping[str, object]] = ( + choice.message.model_dump(exclude_none=True) if isinstance(choice, Choices) else {"text": choice.text} + ) + return any(key != "role" and value not in ("", [], {}) for key, value in payload.items()) + + +def _assembled_stream_without_output(response: ModelResponse | TextCompletionResponse) -> bool: + """The stream wrapper closes a stream whose chunks carried no choice with an empty choice of its own, so the + assembled response has choices and the no-output check on it alone would store that empty answer.""" + return not any(_choice_carries_output(choice) for choice in response.choices) + + def _stream_replay_requested(kwargs: Mapping[str, object]) -> bool: if kwargs.get("stream", False) is True: return True @@ -398,6 +437,9 @@ class LLMCachingHandler: print_verbose("Checking Sync Cache") with response_cache_phase("get"): cached_result = litellm.cache.get_cache(**new_kwargs) + if is_response_without_output(cached_result): + verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss") + return CachingHandlerResponse(cached_result=None) if cached_result is not None: if "detail" in cached_result: # implies an error occurred @@ -836,8 +878,19 @@ class LLMCachingHandler: cache_key=self.preset_cache_key, **request_kwargs, ) + if is_response_without_output(cached_result): + verbose_logger.debug("LiteLLM Cache: cached response has no output, treating it as a miss") + self._forget_worker_copy() + return None return cached_result + def _forget_worker_copy(self) -> None: + """Drop this worker's memory copy of a stored response without output, so the next read reaches Redis, + where the refill lands: the text completion and messages writers skip the memory tier.""" + if self.dual_cache is None or self.preset_cache_key is None: + return + self.dual_cache.in_memory_cache.delete_cache(self.preset_cache_key) + def _convert_cached_result_to_model_response( self, cached_result: Any, @@ -1063,6 +1116,9 @@ class LLMCachingHandler: if litellm.cache is None: return + if is_response_without_output(result): + verbose_logger.debug("LiteLLM Cache: not caching a response with no output") + return cache: Final = litellm.cache new_kwargs: Final = kwargs.copy() @@ -1113,6 +1169,11 @@ class LLMCachingHandler: """ Sync internal method to add the result to the cache """ + if litellm.cache is None: + return + if is_response_without_output(result): + verbose_logger.debug("LiteLLM Cache: not caching a response with no output") + return new_kwargs: Final = kwargs.copy() new_kwargs.update( @@ -1121,8 +1182,6 @@ class LLMCachingHandler: args, ) ) - if litellm.cache is None: - return if self._should_store_result_in_cache(original_function=self.original_function, kwargs=new_kwargs): with response_cache_phase("set"): @@ -1201,13 +1260,13 @@ class LLMCachingHandler: is_async=True, ) ) - # if a complete_streaming_response is assembled, add it to the cache - if complete_streaming_response is not None: - await self.async_set_cache( - result=complete_streaming_response, - original_function=self.original_function, - kwargs=self.request_kwargs, - ) + if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response): + return + await self.async_set_cache( + result=complete_streaming_response, + original_function=self.original_function, + kwargs=self.request_kwargs, + ) def _sync_add_streaming_response_to_cache(self, processed_chunk: ModelResponse): """ @@ -1224,12 +1283,12 @@ class LLMCachingHandler: ) ) - # if a complete_streaming_response is assembled, add it to the cache - if complete_streaming_response is not None: - self.sync_set_cache( - result=complete_streaming_response, - kwargs=self.request_kwargs, - ) + if complete_streaming_response is None or _assembled_stream_without_output(complete_streaming_response): + return + self.sync_set_cache( + result=complete_streaming_response, + kwargs=self.request_kwargs, + ) def _update_litellm_logging_obj_environment( self, diff --git a/litellm/integrations/arize/arize.py b/litellm/integrations/arize/arize.py index 58024cb2e1e..24279b443aa 100644 --- a/litellm/integrations/arize/arize.py +++ b/litellm/integrations/arize/arize.py @@ -17,7 +17,7 @@ from litellm.integrations.arize._utils import ArizeOTELAttributes from litellm.integrations.opentelemetry import _MAX_DYNAMIC_TRACER_PROVIDERS, OpenTelemetry, OpenTelemetryConfig from litellm.types.integrations.arize import ArizeConfig from litellm.types.services import ServiceLoggerPayload -from litellm.types.utils import StandardCallbackDynamicParams +from litellm.types.utils import ArizeOtlpProtocol, StandardCallbackDynamicParams if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -174,7 +174,7 @@ class ArizeLogger(OpenTelemetry): _utils.set_attributes(span, kwargs, response_obj, ArizeOTELAttributes) @staticmethod - def get_arize_config() -> ArizeConfig: + def get_arize_config(otlp_protocol: ArizeOtlpProtocol | None = None) -> ArizeConfig: """ Helper function to get Arize configuration. @@ -195,7 +195,19 @@ class ArizeLogger(OpenTelemetry): endpoint = None protocol: Protocol = "otlp_grpc" - if grpc_endpoint: + if otlp_protocol == "http/protobuf": + protocol = "otlp_http" + if http_endpoint: + endpoint = http_endpoint + elif grpc_endpoint: + base: Final = grpc_endpoint.rstrip("/") + endpoint = f"{base}/traces" if base.endswith("/v1") else base + else: + endpoint = "https://otlp.arize.com/v1/traces" + elif otlp_protocol == "grpc": + protocol = "otlp_grpc" + endpoint = grpc_endpoint or http_endpoint or "https://otlp.arize.com/v1" + elif grpc_endpoint: protocol = "otlp_grpc" endpoint = grpc_endpoint elif http_endpoint: diff --git a/litellm/integrations/clickhouse/clickhouse_spend_logger.py b/litellm/integrations/clickhouse/clickhouse_spend_logger.py index 588f81ec50a..6c9f726056b 100644 --- a/litellm/integrations/clickhouse/clickhouse_spend_logger.py +++ b/litellm/integrations/clickhouse/clickhouse_spend_logger.py @@ -19,6 +19,7 @@ from litellm._logging import verbose_logger from litellm.integrations.clickhouse.clickhouse_batch_logger import ClickHouseBatchLogger from litellm.integrations.clickhouse.context import is_lens_analysis from litellm.integrations.clickhouse.schema import SPEND_LOGS_TABLE +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.tracing.types import SpendLogRecord from litellm.types.utils import StandardLoggingPayload @@ -155,6 +156,11 @@ def spend_log_row_from_payload(payload: StandardLoggingPayload, kwargs: Mapping[ return SpendLogRecord( request_id=request_id, response_id=strip_cache_hit_suffix(request_id), + provider_request_id=( + get_provider_request_id(kwargs.get("response_headers")) + or get_provider_request_id(hidden_params.get("additional_headers")) + or "" + ), litellm_call_id=payload.get("litellm_call_id") or "", call_type=payload.get("call_type") or "", api_key=metadata.get("user_api_key_hash") or "", diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 67450684658..91341a57634 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -63,7 +63,7 @@ def _langfuse_destination(params: StandardCallbackDynamicParams) -> "_Destinatio def _arize_destination(params: StandardCallbackDynamicParams) -> "_Destination | None": from litellm.integrations.arize.arize import ArizeLogger - config: Final = ArizeLogger.get_arize_config() + config: Final = ArizeLogger.get_arize_config(params.get("arize_otlp_protocol")) return (config.endpoint, config.protocol) diff --git a/litellm/integrations/websearch_interception/handler.py b/litellm/integrations/websearch_interception/handler.py index 1db94e82066..15413140d9e 100644 --- a/litellm/integrations/websearch_interception/handler.py +++ b/litellm/integrations/websearch_interception/handler.py @@ -1557,8 +1557,8 @@ class WebSearchInterceptionLogger(CustomLogger): search_litellm_params = dict[str, object](tool_params) search_provider = tool_params.get("search_provider") - # Fallback to perplexity if no router or no search tools configured if not search_provider: + self._authorize_unregistered_search_fallback(kwargs=kwargs) search_provider = "perplexity" verbose_logger.debug( "WebSearchInterception: No search tools configured in router, using default provider '%s'", @@ -1623,6 +1623,18 @@ class WebSearchInterceptionLogger(CustomLogger): verbose_logger.error("WebSearchInterception: Search failed for '%s': %s", query, e) raise + def _authorize_unregistered_search_fallback(self, kwargs: Mapping[str, object] | None) -> None: + user_api_key_auth: Final = self._get_user_api_key_auth_from_kwargs(kwargs) + if user_api_key_auth is None: + return + + from litellm.proxy.auth.auth_checks import check_unregistered_search_fallback, typed_general_settings + from litellm.proxy.proxy_server import general_settings + + check_unregistered_search_fallback( + valid_token=user_api_key_auth, general_settings=typed_general_settings(general_settings) + ) + async def _authorize_search_tool( self, search_tool: Mapping[str, object], @@ -1636,36 +1648,9 @@ class WebSearchInterceptionLogger(CustomLogger): if user_api_key_auth is None: return - from litellm.proxy.auth.auth_checks import ( - can_key_call_search_tool, - can_team_call_search_tool, - get_team_object, - ) + from litellm.proxy.auth.auth_checks import can_token_call_search_tool - await can_key_call_search_tool( - search_tool_name=search_tool_name, - valid_token=user_api_key_auth, - ) - - team_id: Final[str | None] = getattr(user_api_key_auth, "team_id", None) - if team_id: - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) - - team_object: Final = await get_team_object( - team_id=team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=getattr(user_api_key_auth, "parent_otel_span", None), - proxy_logging_obj=proxy_logging_obj, - ) - await can_team_call_search_tool( - search_tool_name=search_tool_name, - team_object=team_object, - ) + await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=user_api_key_auth) @staticmethod def _build_search_request_metadata( diff --git a/litellm/litellm_core_utils/core_helpers.py b/litellm/litellm_core_utils/core_helpers.py index 4fa78640e24..76ea2c25159 100644 --- a/litellm/litellm_core_utils/core_helpers.py +++ b/litellm/litellm_core_utils/core_helpers.py @@ -11,6 +11,7 @@ import httpx from pydantic import TypeAdapter, ValidationError from litellm._logging import verbose_logger +from litellm.litellm_core_utils.llm_response_utils.get_headers import get_provider_request_id from litellm.types.llms.openai import AllMessageValues, OpenAIChatCompletionFinishReason if TYPE_CHECKING: @@ -495,7 +496,8 @@ def process_response_headers( **processed_headers, **additional_headers, } - return additional_headers + request_id: Final = get_provider_request_id(response_headers) + return {**additional_headers, **({"request-id": request_id} if request_id is not None else {})} def preserve_upstream_non_openai_attributes( diff --git a/litellm/litellm_core_utils/error_normalization.py b/litellm/litellm_core_utils/error_normalization.py index 726fdcd1540..00317e502ec 100644 --- a/litellm/litellm_core_utils/error_normalization.py +++ b/litellm/litellm_core_utils/error_normalization.py @@ -107,6 +107,10 @@ _PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType( "key_vector_store_access_denied": PERMISSION_DENIED, "team_vector_store_access_denied": PERMISSION_DENIED, "org_vector_store_access_denied": PERMISSION_DENIED, + "user_vector_store_access_denied": PERMISSION_DENIED, + "key_search_tool_access_denied": PERMISSION_DENIED, + "team_search_tool_access_denied": PERMISSION_DENIED, + "user_search_tool_access_denied": PERMISSION_DENIED, "tool_access_denied": PERMISSION_DENIED, "team_member_permission_error": PERMISSION_DENIED, "not_found_error": RESOURCE_NOT_FOUND, diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index 9ac78e3b917..d2ba17e169a 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -4,7 +4,12 @@ from contextlib import contextmanager from contextvars import ContextVar from typing import Any, Final -from litellm.types.utils import OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams +from litellm.types.utils import ( + ARIZE_OTLP_PROTOCOLS, + OTEL_SPAN_SCOPES, + TRUSTED_CALLBACK_VARS_FIELD, + StandardCallbackDynamicParams, +) _CLIENT_CALLBACK_METADATA_SLOTS: Final[tuple[str, ...]] = ("litellm_metadata", "metadata") _inherited_message_logging_disabled: Final[ContextVar[bool]] = ContextVar( @@ -81,6 +86,11 @@ def validate_langfuse_span_scope_value(value: str) -> None: raise ValueError(f"Invalid langfuse_span_scope {value!r}: must be one of {sorted(OTEL_SPAN_SCOPES)}") +def validate_arize_otlp_protocol_value(value: str) -> None: + if value not in ARIZE_OTLP_PROTOCOLS: + raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}") + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/litellm_core_utils/llm_response_utils/get_headers.py b/litellm/litellm_core_utils/llm_response_utils/get_headers.py index d04abcb6e7b..b86cc954b0e 100644 --- a/litellm/litellm_core_utils/llm_response_utils/get_headers.py +++ b/litellm/litellm_core_utils/llm_response_utils/get_headers.py @@ -1,6 +1,20 @@ from collections.abc import Mapping from typing import Final +from pydantic import TypeAdapter, ValidationError + +_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_REQUEST_ID_HEADERS: Final = ("request-id", "x-request-id", "llm_provider-request-id", "llm_provider-x-request-id") + + +def get_provider_request_id(headers: object) -> str | None: + try: + values: Final = _HEADERS.validate_python(headers) + except ValidationError: + return None + normalized: Final = {name.lower(): value for name, value in values.items()} + return next((normalized[name] for name in _REQUEST_ID_HEADERS if normalized.get(name)), None) + def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> dict: """ @@ -29,7 +43,9 @@ def get_response_headers(_response_headers: Mapping[str, str] | None = None) -> if "x-ratelimit-remaining-tokens" in _response_headers: openai_headers["x-ratelimit-remaining-tokens"] = _response_headers["x-ratelimit-remaining-tokens"] llm_provider_headers: Final = _get_llm_provider_headers(_response_headers) - return {**llm_provider_headers, **openai_headers} + request_id: Final = get_provider_request_id(_response_headers) + native_headers: Final = {"request-id": request_id} if request_id is not None else {} + return {**llm_provider_headers, **openai_headers, **native_headers} def _get_llm_provider_headers(response_headers: Mapping[str, str]) -> dict: diff --git a/litellm/llms/anthropic/pass_through/messages/response_cache.py b/litellm/llms/anthropic/pass_through/messages/response_cache.py index 5dc26c934ab..40dcbd9c8c8 100644 --- a/litellm/llms/anthropic/pass_through/messages/response_cache.py +++ b/litellm/llms/anthropic/pass_through/messages/response_cache.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Final, cast import litellm from litellm._logging import verbose_logger -from litellm.caching.caching_handler import create_cache_write_task +from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( AnthropicMessagesStreamingResponse, BaseAnthropicMessagesStreamingIterator, @@ -13,6 +13,7 @@ from litellm.llms.anthropic.pass_through.messages.streaming_iterator import ( _is_provider_error_chunk, aclose_if_supported, ) +from litellm.types.caching import CACHED_STREAM_EVENTS_KEY if TYPE_CHECKING: from litellm.caching.caching_handler import LLMCachingHandler @@ -20,8 +21,6 @@ if TYPE_CHECKING: from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponseStream -CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events" - _EMPTY_MAPPING: Final[Mapping[str, object]] = MappingProxyType({}) _SSE_EVENT_BOUNDARY: Final = re.compile(r"(?<=\n\n)") @@ -114,6 +113,9 @@ class AnthropicMessagesStreamCacheWriter: verbose_logger.exception("Anthropic Messages stream cache write failed: %s", e) return cached_payload: Final = {CACHED_STREAM_EVENTS_KEY: events} + if is_response_without_output(cached_payload): + verbose_logger.debug("LiteLLM Cache: not caching a stream with no content blocks") + return dual_cache: Final = self.caching_handler.dual_cache async def _write() -> None: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 86c36a7b413..53ea85a2740 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -2214,6 +2214,7 @@ class BaseLLMHTTPHandler: # used for logging + cost tracking logging_obj.model_call_details["httpx_response"] = response + logging_obj.model_call_details["response_headers"] = dict(response.headers) initial_response: AsyncIterator | AnthropicMessagesResponse if stream: diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 291e991f6bd..d02d1f29ded 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -25,6 +25,7 @@ from typing_extensions import NotRequired, ReadOnly, Required, TypedDict from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_arize_otlp_protocol_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, @@ -2383,6 +2384,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_environment_value(callback_vars[key]) if key == "langfuse_span_scope": validate_langfuse_span_scope_value(callback_vars[key]) + if key == "arize_otlp_protocol": + validate_arize_otlp_protocol_value(callback_vars[key]) return values @@ -3019,6 +3022,14 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): None, description="When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata.", ) + vector_store_deny_by_default: bool = Field( + default=False, + description="When True, a vector store must be explicitly listed in object_permission.vector_stores: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing. Dashboard session keys are not yet covered", + ) + search_tool_deny_by_default: bool = Field( + default=False, + description="When True, a search tool must be explicitly listed in object_permission.search_tools: a virtual key needs its own grant plus its team's, a keyless team member needs the team's, and a user with neither needs their own. A missing permission record, an empty list, or an unresolved team grants nothing, and the unregistered search fallback is denied. The master key and dashboard sessions are exempt", + ) missing_session_id: Literal["generate", "reject", "omit"] | None = Field( None, description="What to do with LLM API requests that carry no session id (x-litellm-session-id header, metadata.session_id, etc.). 'generate' stamps one id into litellm_session_id, litellm_trace_id and metadata.session_id so SpendLogs and logging callbacks agree; 'reject' returns 400; 'omit' leaves SpendLogs.session_id null, matching callbacks such as Langfuse that only record a client-established metadata.session_id. Unset keeps the legacy behavior where SpendLogs falls back to the trace id while callbacks get no session id.", @@ -4509,6 +4520,26 @@ class ProxyErrorTypes(str, enum.Enum): Organization does not have access to the vector store """ + user_vector_store_access_denied = "user_vector_store_access_denied" + """ + User does not have access to the vector store + """ + + key_search_tool_access_denied = "key_search_tool_access_denied" + """ + Key does not have access to the search tool + """ + + team_search_tool_access_denied = "team_search_tool_access_denied" + """ + Team does not have access to the search tool + """ + + user_search_tool_access_denied = "user_search_tool_access_denied" + """ + User does not have access to the search tool + """ + team_member_already_in_team = "team_member_already_in_team" """ Team member is already in team @@ -4543,7 +4574,7 @@ class ProxyErrorTypes(str, enum.Enum): @classmethod def get_vector_store_access_error_type_for_object( - cls, object_type: Literal["key", "team", "org"] + cls, object_type: Literal["key", "team", "org", "user"] ) -> "ProxyErrorTypes": """ Get the vector store access error type for object_type @@ -4554,6 +4585,18 @@ class ProxyErrorTypes(str, enum.Enum): return cls.team_vector_store_access_denied elif object_type == "org": return cls.org_vector_store_access_denied + elif object_type == "user": + return cls.user_vector_store_access_denied + + @classmethod + def get_search_tool_access_error_type_for_object( + cls, object_type: Literal["key", "team", "user"] + ) -> "ProxyErrorTypes": + return { + "key": cls.key_search_tool_access_denied, + "team": cls.team_search_tool_access_denied, + "user": cls.user_search_tool_access_denied, + }[object_type] DB_CONNECTION_ERROR_TYPES: Final = ( diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 460b25ddac7..33fcac6230a 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -14,7 +14,9 @@ import math import re import time from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass from functools import partial +from itertools import chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Generic, Literal, Optional, Protocol, TypeAlias @@ -34,6 +36,7 @@ from litellm.constants import ( DEFAULT_MAX_RECURSE_DEPTH, EMAIL_BUDGET_ALERT_MAX_SPEND_ALERT_PERCENTAGE, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, + LITELLM_PROXY_MASTER_KEY_ALIAS, MODEL_ACCESS_GROUP_REGISTRY_MAX_SIZE, REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, @@ -44,7 +47,9 @@ from litellm.litellm_core_utils.safe_json_loads import safe_json_loads from litellm.models.project import LiteLLM_ProjectTable from litellm.proxy._types import ( RBAC_ROLES, + UI_TEAM_ID, CallInfo, + ConfigGeneralSettings, LiteLLM_AccessGroupTable, LiteLLM_BudgetTable, LiteLLM_EndUserTable, @@ -308,17 +313,14 @@ def _user_table(repo: _PrismaTableHolder[_PrismaUserRow]) -> _PrismaAuthTable[_P return _DeadlineBoundedTable(repo.table, "user") +GrantLayer = Literal["key", "team", "user"] + + class _VectorStorePermissionsRow(Protocol): @property def vector_stores(self) -> Sequence[str] | None: ... -def _object_permission_table( - repo: _PrismaTableHolder[_VectorStorePermissionsRow], -) -> _PrismaAuthTable[_VectorStorePermissionsRow]: - return _DeadlineBoundedTable(repo.table, "object_permission") - - class _PrismaTagRow(Protocol): tag_name: str @@ -388,6 +390,9 @@ def _typed_request_body(request_body: dict) -> Mapping[str, object]: return request_body +typed_general_settings: Final = _typed_request_body + + class _JsonLoadsObj(Protocol): def __call__(self, data: str) -> object: ... @@ -1353,6 +1358,8 @@ async def common_checks( request_body=request_body, team_object=team_object, valid_token=valid_token, + user_object=user_object, + deny_by_default=_vector_store_deny_by_default(_typed_request_body(general_settings)), ) # 12. [OPTIONAL] Tool allowlist - key/team allowed_tools (no DB in hot path) @@ -5481,10 +5488,10 @@ def _search_tool_names_from_object_permission( def _can_object_call_search_tool( search_tool_name: str, allowed_search_tools: list[str], - object_type: Literal["key", "team", "project"], + object_type: Literal["key", "team", "project", "user"], ) -> Literal[True]: """ - Check if an object (key/team/project) can access a specific search tool. + Check if an object (key/team/project/user) can access a specific search tool. Similar to _can_object_call_model but for search tools. @@ -5517,84 +5524,188 @@ def _can_object_call_search_tool( ) -async def can_key_call_search_tool( - search_tool_name: str, - valid_token: UserAPIKeyAuth, -) -> Literal[True]: +TeamObjectLoader: TypeAlias = Callable[[], Awaitable[LiteLLM_TeamTable | None]] + + +@dataclass(frozen=True) +class SearchToolGrants: """ - Check if a key can access a specific search tool. - - Similar to can_key_call_model but for search tools. - - Args: - search_tool_name: The search tool being requested - valid_token: The authenticated key - - Returns: - True if access is allowed - - Raises: - ProxyException if access is denied + The key, team and user grants that scope one caller's search tools. Every layer in `strict_layers` + must list the tool, and any other loaded grant only narrows when its list is nonempty """ - return _can_object_call_search_tool( - search_tool_name=search_tool_name, - allowed_search_tools=_search_tool_names_from_object_permission(valid_token.object_permission), - object_type="key", - ) + + grants: Mapping[GrantLayer, LiteLLM_ObjectPermissionTable | None] + strict_layers: frozenset[GrantLayer] -async def can_team_call_search_tool( - search_tool_name: str, - team_object: LiteLLM_TeamTable | None, -) -> Literal[True]: +def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool: """ - Check if a team can access a specific search tool. - - Similar to can_team_access_model but for search tools. - - Args: - search_tool_name: The search tool being requested - team_object: The team object - - Returns: - True if access is allowed - - Raises: - ProxyException if access is denied - """ - if team_object is None: - return True - - return _can_object_call_search_tool( - search_tool_name=search_tool_name, - allowed_search_tools=_search_tool_names_from_object_permission(team_object.object_permission), - object_type="team", - ) - - -async def can_user_view_search_tool( - search_tool_name: str, - valid_token: UserAPIKeyAuth, - team_object: LiteLLM_TeamTable | None, -) -> bool: - """ - Boolean variant of the key + team authorization enforced on /search, used to - scope /search_tools/list so a non-admin caller only sees tools it may invoke. + Startup rejects a non-boolean value from the config file. A non-boolean value that reaches + general_settings another way enables the policy, so only search tool requests are denied. """ try: - await can_key_call_search_tool( - search_tool_name=search_tool_name, - valid_token=valid_token, + return ConfigGeneralSettings.model_validate( + MappingProxyType( + {"search_tool_deny_by_default": general_settings.get("search_tool_deny_by_default", False)} + ) + ).search_tool_deny_by_default + except ValidationError: + return True + + +def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool: + return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token) + + +def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException: + return ProxyException( + message=message, + type=ProxyErrorTypes.get_search_tool_access_error_type_for_object(object_type), + param="search_tool_name", + code=status.HTTP_403_FORBIDDEN, + ) + + +async def _strict_team_object(load_team_object: TeamObjectLoader) -> LiteLLM_TeamTable | None: + try: + return await load_team_object() + except Exception as e: # noqa: BLE001 # an unresolved team grants nothing under deny-by-default + verbose_proxy_logger.debug("Team lookup failed under search_tool_deny_by_default: %s", e) + return None + + +async def _strict_user_object( + valid_token: UserAPIKeyAuth, prisma_client: PrismaClient, user_api_key_cache: UserApiKeyCache +) -> LiteLLM_UserTable | None: + if valid_token.user_id is None: + return None + try: + return await get_user_object( + user_id=valid_token.user_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + user_id_upsert=False, + parent_otel_span=valid_token.parent_otel_span, ) - await can_team_call_search_tool( - search_tool_name=search_tool_name, - team_object=team_object, + except Exception as e: # noqa: BLE001 # an unresolved user grants nothing under deny-by-default + verbose_proxy_logger.debug("User lookup failed under search_tool_deny_by_default: %s", e) + return None + + +async def resolve_search_tool_grants( + valid_token: UserAPIKeyAuth, + general_settings: Mapping[str, object], + load_team_object: TeamObjectLoader, +) -> SearchToolGrants: + """ + Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions, + the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names + must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing + """ + if not is_search_tool_deny_by_default_applied(valid_token, general_settings): + team_object: Final = await load_team_object() + return SearchToolGrants( + grants=MappingProxyType( + {"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None} + ), + strict_layers=frozenset(), ) + + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache + + strict_team_object: Final = await _strict_team_object(load_team_object) + strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object) + if prisma_client is None: + return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers) + user_object: Final = ( + await _strict_user_object(valid_token, prisma_client, user_api_key_cache) if "user" in strict_layers else None + ) + return SearchToolGrants( + grants=MappingProxyType( + dict( + await _identity_grants(valid_token, strict_team_object, user_object, prisma_client, user_api_key_cache) + ) + ), + strict_layers=strict_layers, + ) + + +def check_search_tool_grants(search_tool_name: str, grants: SearchToolGrants) -> Literal[True]: + """Raises a 403 ProxyException naming the first key, team or user layer that does not grant the tool""" + for layer in ("key", "team", "user"): + grant = grants.grants.get(layer) + if layer in grants.strict_layers: + if grant is None or search_tool_name not in (grant.search_tools or ()): + raise _search_tool_denied( + layer, + f"{layer.capitalize()} not allowed to access search tool: {search_tool_name}. " + f"search_tool_deny_by_default is enabled and the {layer} does not grant it", + ) + elif grant is not None: + _can_object_call_search_tool( + search_tool_name=search_tool_name, + allowed_search_tools=_search_tool_names_from_object_permission(grant), + object_type=layer, + ) + return True + + +async def can_caller_call_search_tool( + search_tool_name: str, + valid_token: UserAPIKeyAuth, + general_settings: Mapping[str, object], + load_team_object: TeamObjectLoader, +) -> Literal[True]: + """Key, team and user search tool authorization shared by /search, web search interception and discovery""" + return check_search_tool_grants( + search_tool_name, await resolve_search_tool_grants(valid_token, general_settings, load_team_object) + ) + + +async def can_token_call_search_tool(search_tool_name: str, valid_token: UserAPIKeyAuth) -> Literal[True]: + """`can_caller_call_search_tool` against the proxy's own settings, team cache and database""" + from litellm.proxy.proxy_server import general_settings, prisma_client, proxy_logging_obj, user_api_key_cache + + async def _load_team_object() -> LiteLLM_TeamTable | None: + if not valid_token.team_id: + return None + return await get_team_object( + team_id=valid_token.team_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + parent_otel_span=valid_token.parent_otel_span, + proxy_logging_obj=proxy_logging_obj, + ) + + return await can_caller_call_search_tool( + search_tool_name=search_tool_name, + valid_token=valid_token, + general_settings=typed_general_settings(general_settings), + load_team_object=_load_team_object, + ) + + +def can_grants_view_search_tool(search_tool_name: str, grants: SearchToolGrants) -> bool: + try: + check_search_tool_grants(search_tool_name, grants) except ProxyException: return False return True +def check_unregistered_search_fallback( + valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object] +) -> Literal[True]: + """The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it""" + if not is_search_tool_deny_by_default_applied(valid_token, general_settings): + return True + layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index) + raise _search_tool_denied( + layer, + "No registered search tool is available and search_tool_deny_by_default is enabled", + ) + + async def is_valid_fallback_model( model: str, llm_router: Router | None, @@ -6792,17 +6903,207 @@ def _get_rag_query_vector_store_id(request_body: Mapping[str, object]) -> str | return vector_store_id if isinstance(vector_store_id, str) and vector_store_id else None +def _is_strict_grant_identity(valid_token: UserAPIKeyAuth | None) -> bool: + return ( + valid_token is not None + and valid_token.api_key != LITELLM_PROXY_MASTER_KEY_ALIAS + and valid_token.team_id != UI_TEAM_ID + ) + + +def _strict_grant_layers( + deny_by_default: bool, valid_token: UserAPIKeyAuth | None, team_object: LiteLLM_TeamTable | None +) -> frozenset[GrantLayer]: + """ + The identities that must each grant an object under a deny-by-default policy: a virtual key and its team, + a keyless team member's team, or a keyless user's own grant. The master key and dashboard sessions need none + """ + if not deny_by_default or valid_token is None or not _is_strict_grant_identity(valid_token): + return frozenset() + virtual_key: Final = valid_token.via_virtual_key and not valid_token.is_session_token + has_team: Final = team_object is not None or valid_token.team_id is not None + if not virtual_key and not has_team: + return frozenset(("user",)) + return frozenset(layer for layer, required in (("key", virtual_key), ("team", has_team)) if required) + + +async def _identity_grants( + valid_token: UserAPIKeyAuth | None, + team_object: LiteLLM_TeamTable | None, + user_object: LiteLLM_UserTable | None, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> tuple[tuple[GrantLayer, LiteLLM_ObjectPermissionTable | None], ...]: + return ( + ( + "key", + None + if valid_token is None + else await _cached_object_permission( + valid_token.object_permission_id, valid_token.object_permission, prisma_client, user_api_key_cache + ), + ), + ( + "team", + None + if team_object is None + else await _cached_object_permission( + team_object.object_permission_id, team_object.object_permission, prisma_client, user_api_key_cache + ), + ), + ( + "user", + None + if user_object is None + else await _cached_object_permission( + user_object.object_permission_id, user_object.object_permission, prisma_client, user_api_key_cache + ), + ), + ) + + +def _vector_store_deny_by_default(general_settings: Mapping[str, object]) -> bool: + """ + Startup rejects a non-boolean value from the config file. A non-boolean value that reaches + general_settings another way enables the policy, so only vector store requests are denied. + """ + try: + return ConfigGeneralSettings.model_validate( + MappingProxyType( + {"vector_store_deny_by_default": general_settings.get("vector_store_deny_by_default", False)} + ) + ).vector_store_deny_by_default + except ValidationError: + return True + + +_VECTOR_STORE_IDS_ADAPTER: Final[TypeAdapter[Sequence[str]]] = TypeAdapter(Sequence[str]) +_TOOLS_ADAPTER: Final[TypeAdapter[Sequence[object]]] = TypeAdapter(Sequence[object]) +_TOOL_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) + + +def _validated_vector_store_ids(value: object) -> tuple[str, ...]: + if value is None: + return () + try: + return tuple(_VECTOR_STORE_IDS_ADAPTER.validate_python(value, strict=True)) + except ValidationError: + raise _malformed_vector_store_ids() from None + + +def _malformed_vector_store_ids() -> ProxyException: + return ProxyException( + message="vector_store_ids must be a list of strings", + type="invalid_request_error", + param="vector_store_ids", + code=status.HTTP_400_BAD_REQUEST, + ) + + +def _tools(tools: object) -> tuple[object, ...]: + try: + return tuple(_TOOLS_ADAPTER.validate_python(tools, strict=True)) + except ValidationError: + return () + + +def _tool_vector_store_ids(tool: object) -> tuple[str, ...]: + try: + tool_fields: Final = _TOOL_ADAPTER.validate_python(tool, strict=True) + except ValidationError: + return () + return _validated_vector_store_ids(tool_fields.get("vector_store_ids")) + + +def _strict_requested_vector_store_ids(request_body: Mapping[str, object]) -> tuple[str, ...]: + """ + Same fields VectorStoreRegistry.get_vector_store_ids_to_run reads, but a vector_store_ids that is + not a list of strings is a 400 instead of being skipped or iterated, and tools that are not + objects name no store. + """ + return tuple( + chain( + _validated_vector_store_ids(request_body.get("vector_store_ids")), + chain.from_iterable(_tool_vector_store_ids(tool) for tool in _tools(request_body.get("tools"))), + ) + ) + + +def _require_vector_store_grant( + object_type: GrantLayer, + vector_store_ids_to_run: Sequence[str], + object_permission: _VectorStorePermissionsRow | None, +) -> None: + if object_permission is None or not object_permission.vector_stores: + raise ProxyException( + message=f"{object_type.capitalize()} not allowed to access vector store. Tried to access {vector_store_ids_to_run[0]}. vector_store_deny_by_default is enabled and the {object_type} has no vector store grants", + type=ProxyErrorTypes.get_vector_store_access_error_type_for_object(object_type), + param="vector_store", + code=status.HTTP_401_UNAUTHORIZED, + ) + _can_object_call_vector_stores( + object_type=object_type, + vector_store_ids_to_run=vector_store_ids_to_run, + object_permissions=object_permission, + ) + + +async def _cached_object_permission( + object_permission_id: str | None, + loaded: LiteLLM_ObjectPermissionTable | None, + prisma_client: PrismaClient, + user_api_key_cache: UserApiKeyCache, +) -> LiteLLM_ObjectPermissionTable | None: + """ + The grant row auth already attached to the key, team or user, else the cached row by id. Both are + evicted on every worker when the grant changes, so the request path never reads the table directly. + """ + if object_permission_id is None: + return None + if loaded is not None: + return loaded + return await get_object_permission( + object_permission_id=object_permission_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + + async def vector_store_access_check( request_body: dict, team_object: LiteLLM_TeamTable | None, valid_token: UserAPIKeyAuth | None, + *, + user_object: LiteLLM_UserTable | None = None, + deny_by_default: bool = False, ): """ - Checks if the object (key, team, org) has access to the vector store. + Checks whether the caller may use every vector store the request names. - Raises ProxyException if the object (key, team, org) cannot access the specific vector store. + Requested stores come from `vector_store_ids`, `tools[].vector_store_ids` and the RAG + `retrieval_config.vector_store_id`. Grants come from each identity's + `object_permission.vector_stores`. + + With `deny_by_default=False` (legacy), a key or team only restricts access when its list is + nonempty, and stores are only read from the request when a vector store registry is loaded. + + With `deny_by_default=True` (`general_settings.vector_store_deny_by_default`), stores are read + even without a registry, and a missing record, `null` or `[]` grants nothing: + + - virtual key: the key must grant every store, and so must its team when it has one, even if + the team failed to load + - keyless team member (JWT, `lite login` session token): only the resolved team is checked + - keyless user with no team: the user's own grant is checked + - master key and dashboard sessions: legacy behavior + + The user's personal grant is only consulted in the keyless no-team case, so it can neither + rescue nor restrict a key or team request. + + Raises ProxyException (401, `{key,team,user}_vector_store_access_denied`) on the first identity + that does not grant a requested store, and with the flag on, ProxyException (400, + `invalid_request_error`) when a `vector_store_ids` field is not a list of strings. """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, user_api_key_cache ######################################################### # Get the vector store the user is trying to access @@ -6812,12 +7113,17 @@ async def vector_store_access_check( return True registry_ids: Final = ( - litellm.vector_store_registry.get_vector_store_ids_to_run( - non_default_params=request_body, tools=request_body.get("tools", None) + _strict_requested_vector_store_ids(_typed_request_body(request_body)) + if deny_by_default + else ( + litellm.vector_store_registry.get_vector_store_ids_to_run( + non_default_params=request_body, tools=request_body.get("tools", None) + ) + if litellm.vector_store_registry is not None + else None ) - if litellm.vector_store_registry is not None - else None - ) or () + or () + ) rag_vector_store_id: Final = _get_rag_query_vector_store_id(_typed_request_body(request_body)) rag_ids: Final = (rag_vector_store_id,) if rag_vector_store_id is not None else () vector_store_ids_to_run: Final = tuple(dict.fromkeys((*registry_ids, *rag_ids))) @@ -6828,38 +7134,28 @@ async def vector_store_access_check( ######################################################### # Check if the object (key, team, org) has access to the vector store ######################################################### - # Check if the key can access the vector store - if valid_token is not None and valid_token.object_permission_id is not None: - key_object_permission: Final = await _object_permission_table( - ObjectPermissionRepository(prisma_client) - ).find_unique( - where={"object_permission_id": valid_token.object_permission_id}, - ) - if key_object_permission is not None: + strict_layers: Final = _strict_grant_layers(deny_by_default, valid_token, team_object) + grants: Final = await _identity_grants( + valid_token, + team_object, + user_object if "user" in strict_layers else None, + prisma_client, + user_api_key_cache, + ) + for layer, grant in grants: + if layer in strict_layers: + _require_vector_store_grant(layer, vector_store_ids_to_run, grant) + elif grant is not None: _can_object_call_vector_stores( - object_type="key", + object_type=layer, vector_store_ids_to_run=vector_store_ids_to_run, - object_permissions=key_object_permission, - ) - - # Check if the team can access the vector store - if team_object is not None and team_object.object_permission_id is not None: - team_object_permission: Final = await _object_permission_table( - ObjectPermissionRepository(prisma_client) - ).find_unique( - where={"object_permission_id": team_object.object_permission_id}, - ) - if team_object_permission is not None: - _can_object_call_vector_stores( - object_type="team", - vector_store_ids_to_run=vector_store_ids_to_run, - object_permissions=team_object_permission, + object_permissions=grant, ) return True def _can_object_call_vector_stores( - object_type: Literal["key", "team", "org"], + object_type: Literal["key", "team", "org", "user"], vector_store_ids_to_run: Sequence[str], object_permissions: _VectorStorePermissionsRow | None, ): diff --git a/litellm/proxy/auth/fallback_model_access.py b/litellm/proxy/auth/fallback_model_access.py index 53807f076ea..39346c80b22 100644 --- a/litellm/proxy/auth/fallback_model_access.py +++ b/litellm/proxy/auth/fallback_model_access.py @@ -5,7 +5,8 @@ Authorize router fallback targets against the caller's key, team and project mod Fallbacks configured on the router (`router_settings.fallbacks` and friends) are chosen after auth, inside the router, so this predicate is injected into the router to re-run the same model access checks for each fallback target before it is attempted. Opt-in via -`general_settings.enforce_fallback_model_access: true`. +`general_settings.enforce_fallback_model_access: true`. Fallbacks of a search request always run the +key, team and user search tool grants instead, the same check /search runs on the requested tool. """ from collections.abc import Callable, Mapping @@ -16,8 +17,9 @@ from pydantic import ValidationError from litellm._logging import verbose_proxy_logger from litellm.proxy._types import ProxyException, UserAPIKeyAuth -from litellm.proxy.auth.auth_checks import can_key_call_resolved_model +from litellm.proxy.auth.auth_checks import can_key_call_resolved_model, can_token_call_search_tool from litellm.router import Router +from litellm.search import asearch from litellm.types.llms.base import LiteLLMBaseModel @@ -45,6 +47,23 @@ async def is_model_authorized_for_token(*, model: str, valid_token: UserAPIKeyAu return True +async def is_search_tool_authorized_for_token(*, search_tool_name: str, valid_token: UserAPIKeyAuth) -> bool: + try: + await can_token_call_search_tool(search_tool_name=search_tool_name, valid_token=valid_token) + except ProxyException: + return False + except Exception as e: # noqa: BLE001 # fail closed: a lookup failure must neither run the fallback nor replace the provider error + verbose_proxy_logger.warning( + "Skipping fallback to search tool=%s: authorization lookup failed: %s", search_tool_name, e + ) + return False + return True + + +def _is_search_request(request_kwargs: Mapping[str, object]) -> bool: + return request_kwargs.get("original_generic_function") is asearch + + def _token_in_metadata(metadata: object) -> UserAPIKeyAuth | None: try: return _RequestMetadata.model_validate(metadata).user_api_key_auth @@ -72,19 +91,21 @@ def _enforced_by_general_settings() -> bool: @dataclass(frozen=True, slots=True) class RouterFallbackAccessCheck: """ - `FallbackAccessCheck` for the proxy's router: while `is_enforced()` is true, a fallback target - is attempted only when the key behind the request could have requested it directly. Requests - that carry no key (for example internal health checks) are not restricted. + `FallbackAccessCheck` for the proxy's router: a fallback of a search request, and while `is_enforced()` + is true any other fallback, is attempted only when the key behind the request could have requested it + directly. Requests that carry no key (for example internal health checks) are not restricted. """ is_enforced: Callable[[], bool] async def __call__(self, *, model: str, request_kwargs: Mapping[str, object], llm_router: Router) -> bool: - if not self.is_enforced(): - return True valid_token: Final = _user_api_key_auth_from_request(request_kwargs) if valid_token is None: return True + if _is_search_request(request_kwargs): + return await is_search_tool_authorized_for_token(search_tool_name=model, valid_token=valid_token) + if not self.is_enforced(): + return True return await is_model_authorized_for_token(model=model, valid_token=valid_token, llm_router=llm_router) diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index adb6b86321d..65f9b09a4d3 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -3063,6 +3063,9 @@ async def _run_centralized_common_checks( user_id=user_api_key_auth_obj.user_id or litellm_proxy_admin_name, user_role=LitellmUserRoles.PROXY_ADMIN, spend=user_object.spend if user_object is not None else 0.0, + object_permission_id=( + user_object.object_permission_id if isinstance(user_object, LiteLLM_UserTable) else None + ), ) if project_object is not None: diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 6a0fcbe0bb3..5b9fb2e3977 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -15,15 +15,20 @@ _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" _LANGFUSE_SPAN_SCOPE_VAR: Final = "langfuse_span_scope" _ARIZE_CALLBACK: Final = "arize" +_ARIZE_OTLP_PROTOCOL_VAR: Final = "arize_otlp_protocol" _ARIZE_SAMPLING_RATE_VARS: Final[frozenset[str]] = frozenset( {"arize_success_sampling_rate", "arize_error_sampling_rate"} ) -def callback_config_error(callback_name: str | None, callback_vars: Mapping[str, str] | None) -> str | None: +def callback_config_error( + callback_name: str | None, callback_vars: Mapping[str, str] | None, callback_type: str | None = None +) -> str | None: if not callback_vars: return None - arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) + arize_error: Final = _arize_sampling_rate_error(callback_name, callback_vars) or _arize_otlp_protocol_error( + callback_name, callback_vars, callback_type + ) if arize_error is not None: return arize_error langfuse_error: Final = _langfuse_environment_error(callback_vars) or _langfuse_span_scope_error( @@ -94,7 +99,9 @@ _VAR_FAMILIES: Final[Mapping[str, str]] = MappingProxyType( } ) -_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset({_LANGFUSE_SPAN_SCOPE_VAR, *_ARIZE_SAMPLING_RATE_VARS}) +_FAMILY_OPTION_VARS: Final[frozenset[str]] = frozenset( + {_LANGFUSE_SPAN_SCOPE_VAR, _ARIZE_OTLP_PROTOCOL_VAR, *_ARIZE_SAMPLING_RATE_VARS} +) def _family_of(var: str) -> str | None: @@ -217,7 +224,10 @@ def _logging_entry_error(entry: object) -> str | None: callback_name: Final = entry.get("callback_name") if not isinstance(callback_name, str) or not isinstance(entry.get("callback_vars"), Mapping): return None - return callback_config_error(callback_name, _entry_callback_vars(entry)) + callback_type_raw: Final = entry.get("callback_type") + return callback_config_error( + callback_name, _entry_callback_vars(entry), callback_type_raw if isinstance(callback_type_raw, str) else None + ) def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping[str, str]) -> str | None: @@ -236,6 +246,30 @@ def _arize_sampling_rate_error(callback_name: str | None, callback_vars: Mapping return None +def _arize_otlp_protocol_error( + callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None = None +) -> str | None: + value: Final = callback_vars.get(_ARIZE_OTLP_PROTOCOL_VAR) + if value is None: + return None + if callback_name != _ARIZE_CALLBACK: + return f"{_ARIZE_OTLP_PROTOCOL_VAR} applies to the {_ARIZE_CALLBACK} callback only, not {callback_name!r}" + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_arize_otlp_protocol_value, + ) + + try: + validate_arize_otlp_protocol_value(value) + except ValueError as e: + return str(e) + if callback_type == "failure": + return f"{_ARIZE_OTLP_PROTOCOL_VAR} needs callback_type 'success' or 'success_and_failure'; failure-only Arize callbacks export over the proxy's own Arize transport" + if not is_otel_v2_enabled(): + return "Per-team Arize transport selection requires the proxy to run with LITELLM_OTEL_V2=true." + return None + + def _newrelic_config_error(callback_vars: Mapping[str, str]) -> str | None: """Per-team New Relic routing runs on the OTel v2 path only. diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index 69efb9c209f..98df4ae46b1 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -1458,11 +1458,7 @@ async def _invalidate_cached_user_entitlement(user_id: str | None, object_permis *(object_permission_cache_key(permission_id) for permission_id in dict.fromkeys(object_permission_ids)), *((user_object_permission_id_cache_key(user_id), user_id) if user_id is not None else ()), ) - for key in keys: - try: - await user_api_key_cache.async_delete_cache(key=key) - except Exception as e: # noqa: BLE001 # a cache we cannot clear still expires; never fail the write - verbose_proxy_logger.warning("Failed to invalidate cached entitlement key %r: %s", key, e) + await evict_and_broadcast(cache_keys=keys, user_api_key_cache=user_api_key_cache) async def _update_single_user_helper( diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index a59342fa044..471a0814ae5 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -61,7 +61,7 @@ def _callback_config_error(message: str) -> HTTPException: def _validate_team_callback(data: "AddTeamCallback") -> None: - error: Final = callback_config_error(data.callback_name, data.callback_vars) + error: Final = callback_config_error(data.callback_name, data.callback_vars, data.callback_type) if error is not None: raise _callback_config_error(error) diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index 818c0590707..aeb317c8905 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -165,6 +165,7 @@ from litellm.proxy.management_helpers.object_permission_utils import ( _set_object_permission, enforce_all_proxy_mcp_servers_grant_is_admin_only, handle_update_object_permission_common, + invalidate_cached_object_permissions, ) from litellm.proxy.management_helpers.team_member_permission_checks import ( TeamMemberPermissionChecks, @@ -2546,6 +2547,10 @@ async def update_team( verbose_proxy_logger.info("Successfully updated team - %s, info", team_row.team_id) await sync_team_access_group_membership(prisma_client=prisma_client, team_id=team_row.team_id) + await invalidate_cached_object_permissions( + object_permission_ids=(existing_team.object_permission_id, team_row.object_permission_id), + user_api_key_cache=user_api_key_cache, + ) await _refresh_cached_team( team_row=team_row, user_api_key_cache=user_api_key_cache, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index d1147c48558..5d6ba0f8e13 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6713,6 +6713,17 @@ class ProxyConfig: if general_settings is None: general_settings = {} + typed_general_settings: Final = _GENERAL_SETTINGS_VIEW.validate_python(general_settings) + ConfigGeneralSettings.model_validate( + MappingProxyType( + { + name: typed_general_settings[name] + for name in ("vector_store_deny_by_default", "search_tool_deny_by_default") + if name in typed_general_settings + } + ) + ) + if general_settings.get("mcp_advertised_versions") is not None: from litellm.types.mcp import MCPAdvertisedVersions diff --git a/litellm/proxy/search_endpoints/endpoints.py b/litellm/proxy/search_endpoints/endpoints.py index 349eb5b7861..1e37e3243ea 100644 --- a/litellm/proxy/search_endpoints/endpoints.py +++ b/litellm/proxy/search_endpoints/endpoints.py @@ -135,52 +135,24 @@ async def search( if search_tool_name is not None: data["search_tool_name"] = search_tool_name - if not ( - data.get("search_tool_name") or data.get("model") or general_settings.get("completion_model") or user_model - ): + from litellm.proxy.auth.auth_checks import can_token_call_search_tool + from litellm.proxy.common_utils.http_parsing_utils import resolve_inference_model + + routed_search_tool_name: Final = data.get("search_tool_name") or resolve_inference_model( + data.get("model"), general_settings, user_model + ) + if not isinstance(routed_search_tool_name, str) or not routed_search_tool_name: raise ProxyMissingRequiredParamError(route="/search", param="search_tool_name") + try: + await can_token_call_search_tool(search_tool_name=routed_search_tool_name, valid_token=user_api_key_dict) + except ProxyException as e: + verbose_proxy_logger.debug("Search tool authorization denied: %s", e.type) + raise if "search_tool_name" in data and data["search_tool_name"]: data["model"] = data["search_tool_name"] search_tool_name_value: Final = data["search_tool_name"] - # Authorization check: verify key can access this search tool - from litellm.proxy.auth.auth_checks import ( - can_key_call_search_tool, - can_team_call_search_tool, - get_team_object, - ) - - try: - # Check key-level access - await can_key_call_search_tool( - search_tool_name=search_tool_name_value, - valid_token=user_api_key_dict, - ) - - # Check team-level access if key is associated with a team - if user_api_key_dict.team_id: - from litellm.proxy.proxy_server import ( - prisma_client, - proxy_logging_obj, - user_api_key_cache, - ) - - team_object: Final = await get_team_object( - team_id=user_api_key_dict.team_id, - prisma_client=prisma_client, - user_api_key_cache=user_api_key_cache, - parent_otel_span=user_api_key_dict.parent_otel_span, - proxy_logging_obj=proxy_logging_obj, - ) - await can_team_call_search_tool( - search_tool_name=search_tool_name_value, - team_object=team_object, - ) - except Exception as e: - verbose_proxy_logger.error("Search tool authorization failed for %s: %s", search_tool_name_value, e) - raise - if llm_router is not None and hasattr(llm_router, "search_tools"): verbose_proxy_logger.debug( "Search endpoint - Looking for search_tool_name: %s. Available search tools in router: %s. Total search tools: %s", diff --git a/litellm/proxy/search_endpoints/search_tool_management.py b/litellm/proxy/search_endpoints/search_tool_management.py index 49b868ac71a..1e6ed40bba3 100644 --- a/litellm/proxy/search_endpoints/search_tool_management.py +++ b/litellm/proxy/search_endpoints/search_tool_management.py @@ -2,7 +2,7 @@ CRUD ENDPOINTS FOR SEARCH TOOLS """ -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import Awaitable, Callable, Mapping, Sequence from datetime import datetime from typing import Any, Final, TypeAlias @@ -117,34 +117,39 @@ async def _filter_visible_search_tools( search_tools: list[SearchToolInfoResponse], user_api_key_dict: UserAPIKeyAuth, lookup_team_object: TeamObjectLookup = _team_object_from_db, + general_settings: Mapping[str, object] | None = None, ) -> list[SearchToolInfoResponse]: """ Drop search tools the caller is not authorized to invoke, applying the same - key/team object_permission allowlists enforced on /search. Admins see all tools. + key/team/user grants enforced on /search. Admins see all tools unless + search_tool_deny_by_default applies to their credential. """ + from litellm.proxy.auth.auth_checks import ( + can_grants_view_search_tool, + is_search_tool_deny_by_default_applied, + resolve_search_tool_grants, + typed_general_settings, + ) + from litellm.proxy.proxy_server import general_settings as proxy_general_settings + + settings: Final = typed_general_settings(proxy_general_settings) if general_settings is None else general_settings if user_api_key_dict.user_role in ( LitellmUserRoles.PROXY_ADMIN, LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ): + ) and not is_search_tool_deny_by_default_applied(user_api_key_dict, settings): return search_tools - from litellm.proxy.auth.auth_checks import can_user_view_search_tool - allowlist_team_id: Final = _allowlist_team_id(user_api_key_dict) - team_object: Final[LiteLLM_TeamTable | None] = ( - await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None - ) - visible: Final[list[SearchToolInfoResponse]] = [] - for tool in search_tools: - tool_name = tool.get("search_tool_name") - if tool_name and await can_user_view_search_tool( - search_tool_name=tool_name, - valid_token=user_api_key_dict, - team_object=team_object, - ): - visible.append(tool) - return visible + async def _load_team_object() -> LiteLLM_TeamTable | None: + return await lookup_team_object(allowlist_team_id, user_api_key_dict) if allowlist_team_id else None + + grants: Final = await resolve_search_tool_grants(user_api_key_dict, settings, _load_team_object) + return [ + tool + for tool in search_tools + if (tool_name := tool.get("search_tool_name")) and can_grants_view_search_tool(tool_name, grants) + ] @router.get( diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index 4154452af05..eb7a6556386 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -691,8 +691,10 @@ class BaseResponsesAPIStreamingIterator: if getattr(completed_response, "type", None) != openai_types.ResponsesAPIStreamEvents.RESPONSE_COMPLETED: return + from litellm.caching.caching_handler import create_cache_write_task, is_response_without_output + response_obj: Final = self._get_completed_response_object() - if response_obj is None: + if response_obj is None or is_response_without_output(response_obj): return caching_handler: Final[LLMCachingHandler | None] = getattr(self.logging_obj, "_llm_caching_handler", None) @@ -732,8 +734,6 @@ class BaseResponsesAPIStreamingIterator: if cached_response is None: return if is_async: - from litellm.caching.caching_handler import create_cache_write_task - cache_write_task: Final = create_cache_write_task( lambda: cache.async_add_cache( cached_response, diff --git a/litellm/rust_bridge/callbacks_legacy_python.py b/litellm/rust_bridge/callbacks_legacy_python.py index e011b795000..46af4e236eb 100644 --- a/litellm/rust_bridge/callbacks_legacy_python.py +++ b/litellm/rust_bridge/callbacks_legacy_python.py @@ -21,6 +21,8 @@ from typing import ( cast, # noqa: TID251 # bounded compatibility calls into legacy Python integrations ) +from pydantic import TypeAdapter + if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging @@ -337,9 +339,15 @@ def after_deployment_failure(kwargs: dict[str, object], error: Exception, call_t return hook(kwargs, error, call_type) -def stream_opened(logger: Logging) -> None: +_STREAM_HEADERS: Final = TypeAdapter(Mapping[str, str]) + + +def stream_opened(logger: Logging, head: Mapping[str, object]) -> None: logger.stream = True logger.model_call_details["stream"] = True + logger.model_call_details["response_headers"] = _STREAM_HEADERS.validate_python( + head.get("additional_headers") or {} + ) def stream_success( diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 127e86e9160..4a09e731228 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -59,6 +59,7 @@ class AgentNode(typing_extensions.TypedDict): tool_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] duration_ms: ReadOnly[float] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] SpanType: TypeAlias = Literal[ @@ -77,6 +78,9 @@ SpanType: TypeAlias = Literal[ ] +SpendMatch: TypeAlias = Literal["matched", "no_call_id", "no_spend_log", "ambiguous", "incomplete_evidence"] + + class TraceScope(typing_extensions.TypedDict): all_teams: ReadOnly[Literal[0, 1]] user_id: ReadOnly[str] @@ -120,6 +124,7 @@ class TraceSummary(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] models: ReadOnly[tuple[str, ...]] spend: ReadOnly[float | None] + priced_calls: ReadOnly[Annotated[int, Field(ge=0, le=18446744073709551615)]] class Span(typing_extensions.TypedDict): @@ -140,6 +145,8 @@ class Span(typing_extensions.TypedDict): output_tokens: ReadOnly[Annotated[int, Field(ge=0, le=4294967295)]] litellm_request_id: ReadOnly[str | None] spend: ReadOnly[float | None] + spend_log_request_id: ReadOnly[str | None] + spend_match: ReadOnly[SpendMatch | None | None] class Trace(typing_extensions.TypedDict): diff --git a/litellm/tracing/types.py b/litellm/tracing/types.py index e9930dc7bf7..8bd119940c2 100644 --- a/litellm/tracing/types.py +++ b/litellm/tracing/types.py @@ -8,6 +8,7 @@ class SpendLogRecord(TypedDict): request_id: ReadOnly[str] response_id: ReadOnly[str] + provider_request_id: ReadOnly[str] litellm_call_id: ReadOnly[str] call_type: ReadOnly[str] api_key: ReadOnly[str] diff --git a/litellm/types/caching.py b/litellm/types/caching.py index 036f7c452ea..b9bcbbb93dd 100644 --- a/litellm/types/caching.py +++ b/litellm/types/caching.py @@ -140,6 +140,8 @@ class HealthCheckCacheParams(LiteLLMBaseModel): EMBEDDING_CACHE_FORMAT_VERSION: Final = 2 +CACHED_STREAM_EVENTS_KEY: Final = "litellm_cached_anthropic_sse_events" + class CachedEmbedding(TypedDict): """Type definition for cached embedding objects""" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d3fe65037e2..bd7ccc5944f 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3648,6 +3648,9 @@ OPENAI_RESPONSE_HEADERS: Final = [ OtelSpanScope = Literal["full", "llm_only"] OTEL_SPAN_SCOPES: Final[frozenset[str]] = frozenset(get_args(OtelSpanScope)) +ArizeOtlpProtocol = Literal["grpc", "http/protobuf"] +ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol)) + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params @@ -3681,6 +3684,7 @@ class StandardCallbackDynamicParams(TypedDict, total=False): arize_space_id: str | None arize_success_sampling_rate: ReadOnly[float | None] arize_error_sampling_rate: ReadOnly[float | None] + arize_otlp_protocol: ReadOnly[ArizeOtlpProtocol | None] # PostHog dynamic params posthog_api_key: str | None diff --git a/scripts/trace_codegen/schemas/traces/Trace.json b/scripts/trace_codegen/schemas/traces/Trace.json index dc66f77412c..ebd99880956 100644 --- a/scripts/trace_codegen/schemas/traces/Trace.json +++ b/scripts/trace_codegen/schemas/traces/Trace.json @@ -28,6 +28,12 @@ "null" ] }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "spend": { "format": "double", "type": [ @@ -49,7 +55,8 @@ "llm_calls", "tool_calls", "duration_ms", - "spend" + "spend", + "priced_calls" ], "type": "object" }, @@ -120,6 +127,22 @@ "null" ] }, + "spend_log_request_id": { + "type": [ + "string", + "null" + ] + }, + "spend_match": { + "anyOf": [ + { + "$ref": "#/$defs/SpendMatch" + }, + { + "type": "null" + } + ] + }, "start_offset_ms": { "format": "double", "type": "number" @@ -148,7 +171,9 @@ "input_tokens", "output_tokens", "litellm_request_id", - "spend" + "spend", + "spend_log_request_id", + "spend_match" ], "type": "object" }, @@ -177,6 +202,16 @@ ], "type": "string" }, + "SpendMatch": { + "enum": [ + "matched", + "no_call_id", + "no_spend_log", + "ambiguous", + "incomplete_evidence" + ], + "type": "string" + }, "TraceSummary": { "properties": { "agent_count": { @@ -245,6 +280,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -306,7 +347,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/scripts/trace_codegen/schemas/traces/TracePage.json b/scripts/trace_codegen/schemas/traces/TracePage.json index dfa02ba0d39..429635ec3b1 100644 --- a/scripts/trace_codegen/schemas/traces/TracePage.json +++ b/scripts/trace_codegen/schemas/traces/TracePage.json @@ -76,6 +76,12 @@ "minimum": 0, "type": "integer" }, + "priced_calls": { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, "resolution_limited": { "type": "boolean", "x-python-optional": true @@ -137,7 +143,8 @@ "input_tokens", "output_tokens", "models", - "spend" + "spend", + "priced_calls" ], "type": "object" } diff --git a/tests/integration/_support/manifest.py b/tests/integration/_support/manifest.py index fb17ebb3864..47c02470f13 100644 --- a/tests/integration/_support/manifest.py +++ b/tests/integration/_support/manifest.py @@ -18,6 +18,7 @@ OWNED_DIRECTORIES: Final = frozenset( "compatibility", "sdk", "cost_calculation", + "caching", "security", } ) diff --git a/tests/integration/authorization/test_object_permission_lookup.py b/tests/integration/authorization/test_object_permission_lookup.py index 21fba30c165..a977243e642 100644 --- a/tests/integration/authorization/test_object_permission_lookup.py +++ b/tests/integration/authorization/test_object_permission_lookup.py @@ -57,18 +57,18 @@ def _assert_forbidden_vector_store_denied(gateway: Gateway, model: str, key: str @pytest.mark.covers("authorization.vector_store.plain_request_skips_object_permission_lookup") -def test_chat_request_without_vector_stores_does_not_read_object_permission_table(gateway: Gateway) -> None: +def test_requests_do_not_read_object_permission_table_once_the_grant_is_cached(gateway: Gateway) -> None: with gateway.scenario() as scenario: model: Final = scenario.model() key: Final = scenario.key(models=[model], object_permission={"vector_stores": ["vs_allowed"]}) - _assert_plain_chat_served(gateway, model, key) - before_control: Final = _object_permission_reads() + before_first_use: Final = _object_permission_reads() _assert_forbidden_vector_store_denied(gateway, model, key) - eventually(_object_permission_reads, lambda reads: reads > before_control, seconds=15) + eventually(_object_permission_reads, lambda reads: reads > before_first_use, seconds=15) baseline: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic()) for _ in range(PLAIN_REQUESTS): _assert_plain_chat_served(gateway, model, key) + _assert_forbidden_vector_store_denied(gateway, model, key) after: Final = _settled_object_permission_reads(_object_permission_reads(), time.monotonic()) assert after - baseline < PLAIN_REQUESTS, ( - f"{PLAIN_REQUESTS} plain chat requests added {after - baseline} object permission reads" + f"{2 * PLAIN_REQUESTS} requests after the first added {after - baseline} object permission reads" ) diff --git a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py index 896c88c68bb..d3f16e46a1e 100644 --- a/tests/integration/authorization/test_rag_query_vector_store_allowlist.py +++ b/tests/integration/authorization/test_rag_query_vector_store_allowlist.py @@ -1,5 +1,8 @@ from __future__ import annotations +import json +import os +import time import uuid from collections.abc import Iterator, Mapping from pathlib import Path @@ -7,16 +10,22 @@ from types import MappingProxyType from typing import Final, Literal, TypeAlias import httpx +import jwt import pytest import yaml -from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value +from cryptography.hazmat.primitives.asymmetric import rsa +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server from integration.authorization._guardrail_opt_out import upstream_observations from pydantic import JsonValue +from redis import Redis CONFIG_STORE_ID: Final = "vs_integration_config_store" PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" REMOVE_OPENAI_API_BASE: Final = ("OPENAI_API_BASE",) +JWT_KEY_ID: Final = "integration-vector-store-jwt-key" +AUTH_CACHE_INVALIDATION_CHANNEL: Final = "litellm_proxy.auth_cache_invalidation" JsonObject: TypeAlias = dict[str, JsonValue] @@ -99,6 +108,211 @@ def no_registry_gateways(tmp_path_factory: pytest.TempPathFactory) -> Iterator[t yield no_registry_gateway, upstream_gateway +class StrictGateway: + def __init__( + self, + gateway: Gateway, + upstream: Gateway, + signing_key: rsa.RSAPrivateKey, + config: Path, + environment: Mapping[str, str], + ) -> None: + self.gateway: Final = gateway + self.upstream: Final = upstream + self._signing_key: Final = signing_key + self.config: Final = config + self.environment: Final = environment + + def jwt(self, subject: str, groups: tuple[str, ...] = ()) -> str: + claims: Final[JsonObject] = { + "sub": subject, + "groups": _json_array(*groups), + "iat": int(time.time()), + "exp": int(time.time()) + 300, + } + return jwt.encode(claims, self._signing_key, algorithm="RS256", headers={"kid": JWT_KEY_ID}) + + +def _strict_config(directory: Path, *, deny_by_default: bool = True) -> Path: + config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text())) + general_settings: Final = object_value(config["general_settings"]) + strict: Final[JsonObject] = { + **config, + "general_settings": { + **general_settings, + "vector_store_deny_by_default": deny_by_default, + "enable_jwt_auth": True, + "litellm_jwtauth": { + "user_id_jwt_field": "sub", + "team_ids_jwt_field": "groups", + "team_allowed_routes": _json_array("openai_routes", "info_routes", "/v1/rag/query"), + }, + }, + } + path: Final = directory / f"proxy_vector_store_deny_by_default_{deny_by_default}.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +@pytest.fixture(scope="module") +def strict_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[StrictGateway]: + signing_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + public_jwk: Final = jwt.algorithms.RSAAlgorithm.to_jwk(signing_key.public_key()) + jwks_body: Final = json.dumps({"keys": [{**json.loads(public_jwk), "kid": JWT_KEY_ID}]}).encode() + + def respond(request: Request) -> Reply: + assert request.method == "GET", request + return Reply(body=jwks_body) + + with gateway_from_environment() as upstream_gateway, wire_server(respond) as jwks: + directory: Final = tmp_path_factory.mktemp("rag_query_deny_by_default") + config: Final = _strict_config(directory) + environment: Final = MappingProxyType({**_openai_environment(upstream_gateway), "JWT_PUBLIC_KEY_URL": jwks.url}) + with owned_proxy( + upstream_gateway, + directory, + environment, + config=config, + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + yield StrictGateway(gateway, upstream_gateway, signing_key, config, environment) + + +StrictCase: TypeAlias = Literal[ + "standalone_key_no_permission", + "team_key_empty_key_grants", + "team_key_empty_team_grants", + "multi_store_one_ungranted", + "jwt_user_without_grant", +] + + +def _strict_denied_request( + strict: StrictGateway, scenario: Scenario, case: StrictCase, model: str, marker: str, store_id: str +) -> tuple[httpx.Response, str]: + models: Final = _json_array(model) + granted: Final = _permission_for_stores(store_id) + empty: Final = _permission_for_stores() + if case == "standalone_key_no_permission": + standalone_key: Final = scenario.key(models=models) + return strict.gateway.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=standalone_key + ), ("key_vector_store_access_denied") + if case in ("team_key_empty_key_grants", "team_key_empty_team_grants"): + key_grants_store: Final = case == "team_key_empty_team_grants" + team: Final = scenario.team(models=models, object_permission=empty if key_grants_store else granted) + team_key: Final = scenario.key( + team_id=team, models=models, object_permission=granted if key_grants_store else empty + ) + error_type: Final = "team_vector_store_access_denied" if key_grants_store else "key_vector_store_access_denied" + return strict.gateway.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=team_key + ), (error_type) + if case == "multi_store_one_ungranted": + partial_key: Final = scenario.key(models=models, object_permission=_permission_for_stores(store_id)) + body: Final[JsonObject] = { + **_rag_query_body(model, marker, store_id), + "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(CONFIG_STORE_ID)}), + } + return strict.gateway.request( + "POST", "/v1/chat/completions", body, key=partial_key + ), "key_vector_store_access_denied" + user: Final = scenario.user(user_role="internal_user") + return ( + strict.gateway.request("POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=strict.jwt(user)), + "user_vector_store_access_denied", + ) + + +@pytest.mark.parametrize( + "case", + ( + "standalone_key_no_permission", + "team_key_empty_key_grants", + "team_key_empty_team_grants", + "multi_store_one_ungranted", + "jwt_user_without_grant", + ), +) +def test_deny_by_default_rejects_ungranted_store_before_upstream_search( + strict_gateway: StrictGateway, case: StrictCase +) -> None: + with strict_gateway.gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + marker: Final = f"lit6035 deny by default {case} {uuid.uuid4().hex}" + + response, error_type = _strict_denied_request(strict_gateway, scenario, case, model, marker, store_id) + observations: Final = tuple( + observation + for observation in upstream_observations(strict_gateway.upstream) + if marker in str(observation["body"]) + ) + + assert response.status_code == 401, f"{response.text}; scripted_upstream_observations={observations!r}" + assert response.json()["error"]["type"] == error_type, response.text + assert observations == () + + +GrantedCase: TypeAlias = Literal[ + "standalone_key_granted_registered_store", + "team_key_both_grant_unregistered_store", + "jwt_team_member_team_grant_only", + "jwt_user_personal_grant", + "master_key_without_grants", +] + + +def _strict_granted_request( + strict: StrictGateway, scenario: Scenario, case: GrantedCase, model: str, marker: str, store_id: str +) -> httpx.Response: + models: Final = _json_array(model) + granted: Final = _permission_for_stores(store_id) + body: Final = _rag_query_body(model, marker, store_id) + if case in ("standalone_key_granted_registered_store", "team_key_both_grant_unregistered_store"): + key_team: Final = scenario.team(models=models, object_permission=granted) if case.startswith("team") else None + granted_key: Final = scenario.key( + models=models, object_permission=granted, **({} if key_team is None else {"team_id": key_team}) + ) + return strict.gateway.request("POST", "/v1/rag/query", body, key=granted_key) + if case == "jwt_team_member_team_grant_only": + member_team: Final = scenario.team(models=models, object_permission=granted) + member: Final = scenario.member(member_team) + return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(member, (member_team,))) + if case == "jwt_user_personal_grant": + user: Final = scenario.user(user_role="internal_user", object_permission=granted) + return strict.gateway.request("POST", "/v1/rag/query", body, key=strict.jwt(user)) + return strict.gateway.request("POST", "/v1/rag/query", body) + + +@pytest.mark.parametrize( + "case", + ( + "standalone_key_granted_registered_store", + "team_key_both_grant_unregistered_store", + "jwt_team_member_team_grant_only", + "jwt_user_personal_grant", + "master_key_without_grants", + ), +) +def test_deny_by_default_searches_explicitly_granted_store(strict_gateway: StrictGateway, case: GrantedCase) -> None: + with strict_gateway.gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = ( + CONFIG_STORE_ID + if case == "standalone_key_granted_registered_store" + else f"vs_unregistered_{uuid.uuid4().hex}" + ) + marker: Final = f"lit6035 granted {case} {uuid.uuid4().hex}" + + response: Final = _strict_granted_request(strict_gateway, scenario, case, model, marker, store_id) + assert response.status_code == 200, response.text + + searches: Final = _searches_for_marker(strict_gateway.upstream, marker, store_id) + assert len(searches) == 1, searches + assert marker in str(object_value(searches[0]["body"])["query"]), searches + + @pytest.mark.parametrize( ("scope", "error_type"), (("key", "key_vector_store_access_denied"), ("team", "team_vector_store_access_denied")), @@ -220,3 +434,136 @@ def test_rag_query_alias_denies_store_when_team_allowlist_excludes(gateway: Gate assert response.status_code == 401, response.text assert response.json()["error"]["type"] == "team_vector_store_access_denied", response.text assert _searches_for_marker(gateway, marker) == () + + +def test_explicit_false_flag_keeps_legacy_vector_store_outcomes(tmp_path: Path) -> None: + with gateway_from_environment() as upstream_gateway: + with owned_proxy( + upstream_gateway, + tmp_path, + _openai_environment(upstream_gateway), + config=_strict_config(tmp_path, deny_by_default=False), + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + with gateway.scenario() as scenario: + model: Final = scenario.model() + allowed_marker: Final = f"flag-false-allowed-{uuid.uuid4().hex}" + denied_marker: Final = f"flag-false-denied-{uuid.uuid4().hex}" + no_permission_key: Final = scenario.key(models=_json_array(model)) + excluding_key: Final = scenario.key( + models=_json_array(model), object_permission=_permission_for_stores("vs_some_other_store") + ) + + allowed: Final = _rag_query(gateway, model, allowed_marker, no_permission_key) + denied: Final = _rag_query(gateway, model, denied_marker, excluding_key) + + assert allowed.status_code == 200, allowed.text + assert len(_searches_for_marker(upstream_gateway, allowed_marker)) == 1 + assert denied.status_code == 401, denied.text + assert denied.json()["error"]["type"] == "key_vector_store_access_denied" + assert _searches_for_marker(upstream_gateway, denied_marker) == () + + +def _strict_no_registry_config(directory: Path) -> Path: + config: Final = object_value(yaml.safe_load(_no_registry_config(directory).read_text())) + general_settings: Final = object_value(config["general_settings"]) + strict: Final[JsonObject] = { + **config, + "general_settings": {**general_settings, "vector_store_deny_by_default": True}, + } + path: Final = directory / "proxy_vector_store_deny_by_default_no_registry.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +def test_deny_by_default_without_registry_checks_search_route_and_file_search_tools(tmp_path: Path) -> None: + with gateway_from_environment() as upstream_gateway: + with owned_proxy( + upstream_gateway, + tmp_path, + _openai_environment(upstream_gateway), + config=_strict_no_registry_config(tmp_path), + remove_environment=REMOVE_OPENAI_API_BASE, + ) as gateway: + with gateway.scenario() as scenario: + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + search_marker: Final = f"lit6035 no registry search {uuid.uuid4().hex}" + responses_marker: Final = f"lit6035 no registry responses {uuid.uuid4().hex}" + granted_marker: Final = f"lit6035 no registry granted search {uuid.uuid4().hex}" + ungranted_key: Final = scenario.key(models=_json_array(model)) + granted_key: Final = scenario.key( + models=_json_array(model), object_permission=_permission_for_stores(store_id) + ) + + search_denied: Final = gateway.request( + "POST", f"/v1/vector_stores/{store_id}/search", {"query": search_marker}, key=ungranted_key + ) + responses_denied: Final = gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "input": responses_marker, + "tools": _json_array({"type": "file_search", "vector_store_ids": _json_array(store_id)}), + }, + key=ungranted_key, + ) + search_granted: Final = gateway.request( + "POST", f"/v1/vector_stores/{store_id}/search", {"query": granted_marker}, key=granted_key + ) + + observations: Final = upstream_observations(upstream_gateway) + denied_observations: Final = tuple( + observation + for observation in observations + if search_marker in str(observation["body"]) or responses_marker in str(observation["body"]) + ) + granted_searches: Final = tuple( + observation + for observation in observations + if observation["path"] == f"/vector_stores/{store_id}/search" + and granted_marker in str(observation["body"]) + ) + assert search_denied.status_code == 401, search_denied.text + assert search_denied.json()["error"]["type"] == "key_vector_store_access_denied", search_denied.text + assert responses_denied.status_code == 401, responses_denied.text + assert responses_denied.json()["error"]["type"] == "key_vector_store_access_denied", responses_denied.text + assert denied_observations == () + assert search_granted.status_code == 200, search_granted.text + assert len(granted_searches) == 1, granted_searches + + +def _auth_cache_subscribers() -> int: + with Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) as cache: + return int(cache.pubsub_numsub(AUTH_CACHE_INVALIDATION_CHANNEL)[0][1]) + + +def test_revoked_user_grant_stops_working_on_another_proxy(strict_gateway: StrictGateway, tmp_path: Path) -> None: + subscribers_before_peer: Final = _auth_cache_subscribers() + with ( + owned_proxy( + strict_gateway.upstream, + tmp_path, + strict_gateway.environment, + config=strict_gateway.config, + remove_environment=REMOVE_OPENAI_API_BASE, + ) as peer, + strict_gateway.gateway.scenario() as scenario, + ): + eventually(_auth_cache_subscribers, lambda count: count > subscribers_before_peer) + model: Final = scenario.model() + store_id: Final = f"vs_unregistered_{uuid.uuid4().hex}" + user: Final = scenario.user(user_role="internal_user", object_permission=_permission_for_stores(store_id)) + token: Final = strict_gateway.jwt(user) + + def peer_status() -> int: + marker: Final = f"lit6035 revoked user grant {uuid.uuid4().hex}" + return peer.request( + "POST", "/v1/rag/query", _rag_query_body(model, marker, store_id), key=token + ).status_code + + assert eventually(peer_status, lambda status: status == 200, seconds=30, return_last_on_timeout=True) == 200 + strict_gateway.gateway.post("/user/update", {"user_id": user, "object_permission": _permission_for_stores()}) + + assert eventually(peer_status, lambda status: status == 401, seconds=10, return_last_on_timeout=True) == 401 diff --git a/tests/integration/authorization/test_search_tool_deny_by_default.py b/tests/integration/authorization/test_search_tool_deny_by_default.py new file mode 100644 index 00000000000..f29940f68b2 --- /dev/null +++ b/tests/integration/authorization/test_search_tool_deny_by_default.py @@ -0,0 +1,221 @@ +from __future__ import annotations + +import json +import uuid +from collections.abc import Iterator +from pathlib import Path +from typing import Final, Literal, TypeAlias + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, gateway_from_environment, object_value +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +PROXY_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" +SEARCH_TOOL: Final = "integration-search" +OTHER_SEARCH_TOOL: Final = "integration-other-search" +FAILING_SEARCH_TOOL: Final = "integration-failing-search" +SEARCH_RESULT: Final = {"title": "Synthetic result", "url": "https://example.test/result", "content": "snippet"} +JsonObject: TypeAlias = dict[str, JsonValue] + + +def _json_array(*values: JsonValue) -> JsonValue: + return [*values] # mutable-ok: request payloads and YAML sequences require list values + + +def _grant(*search_tools: str) -> JsonObject: + permission: Final[JsonObject] = {"search_tools": _json_array(*search_tools)} + return permission + + +def _respond(request: Request) -> Reply: + if (request.method, request.target) == ("POST", "/failing/search"): + return Reply(status=500, body=b'{"error": "synthetic outage"}') + assert (request.method, request.target) == ("POST", "/tavily/search"), request + query: Final = json.loads(request.body)["query"] + return Reply(body=json.dumps({"query": query, "results": [SEARCH_RESULT]}).encode()) + + +def _config(directory: Path, wire: Wire) -> Path: + config: Final = object_value(yaml.safe_load(PROXY_CONFIG.read_text())) + tool_params: Final[JsonObject] = { + "search_provider": "tavily", + "api_key": "synthetic-tavily-key", + "api_base": f"{wire.url}/tavily", + } + failing_tool_params: Final[JsonObject] = {**tool_params, "api_base": f"{wire.url}/failing"} + strict: Final[JsonObject] = { + **config, + "general_settings": {**object_value(config["general_settings"]), "search_tool_deny_by_default": True}, + "router_settings": { + **object_value(config["router_settings"]), + "fallbacks": _json_array({FAILING_SEARCH_TOOL: _json_array(OTHER_SEARCH_TOOL)}), + }, + "search_tools": _json_array( + {"search_tool_name": SEARCH_TOOL, "litellm_params": tool_params}, + {"search_tool_name": OTHER_SEARCH_TOOL, "litellm_params": tool_params}, + {"search_tool_name": FAILING_SEARCH_TOOL, "litellm_params": failing_tool_params}, + ), + } + path: Final = directory / "proxy_search_tool_deny_by_default.yaml" + path.write_text(yaml.safe_dump(strict)) + return path + + +@pytest.fixture(scope="module") +def strict_search(tmp_path_factory: pytest.TempPathFactory) -> Iterator[tuple[Gateway, Wire]]: + with gateway_from_environment() as upstream, wire_server(_respond) as wire: + directory: Final = tmp_path_factory.mktemp("search_tool_deny_by_default") + with owned_proxy(upstream, directory, {}, config=_config(directory, wire)) as gateway: + yield gateway, wire + + +def _search(gateway: Gateway, key: str, marker: str) -> httpx.Response: + return gateway.request("POST", f"/v1/search/{SEARCH_TOOL}", {"query": marker}, key=key) + + +def _searched(wire: Wire, marker: str) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if marker in request.body.decode()) + + +Caller: TypeAlias = Literal[ + "standalone_key_no_permission", + "standalone_key_empty_grant", + "standalone_key_other_tool", + "team_key_empty_key_grant", + "team_key_empty_team_grant", + "proxy_admin_key_no_grant", +] + + +def _denied_key(scenario: Scenario, caller: Caller) -> tuple[str, str]: + if caller == "standalone_key_no_permission": + return scenario.key(), "key_search_tool_access_denied" + if caller == "standalone_key_empty_grant": + return scenario.key(object_permission=_grant()), "key_search_tool_access_denied" + if caller == "standalone_key_other_tool": + return scenario.key(object_permission=_grant(OTHER_SEARCH_TOOL)), "key_search_tool_access_denied" + if caller == "team_key_empty_key_grant": + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + return scenario.key(team_id=team, object_permission=_grant()), "key_search_tool_access_denied" + if caller == "team_key_empty_team_grant": + empty_team: Final = scenario.team(object_permission=_grant()) + return scenario.key(team_id=empty_team, object_permission=_grant(SEARCH_TOOL)), "team_search_tool_access_denied" + admin: Final = scenario.user(user_role="proxy_admin") + return scenario.key(user_id=admin), "key_search_tool_access_denied" + + +@pytest.mark.parametrize( + "caller", + [ + "standalone_key_no_permission", + "standalone_key_empty_grant", + "standalone_key_other_tool", + "team_key_empty_key_grant", + "team_key_empty_team_grant", + "proxy_admin_key_no_grant", + ], +) +def test_deny_by_default_rejects_ungranted_search_before_the_provider_is_called( + strict_search: tuple[Gateway, Wire], caller: Caller +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + key, error_type = _denied_key(scenario, caller) + marker: Final = f"search deny {caller} {uuid.uuid4().hex}" + + response: Final = _search(gateway, key, marker) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == error_type, response.text + assert _searched(wire, marker) == () + + +@pytest.mark.parametrize("scope", ["standalone_key", "team_key", "master_key"]) +def test_deny_by_default_serves_explicitly_granted_search( + strict_search: tuple[Gateway, Wire], scope: Literal["standalone_key", "team_key", "master_key"] +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + if scope == "standalone_key": + key = scenario.key(object_permission=_grant(SEARCH_TOOL)) + elif scope == "team_key": + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + key = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL)) + else: + key = gateway.key + marker: Final = f"search allow {scope} {uuid.uuid4().hex}" + + response: Final = _search(gateway, key, marker) + + assert response.status_code == 200, response.text + assert response.json()["results"][0]["url"] == SEARCH_RESULT["url"] + assert len(_searched(wire, marker)) == 1 + + +def test_search_tools_list_shows_only_the_tools_the_key_and_its_team_both_grant( + strict_search: tuple[Gateway, Wire], +) -> None: + gateway, _ = strict_search + with gateway.scenario() as scenario: + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL, OTHER_SEARCH_TOOL)) + member: Final = scenario.member(team) + key: Final = scenario.key(team_id=team, user_id=member, object_permission=_grant(SEARCH_TOOL)) + + response: Final = gateway.request("GET", "/search_tools/list", key=key) + + assert response.status_code == 200, response.text + assert [tool["search_tool_name"] for tool in response.json()["search_tools"]] == [SEARCH_TOOL] + + +def test_revoking_a_team_search_grant_takes_effect_on_the_next_request(strict_search: tuple[Gateway, Wire]) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + team: Final = scenario.team(object_permission=_grant(SEARCH_TOOL)) + key: Final = scenario.key(team_id=team, object_permission=_grant(SEARCH_TOOL)) + assert _search(gateway, key, f"search warm {uuid.uuid4().hex}").status_code == 200 + + gateway.post("/team/update", {"team_id": team, "object_permission": _grant()}) + marker: Final = f"search revoked {uuid.uuid4().hex}" + response: Final = _search(gateway, key, marker) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "team_search_tool_access_denied", response.text + assert _searched(wire, marker) == () + + +def test_deny_by_default_authorizes_a_search_tool_named_by_the_model_field(strict_search: tuple[Gateway, Wire]) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + key: Final = scenario.key() + marker: Final = f"search model field {uuid.uuid4().hex}" + + response: Final = gateway.request("POST", "/v1/search", {"model": SEARCH_TOOL, "query": marker}, key=key) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "key_search_tool_access_denied", response.text + assert _searched(wire, marker) == () + + +@pytest.mark.parametrize("fallback_granted", [False, True]) +def test_router_search_fallback_only_reaches_a_granted_tool( + strict_search: tuple[Gateway, Wire], fallback_granted: bool +) -> None: + gateway, wire = strict_search + with gateway.scenario() as scenario: + granted: Final = (FAILING_SEARCH_TOOL, OTHER_SEARCH_TOOL) if fallback_granted else (FAILING_SEARCH_TOOL,) + key: Final = scenario.key(object_permission=_grant(*granted)) + marker: Final = f"search fallback {fallback_granted} {uuid.uuid4().hex}" + + response: Final = gateway.request("POST", f"/v1/search/{FAILING_SEARCH_TOOL}", {"query": marker}, key=key) + + searched: Final = tuple(request.target for request in _searched(wire, marker)) + if fallback_granted: + assert response.status_code == 200, response.text + assert searched == ("/failing/search", "/tavily/search") + else: + assert response.status_code == 500, response.text + assert searched == ("/failing/search",) diff --git a/tests/integration/caching/response_cache_case.py b/tests/integration/caching/response_cache_case.py new file mode 100644 index 00000000000..9df34721202 --- /dev/null +++ b/tests/integration/caching/response_cache_case.py @@ -0,0 +1,364 @@ +"""Scripted provider answers, upstream observations and Redis helpers shared by the response cache cells. + +The bodies carry ``$UNIQUE_ID`` so every upstream answer has its own id: an answer served from the +cache repeats the id of the request that filled it, an answer that reached the upstream again does not. +""" + +from __future__ import annotations + +import ast +import json +import os +import re +import uuid +from collections.abc import Iterator, Mapping +from contextlib import contextmanager +from typing import Final + +import httpx +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import JSON_OBJECT, Scenario, eventually, object_value, string_value +from tests.integration._support.upstream import CONTROL_URL, ScenarioHandle, delete_scenario, register_scenario +from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse + +CACHE_KEY: Final = re.compile(r"^[0-9a-f]{64}$") +PROVIDER_KEY: Final = "integration-provider-key" +CHAT_MODEL: Final = "openai/gpt-4o-mini" +TEXT_MODEL: Final = "openai/gpt-3.5-turbo-instruct" +MESSAGES_MODEL: Final = "anthropic/claude-haiku-4-5" + +CHAT: Final[dict[str, JsonValue]] = { + "id": "chatcmpl-$UNIQUE_ID", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "scripted"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +TEXT: Final[dict[str, JsonValue]] = { + "id": "cmpl-$UNIQUE_ID", + "object": "text_completion", + "created": 1, + "model": "gpt-3.5-turbo-instruct", + "choices": [{"text": "scripted", "index": 0, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} +RESPONSE: Final[dict[str, JsonValue]] = { + "id": "resp_$UNIQUE_ID", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_$UNIQUE_ID", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "scripted", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, +} +MESSAGE: Final[dict[str, JsonValue]] = { + "id": "msg-$UNIQUE_ID", + "type": "message", + "role": "assistant", + "model": "claude-haiku-4-5", + "content": [{"type": "text", "text": "scripted"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, +} +EMBEDDING: Final[dict[str, JsonValue]] = { + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "text-embedding-$UNIQUE_ID", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, +} +RERANK: Final[dict[str, JsonValue]] = { + "id": "rerank-$UNIQUE_ID", + "results": [{"index": 0, "relevance_score": 0.5}], + "meta": {}, +} +TRANSCRIPT: Final[dict[str, JsonValue]] = {"text": "scripted $UNIQUE_ID"} + +CHAT_CHUNK: Final = ( + 'data: {"id":"chatcmpl-$UNIQUE_ID","object":"chat.completion.chunk","created":1,"model":"gpt-4o-mini","choices":' +) +CHAT_STREAM: Final = SseResponse( + content_type="text/event-stream", + frames=( + CHAT_CHUNK + '[{"index":0,"delta":{"role":"assistant"},"finish_reason":null}]}', + CHAT_CHUNK + '[{"index":0,"delta":{"content":"streamed "},"finish_reason":null}]}', + CHAT_CHUNK + '[{"index":0,"delta":{"content":"response"},"finish_reason":null}]}', + CHAT_CHUNK + '[{"index":0,"delta":{},"finish_reason":"stop"}]}', + "data: [DONE]", + ), +) +CHAT_STREAM_WITHOUT_CHOICES: Final = SseResponse( + content_type="text/event-stream", + frames=(CHAT_CHUNK + "[]}", "data: [DONE]"), +) +TEXT_CHUNK: Final = ( + 'data: {"id":"cmpl-$UNIQUE_ID","object":"text_completion","created":1,"model":"gpt-3.5-turbo-instruct","choices":' +) +TEXT_STREAM: Final = SseResponse( + content_type="text/event-stream", + frames=( + TEXT_CHUNK + '[{"text":"streamed ","index":0,"finish_reason":null}]}', + TEXT_CHUNK + '[{"text":"response","index":0,"finish_reason":null}]}', + TEXT_CHUNK + '[{"text":"","index":0,"finish_reason":"stop"}]}', + "data: [DONE]", + ), +) +TEXT_STREAM_WITHOUT_CHOICES: Final = SseResponse( + content_type="text/event-stream", + frames=(TEXT_CHUNK + "[]}", "data: [DONE]"), +) +RESPONSE_CREATED: Final = ( + "event: response.created\n" + 'data: {"type":"response.created","response":{"id":"resp_$UNIQUE_ID","object":"response",' + '"created_at":1,"status":"in_progress","model":"gpt-4o-mini","output":[],"usage":null}}' +) +RESPONSE_STREAM: Final = SseResponse( + content_type="text/event-stream", + frames=( + RESPONSE_CREATED, + ( + "event: response.output_item.added\n" + 'data: {"type":"response.output_item.added","output_index":0,' + '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"in_progress","role":"assistant","content":[]}}' + ), + ( + "event: response.content_part.added\n" + 'data: {"type":"response.content_part.added","item_id":"msg_$UNIQUE_ID","output_index":0,' + '"content_index":0,"part":{"type":"output_text","text":"","annotations":[]}}' + ), + ( + "event: response.output_text.delta\n" + 'data: {"type":"response.output_text.delta","item_id":"msg_$UNIQUE_ID","output_index":0,' + '"content_index":0,"delta":"streamed response"}' + ), + ( + "event: response.output_text.done\n" + 'data: {"type":"response.output_text.done","item_id":"msg_$UNIQUE_ID","output_index":0,' + '"content_index":0,"text":"streamed response"}' + ), + ( + "event: response.content_part.done\n" + 'data: {"type":"response.content_part.done","item_id":"msg_$UNIQUE_ID","output_index":0,' + '"content_index":0,"part":{"type":"output_text","text":"streamed response","annotations":[]}}' + ), + ( + "event: response.output_item.done\n" + 'data: {"type":"response.output_item.done","output_index":0,' + '"item":{"type":"message","id":"msg_$UNIQUE_ID","status":"completed","role":"assistant",' + '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}}' + ), + ( + "event: response.completed\n" + 'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",' + '"created_at":1,"status":"completed","model":"gpt-4o-mini",' + '"output":[{"id":"msg_$UNIQUE_ID","type":"message","status":"completed","role":"assistant",' + '"content":[{"type":"output_text","text":"streamed response","annotations":[]}]}],' + '"usage":{"input_tokens":1,"output_tokens":2,"total_tokens":3}}}' + ), + ), +) +RESPONSE_STREAM_WITHOUT_OUTPUT: Final = SseResponse( + content_type="text/event-stream", + frames=( + RESPONSE_CREATED, + ( + "event: response.completed\n" + 'data: {"type":"response.completed","response":{"id":"resp_$UNIQUE_ID","object":"response",' + '"created_at":1,"status":"completed","model":"gpt-4o-mini","output":[],' + '"usage":{"input_tokens":1,"output_tokens":0,"total_tokens":1}}}' + ), + ), +) +MESSAGE_START: Final = ( + "event: message_start\n" + 'data: {"type":"message_start","message":{"id":"msg_$UNIQUE_ID","type":"message","role":"assistant",' + '"model":"claude-haiku-4-5","content":[],"stop_reason":null,"stop_sequence":null,' + '"usage":{"input_tokens":1,"output_tokens":0}}}' +) +MESSAGE_END: Final = ( + ( + "event: message_delta\n" + 'data: {"type":"message_delta","delta":{"stop_reason":"end_turn","stop_sequence":null},' + '"usage":{"output_tokens":2}}' + ), + 'event: message_stop\ndata: {"type":"message_stop"}', +) +MESSAGE_STREAM: Final = SseResponse( + content_type="text/event-stream", + frames=( + MESSAGE_START, + ( + "event: content_block_start\n" + 'data: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}' + ), + ( + "event: content_block_delta\n" + 'data: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"streamed response"}}' + ), + 'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}', + *MESSAGE_END, + ), +) +MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS: Final = SseResponse( + content_type="text/event-stream", frames=(MESSAGE_START, *MESSAGE_END) +) + + +class Upstream: + """The owned upstream's observations, drained once per read and kept, filtered per deployment.""" + + def __init__(self, url: str = CONTROL_URL) -> None: + self._url: Final = url.rstrip("/") + self._seen: tuple[dict[str, JsonValue], ...] = () + + def received(self, identity: str) -> tuple[dict[str, JsonValue], ...]: + with httpx.Client(timeout=10, trust_env=False) as client: + payload: Final = JSON_OBJECT.validate_json(client.get(f"{self._url}/__observations").content) + requests: Final = payload["requests"] + assert isinstance(requests, list) + self._seen = ( + *self._seen, + *(object_value(item) for item in requests), + ) # rebind-ok: drained observations accumulate + return tuple(item for item in self._seen if f"/{identity}/" in string_value(item.get("path", ""))) + + def calls(self, identity: str) -> int: + """The LLM calls the deployment took; the proxy's boot-time ``GET /v1/models`` discovery of a config + deployment is not one.""" + return sum(1 for item in self.received(identity) if item.get("method") == "POST") + + +def json_response(body: Mapping[str, JsonValue], status: int = 200) -> JsonResponse: + return JsonResponse(content_type="application/json", body=dict(body), status=status) + + +def slowed(stream: SseResponse, frame_delay_ms: int) -> SseResponse: + return stream.model_copy(update={"frame_delay_ms": frame_delay_ms}) + + +def emptied(body: Mapping[str, JsonValue], field: str) -> dict[str, JsonValue]: + return {**body, field: []} + + +def scenario_id() -> str: + return f"cache-{uuid.uuid4().hex[:12]}" + + +def scripted(scenario: Scenario, response: JsonResponse | SseResponse) -> ScenarioHandle: + handle: Final = register_scenario(scenario_id(), response, control_url=scenario.gateway.upstream_url) + scenario.cleanups.callback(delete_scenario, handle) + return handle + + +@contextmanager +def scripted_in_process(response: JsonResponse | SseResponse) -> Iterator[ScenarioHandle]: + handle: Final = register_scenario(scenario_id(), response) + try: + yield handle + finally: + delete_scenario(handle) + + +def rescript(handle: ScenarioHandle, response: JsonResponse | SseResponse) -> None: + register_scenario(handle.scenario_id, response, control_url=handle.control_url) + + +def prompt() -> str: + return f"cache probe {uuid.uuid4().hex}" + + +def redis_store() -> Redis: + return Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"])) + + +def entry_key(store: Redis, needle: str) -> str | None: + """The response cache key (a bare sha256) whose stored value names ``needle``, or ``None``.""" + for raw_key in store.scan_iter(count=1000): + key: Final = raw_key.decode() + if not CACHE_KEY.match(key) or store.type(key) != b"string": + continue + value: Final = store.get(key) + if value is not None and needle.encode() in value: + return key + return None + + +def await_entry(store: Redis, needle: str) -> str: + key: Final = eventually(lambda: entry_key(store, needle), lambda found: found is not None) + assert key is not None + return key + + +def _decode_entry(raw: bytes) -> tuple[dict[str, JsonValue], bool]: + """The stored entry and whether it is JSON; the sync SDK path stores ``str(dict)`` instead.""" + text: Final = raw.decode() + if text.startswith('{"'): + return object_value(json.loads(text)), True + return object_value(ast.literal_eval(text)), False + + +SHORT_TTL_SECONDS: Final = 2 +SHORT_TTL_CACHE_CONTROL: Final[dict[str, JsonValue]] = {"ttl": SHORT_TTL_SECONDS} +STALE_ENTRY_TTL_SECONDS: Final = 60 +STALE_ID_SUFFIX: Final = "-stale" + + +def flip_entry_to_empty(store: Redis, key: str, field: str) -> str: + """Rewrite the stored response so ``field`` is ``[]`` and its id carries ``-stale``, keeping the encoding. + + Returns the stale id. The proxy worker that wrote the entry keeps a copy in its own memory for the entry's ttl, + so a cell fills the entry with ``SHORT_TTL_CACHE_CONTROL`` and reads the rewritten entry once that copy lapsed. + """ + raw: Final = store.get(key) + assert raw is not None, key + entry, is_json = _decode_entry(raw) + response: Final = entry["response"] + stored: Final = object_value(json.loads(response) if isinstance(response, str) else response) + assert field in stored, f"{field!r} missing from the cached response {sorted(stored)}" + stale_id: Final = f"{string_value(stored['id'])}{STALE_ID_SUFFIX}" + flipped: Final[dict[str, JsonValue]] = {**stored, field: [], "id": stale_id} + rewritten: Final[dict[str, JsonValue]] = { + **entry, + "response": json.dumps(flipped) if isinstance(response, str) else flipped, + } + store.set(key, json.dumps(rewritten) if is_json else str(rewritten), ex=STALE_ENTRY_TTL_SECONDS) + return stale_id + + +def chat_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": text}], **extra} + + +def message_body(model: str, text: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": text}], **extra} + + +def response_id(response: httpx.Response) -> str: + assert response.status_code == 200, response.text + return string_value(JSON_OBJECT.validate_json(response.content)["id"]) + + +def choices(response: httpx.Response) -> list[JsonValue]: + assert response.status_code == 200, response.text + found: Final = JSON_OBJECT.validate_json(response.content)["choices"] + assert isinstance(found, list), response.text + return found + + +def content_of_first_choice(response: httpx.Response) -> JsonValue: + return object_value(object_value(choices(response)[0])["message"])["content"] + + +def text_of_first_choice(response: httpx.Response) -> JsonValue: + return object_value(choices(response)[0])["text"] diff --git a/tests/integration/caching/test_response_cache_chaos.py b/tests/integration/caching/test_response_cache_chaos.py new file mode 100644 index 00000000000..2e663c97d1f --- /dev/null +++ b/tests/integration/caching/test_response_cache_chaos.py @@ -0,0 +1,503 @@ +"""Chaos rows for the response cache on an owned two-worker proxy. + +Each owned proxy carries its own deployments pointed at scripted upstream scenarios, so both workers know +them from boot. A 24-request burst (chat, text completions, Responses and Anthropic Messages, two of each +streamed at 500 ms per frame so they are still in flight when the outage lands) runs while the cache store +is stopped (C1), paused (C2) or while one worker has just been SIGKILLed (C3): every request answers 200 +with content and lands exactly once in the spend log. After recovery an upstream answer with no choices +still reaches the upstream again on the next identical request. C4 runs the in-memory cache mode, where +each worker keeps its own store and the cached object is read back without a JSON round trip. +""" + +from __future__ import annotations + +import os +import threading +import uuid +from collections.abc import Callable, Iterator, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from dataclasses import dataclass +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import psutil +import pytest +import yaml +from pydantic import JsonValue +from redis import Redis + +from tests.integration._support.client import JSON_OBJECT, Gateway, Scenario, eventually, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.process import owned_proxy_process +from tests.integration._support.redis_process import owned_redis +from tests.integration._support.upstream import ScenarioHandle +from tests.integration.cost_calculation.cost_tracking_case import JsonResponse, SseResponse +from tests.integration.caching.response_cache_case import ( + CHAT, + CHAT_MODEL, + CHAT_STREAM, + MESSAGE, + MESSAGE_STREAM, + MESSAGES_MODEL, + PROVIDER_KEY, + RESPONSE, + RESPONSE_STREAM, + TEXT, + TEXT_MODEL, + TEXT_STREAM, + Upstream, + chat_body, + choices, + content_of_first_choice, + emptied, + json_response, + message_body, + prompt, + rescript, + response_id, + scripted, + slowed, +) + +RecordProperty = Callable[[str, object], None] + +WORKERS: Final = 2 +REMOVE_FROM_ENVIRONMENT: Final = ("DATABASE_URL_READ_REPLICA",) +BURST_TIMEOUT_SECONDS: Final = 60 +CHAOS_AFTER_ANSWERS: Final = 4 +STREAM_FRAME_DELAY_MS: Final = 500 +PER_ENDPOINT: Final = 6 +STREAMS_PER_ENDPOINT: Final = 2 +LOCAL_BATCH: Final = 20 +SPEND_SQL: Final = 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE api_key = %s' + + +@dataclass(frozen=True, slots=True) +class Deployment: + name: str + litellm_model: str + handle: ScenarioHandle + + def entry(self) -> dict[str, JsonValue]: + return { + "model_name": self.name, + "litellm_params": { + "model": self.litellm_model, + "api_base": self.handle.api_base(), + "api_key": PROVIDER_KEY, + }, + } + + +@dataclass(frozen=True, slots=True) +class Fleet: + chat: Deployment + chat_stream: Deployment + text: Deployment + text_stream: Deployment + responses: Deployment + responses_stream: Deployment + messages: Deployment + messages_stream: Deployment + flip: Deployment + + def burst_members(self) -> tuple[Deployment, ...]: + return ( + self.chat, + self.chat_stream, + self.text, + self.text_stream, + self.responses, + self.responses_stream, + self.messages, + self.messages_stream, + ) + + def entries(self) -> list[dict[str, JsonValue]]: + return [deployment.entry() for deployment in (*self.burst_members(), self.flip)] + + +@dataclass(frozen=True, slots=True) +class Call: + label: str + path: str + body: dict[str, JsonValue] + + @property + def streamed(self) -> bool: + return self.label.startswith("stream-") + + +@dataclass(frozen=True, slots=True) +class Answer: + call: Call + status: int | None + text: str + + @property + def has_content(self) -> bool: + marker: Final = "streamed" if self.call.streamed else "scripted" + return self.status == 200 and marker in self.text + + def describe(self) -> str: + return f"{self.call.label}: {self.status} {self.text[:160]!r}" + + +def _deployment(scenario: Scenario, kind: str, litellm_model: str, response: JsonResponse | SseResponse) -> Deployment: + return Deployment(f"chaos-{kind}-{uuid.uuid4().hex[:8]}", litellm_model, scripted(scenario, response)) + + +def _fleet(scenario: Scenario) -> Fleet: + return Fleet( + chat=_deployment(scenario, "chat", CHAT_MODEL, json_response(CHAT)), + chat_stream=_deployment(scenario, "chat-stream", CHAT_MODEL, slowed(CHAT_STREAM, STREAM_FRAME_DELAY_MS)), + text=_deployment(scenario, "text", TEXT_MODEL, json_response(TEXT)), + text_stream=_deployment(scenario, "text-stream", TEXT_MODEL, slowed(TEXT_STREAM, STREAM_FRAME_DELAY_MS)), + responses=_deployment(scenario, "responses", CHAT_MODEL, json_response(RESPONSE)), + responses_stream=_deployment( + scenario, "responses-stream", CHAT_MODEL, slowed(RESPONSE_STREAM, STREAM_FRAME_DELAY_MS) + ), + messages=_deployment(scenario, "messages", MESSAGES_MODEL, json_response(MESSAGE)), + messages_stream=_deployment( + scenario, "messages-stream", MESSAGES_MODEL, slowed(MESSAGE_STREAM, STREAM_FRAME_DELAY_MS) + ), + flip=_deployment(scenario, "flip", CHAT_MODEL, json_response(emptied(CHAT, "choices"))), + ) + + +def _config(directory: Path, fleet: Fleet, cache_params: dict[str, JsonValue] | None = None) -> Path: + """The rig's proxy config with the fleet as its model list and, when given, another response cache.""" + rig: Final = yaml.safe_load((Path(__file__).resolve().parents[1] / "proxy_config.yaml").read_text()) + assert isinstance(rig, dict) + litellm_settings: Final = rig["litellm_settings"] + assert isinstance(litellm_settings, dict) + config: Final = { + **rig, + "model_list": fleet.entries(), + "litellm_settings": {**litellm_settings, **({"cache_params": cache_params} if cache_params else {})}, + } + path: Final = directory / f"proxy_config-{uuid.uuid4().hex[:8]}.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False)) + return path + + +def _overrides() -> dict[str, str]: + return {"DATABASE_URL": os.environ["DATABASE_URL"]} + + +def _burst_calls(fleet: Fleet, per_endpoint: int, streams_per_endpoint: int) -> tuple[Call, ...]: + def calls() -> Iterator[Call]: + for index in range(per_endpoint): + streamed: Final = index < streams_per_endpoint + prefix: Final = "stream" if streamed else "plain" + extra: Final[dict[str, JsonValue]] = {"stream": True} if streamed else {} + chat: Final = fleet.chat_stream if streamed else fleet.chat + text: Final = fleet.text_stream if streamed else fleet.text + responses: Final = fleet.responses_stream if streamed else fleet.responses + messages: Final = fleet.messages_stream if streamed else fleet.messages + yield Call(f"{prefix}-chat-{index}", "/v1/chat/completions", chat_body(chat.name, prompt(), **extra)) + yield Call(f"{prefix}-text-{index}", "/v1/completions", {"model": text.name, "prompt": prompt(), **extra}) + yield Call( + f"{prefix}-responses-{index}", "/v1/responses", {"model": responses.name, "input": prompt(), **extra} + ) + yield Call(f"{prefix}-messages-{index}", "/v1/messages", message_body(messages.name, prompt(), **extra)) + + return tuple(calls()) + + +class Burst: + """Every call submitted at once against ``target``; ``chaos_point`` is set once ``CHAOS_AFTER_ANSWERS`` + have answered (or failed), while the slowed streams are still in flight.""" + + def __init__(self, target: str, key: str) -> None: + self._target: Final = target + self._key: Final = key + self._lock: Final = threading.Lock() + self._answers = 0 # rebind-ok: counter behind _lock + self._futures: dict[str, Future[Answer]] = {} + self.chaos_point: Final = threading.Event() + + def start(self, pool: ThreadPoolExecutor, calls: Sequence[Call]) -> None: + assert not self._futures, "burst already started" + self._futures.update((call.label, pool.submit(self._send, call)) for call in calls) + assert self.chaos_point.wait(BURST_TIMEOUT_SECONDS), ( + f"fewer than {CHAOS_AFTER_ANSWERS} requests answered within {BURST_TIMEOUT_SECONDS}s" + ) + + def _send(self, call: Call) -> Answer: + try: + with httpx.Client(base_url=self._target, timeout=BURST_TIMEOUT_SECONDS, trust_env=False) as client: + response: Final = client.post( + call.path, json=call.body, headers={"Authorization": f"Bearer {self._key}"} + ) + answer = Answer(call, response.status_code, response.text) + except httpx.HTTPError as error: + answer = Answer(call, None, f"{type(error).__name__}: {error}"[:200]) + with self._lock: + self._answers += 1 + if self._answers >= CHAOS_AFTER_ANSWERS: + self.chaos_point.set() + return answer + + def answered(self) -> int: + with self._lock: + return self._answers + + def pending_streams(self) -> tuple[str, ...]: + return tuple( + label for label, future in self._futures.items() if label.startswith("stream-") and not future.done() + ) + + def outcomes(self) -> tuple[Answer, ...]: + return tuple(future.result(timeout=BURST_TIMEOUT_SECONDS + 30) for future in self._futures.values()) + + +def _assert_all_answered_with_content(outcomes: Sequence[Answer]) -> None: + failures: Final = tuple(answer.describe() for answer in outcomes if not answer.has_content) + assert not failures, ( + f"{len(failures)} of {len(outcomes)} burst requests lack a 200 with content:\n " + "\n ".join(failures) + ) + + +def _plain_ids(outcomes: Sequence[Answer]) -> frozenset[str]: + """The ids of the non-streamed chat, text and Messages answers, which the spend log records verbatim.""" + return frozenset( + string_value(JSON_OBJECT.validate_json(answer.text)["id"]) + for answer in outcomes + if not answer.call.streamed and answer.call.path != "/v1/responses" + ) # comprehension-ok: one filter over the burst + + +def _spend_request_ids(key: str) -> frozenset[str]: + return frozenset( + string_value(row["request_id"]) for row in read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),)) + ) + + +def _assert_every_burst_request_landed_once(key: str, outcomes: Sequence[Answer]) -> None: + rows: Final = eventually(lambda: _spend_request_ids(key), lambda ids: len(ids) >= len(outcomes), seconds=70) + assert len(rows) == len(outcomes), f"{len(rows)} spend rows for {len(outcomes)} burst requests: {sorted(rows)}" + missing: Final = _plain_ids(outcomes) - rows + assert not missing, f"burst answers without a spend row: {sorted(missing)}" + + +def _chat(target: str, key: str, body: dict[str, JsonValue]) -> httpx.Response: + with httpx.Client(base_url=target, timeout=30, trust_env=False) as client: + return client.post("/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"}) + + +def _assert_empty_answer_reaches_upstream_again(target: str, key: str, upstream: Upstream, flip: Deployment) -> None: + """Through ``target`` after the outage: the scripted empty answer is never served again once the + upstream answers for real, and the real answer is the one served from the cache afterwards.""" + request: Final = chat_body(flip.name, prompt()) + empty: Final = _chat(target, key, request) + assert choices(empty) == [], empty.text + assert upstream.calls(flip.handle.scenario_id) == 1 + rescript(flip.handle, json_response(CHAT)) + refilled: Final = _chat(target, key, request) + assert content_of_first_choice(refilled) == "scripted", f"the empty answer was served: {refilled.text}" + assert upstream.calls(flip.handle.scenario_id) == 2 + hit: Final = eventually(lambda: _chat(target, key, request), lambda r: response_id(r) == response_id(refilled)) + assert content_of_first_choice(hit) == "scripted", hit.text + assert upstream.calls(flip.handle.scenario_id) >= 2 + + +def _workers(root: psutil.Process) -> tuple[psutil.Process, ...]: + """uvicorn's worker children of the owned proxy root, spawned through ``multiprocessing.spawn``.""" + + def spawned() -> Iterator[psutil.Process]: + for child in root.children(): + try: + cmdline = child.cmdline() + except (psutil.NoSuchProcess, psutil.AccessDenied): + continue + if any("multiprocessing.spawn" in part for part in cmdline): + yield child + + return tuple(sorted(spawned(), key=lambda process: process.pid)) + + +def _cache_ping(target: str, key: str) -> httpx.Response: + with httpx.Client(base_url=target, timeout=30, trust_env=False) as client: + return client.get("/cache/ping", headers={"Authorization": f"Bearer {key}"}) + + +def _cache_status(response: httpx.Response) -> str: + assert response.status_code == 200, f"/cache/ping: {response.status_code} {response.text}" + return string_value(JSON_OBJECT.validate_json(response.content)["status"]) + + +def _proxy_url(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +@pytest.mark.timeout(300) +def test_redis_stopped_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario, owned_redis(tmp_path) as store: + fleet: Final = _fleet(scenario) + key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)]) + with ( + owned_proxy_process( + gateway, + tmp_path, + { + **_overrides(), + "REDIS_HOST": store.host, + "REDIS_PORT": str(store.port), + "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "0", + }, + config=_config(tmp_path, fleet), + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned, + ThreadPoolExecutor(PER_ENDPOINT * 4) as pool, + ): + target: Final = _proxy_url(owned.gateway) + assert _cache_status(_cache_ping(target, gateway.key)) == "healthy" + burst: Final = Burst(target, key) + burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT)) + pending: Final = burst.pending_streams() + record_property("answered_before_outage", burst.answered()) + assert pending, "no stream was in flight when Redis stopped" + store.stop() + down: Final = _cache_ping(target, gateway.key) + assert down.status_code == 503, f"/cache/ping with Redis stopped: {down.status_code} {down.text}" + assert "Service Unhealthy" in down.text, down.text + outcomes: Final = burst.outcomes() + store.start() + recovered: Final = eventually( + lambda: _cache_ping(target, gateway.key), lambda r: r.status_code == 200, seconds=60 + ) + assert _cache_status(recovered) == "healthy" + _assert_all_answered_with_content(outcomes) + _assert_every_burst_request_landed_once(key, outcomes) + _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip) + + +@pytest.mark.timeout(300) +def test_redis_paused_mid_burst_answers_every_request_and_never_pins_an_empty_answer_after( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario, owned_redis(tmp_path) as store: + fleet: Final = _fleet(scenario) + key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)]) + with ( + owned_proxy_process( + gateway, + tmp_path, + {**_overrides(), "REDIS_HOST": store.host, "REDIS_PORT": str(store.port)}, + config=_config(tmp_path, fleet), + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned, + ThreadPoolExecutor(PER_ENDPOINT * 4) as pool, + Redis(host=store.host, port=store.port) as control, + ): + target: Final = _proxy_url(owned.gateway) + assert _cache_status(_cache_ping(target, gateway.key)) == "healthy" + burst: Final = Burst(target, key) + burst.start(pool, _burst_calls(fleet, PER_ENDPOINT, STREAMS_PER_ENDPOINT)) + pending: Final = burst.pending_streams() + record_property("answered_before_pause", burst.answered()) + assert pending, "no stream was in flight when Redis was paused" + assert control.client_pause(2000) is True + outcomes: Final = burst.outcomes() + assert _cache_status(_cache_ping(target, gateway.key)) == "healthy" + _assert_all_answered_with_content(outcomes) + _assert_every_burst_request_landed_once(key, outcomes) + _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip) + + +@pytest.mark.timeout(300) +def test_worker_killed_before_burst_leaves_the_survivor_serving_and_never_pins_an_empty_answer( + gateway: Gateway, tmp_path: Path, record_property: RecordProperty +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + fleet: Final = _fleet(scenario) + key: Final = scenario.key(models=[deployment.name for deployment in (*fleet.burst_members(), fleet.flip)]) + with ( + owned_proxy_process( + gateway, + tmp_path, + _overrides(), + config=_config(tmp_path, fleet), + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned, + ThreadPoolExecutor(PER_ENDPOINT * 4) as pool, + ): + target: Final = _proxy_url(owned.gateway) + root: Final = psutil.Process(owned.process.pid) + before: Final = _workers(root) + assert len(before) == WORKERS, [process.pid for process in before] + victim: Final = before[0] + victim.kill() + victim.wait(timeout=10) + with httpx.Client(base_url=target, timeout=15, trust_env=False) as fresh: + readiness: Final = fresh.get("/health/readiness") + assert readiness.status_code == 200, f"/health/readiness with worker {victim.pid} dead: {readiness.text}" + burst: Final = Burst(target, key) + burst.start(pool, _burst_calls(fleet, PER_ENDPOINT // 2, STREAMS_PER_ENDPOINT // 2)) + outcomes: Final = burst.outcomes() + respawned: Final = eventually( + lambda: tuple(process.pid for process in _workers(root)), + lambda pids: len(pids) == WORKERS and victim.pid not in pids, + seconds=60, + ) + record_property( + "worker_pids", {"before": [process.pid for process in before], "killed": victim.pid, "after": respawned} + ) + _assert_all_answered_with_content(outcomes) + _assert_every_burst_request_landed_once(key, outcomes) + _assert_empty_answer_reaches_upstream_again(target, key, upstream, fleet.flip) + + +def _batch(target: str, key: str, body: dict[str, JsonValue], size: int) -> tuple[httpx.Response, ...]: + """``size`` identical requests, each on a fresh connection so both workers take traffic.""" + return tuple(_chat(target, key, body) for _ in range(size)) + + +@pytest.mark.timeout(300) +def test_in_memory_cache_mode_never_serves_an_empty_answer_from_a_worker_store( + gateway: Gateway, tmp_path: Path +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + fleet: Final = _fleet(scenario) + key: Final = scenario.key(models=[fleet.flip.name]) + with owned_proxy_process( + gateway, + tmp_path, + _overrides(), + config=_config(tmp_path, fleet, cache_params={"type": "local"}), + remove_environment=REMOVE_FROM_ENVIRONMENT, + workers=WORKERS, + ) as owned: + target: Final = _proxy_url(owned.gateway) + request: Final = chat_body(fleet.flip.name, prompt()) + first_batch: Final = _batch(target, key, request, LOCAL_BATCH) + assert all(choices(response) == [] for response in first_batch), [r.text for r in first_batch] + assert upstream.calls(fleet.flip.handle.scenario_id) == LOCAL_BATCH, ( + "an empty answer was served from a worker store" + ) + rescript(fleet.flip.handle, json_response(CHAT)) + second_batch: Final = _batch(target, key, request, LOCAL_BATCH) + served_empty: Final = tuple(response.text for response in second_batch if choices(response) == []) + assert not served_empty, ( + f"{len(served_empty)} of {LOCAL_BATCH} answers still empty after the upstream recovered" + ) + assert all(content_of_first_choice(response) == "scripted" for response in second_batch) + after_second: Final = upstream.calls(fleet.flip.handle.scenario_id) + assert LOCAL_BATCH < after_second <= 2 * LOCAL_BATCH, after_second + third_batch: Final = _batch(target, key, request, LOCAL_BATCH) + assert all(content_of_first_choice(response) == "scripted" for response in third_batch) + assert upstream.calls(fleet.flip.handle.scenario_id) - after_second <= WORKERS, ( + "the real answer is not cached" + ) diff --git a/tests/integration/caching/test_response_cache_empty_output.py b/tests/integration/caching/test_response_cache_empty_output.py new file mode 100644 index 00000000000..ce5a4d64ca7 --- /dev/null +++ b/tests/integration/caching/test_response_cache_empty_output.py @@ -0,0 +1,813 @@ +"""The response cache never stores an answer with no output and never serves a stored one. + +Every cell scripts the owned upstream for one deployment, drives the rig proxy through the client a +user would hold (OpenAI SDK, Anthropic SDK, raw httpx) and reads three things: what the caller got, +how often the upstream was called for that deployment (an answer served from the cache never reaches +it), and what landed in Redis or the spend log. An upstream answer carrying no choices, no output items +or no content blocks reaches the upstream again on the next identical request, and the real answer +that follows is the one the cache keeps. Answers with output, errors, ``no-cache`` requests, the +``/cache/delete`` workaround and the other cached call types are unchanged. +""" + +from __future__ import annotations + +import asyncio +import json +import uuid +from collections.abc import Callable, Iterator, Mapping +from dataclasses import dataclass +from functools import partial +from hashlib import sha256 +from typing import Final, TypeAlias, TypeVar + +import httpx +import pytest +from anthropic import Anthropic, AsyncAnthropic +from openai import AsyncOpenAI, OpenAI +from pydantic import JsonValue + +from tests.integration._support.client import JSON_OBJECT, Gateway, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration.caching.response_cache_case import ( + CHAT, + CHAT_MODEL, + CHAT_STREAM, + CHAT_STREAM_WITHOUT_CHOICES, + EMBEDDING, + MESSAGE, + MESSAGE_STREAM, + MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS, + MESSAGES_MODEL, + RERANK, + RESPONSE, + RESPONSE_STREAM, + RESPONSE_STREAM_WITHOUT_OUTPUT, + SHORT_TTL_CACHE_CONTROL, + TEXT, + TEXT_MODEL, + TEXT_STREAM_WITHOUT_CHOICES, + TRANSCRIPT, + Upstream, + await_entry, + chat_body, + choices, + content_of_first_choice, + emptied, + flip_entry_to_empty, + json_response, + message_body, + prompt, + redis_store, + rescript, + response_id, + scripted, + text_of_first_choice, +) + +MISSING: Final = object() +EMPTY_STREAMS: Final = 6 +REPLAY_ATTEMPTS: Final = 8 +SETTLE_SECONDS: Final = 15 +T = TypeVar("T") +_Answer: TypeAlias = tuple[frozenset[str], str, tuple[str, ...]] +SPEND_SQL: Final = ( + 'SELECT request_id, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s ORDER BY "startTime", request_id' +) + + +def _proxy_root(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def _openai(gateway: Gateway, key: str | None = None) -> OpenAI: + return OpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0) + + +def _async_openai(gateway: Gateway, key: str | None = None) -> AsyncOpenAI: + return AsyncOpenAI(base_url=f"{_proxy_root(gateway)}/v1", api_key=key or gateway.key, max_retries=0) + + +def _anthropic(gateway: Gateway) -> Anthropic: + return Anthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0) + + +def _async_anthropic(gateway: Gateway) -> AsyncAnthropic: + return AsyncAnthropic(base_url=_proxy_root(gateway), api_key=gateway.key, max_retries=0) + + +def _chat(gateway: Gateway, body: Mapping[str, JsonValue], *, key: str | None = None) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", body, key=key) + + +def _spend_rows(key: str) -> list[dict[str, JsonValue]]: + return read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),)) + + +def _assert_prompt_reached_upstream(upstream: Upstream, identity: str, text: str, times: int) -> None: + received: Final = upstream.received(identity) + assert len(received) == times, f"{identity}: {len(received)} upstream calls, expected {times}" + assert all(text in json.dumps(item["body"]) for item in received), received + + +def _served_without_upstream(upstream: Upstream, identity: str, call: Callable[[], T]) -> T | None: + before: Final = upstream.calls(identity) + answer: Final = call() + return None if upstream.calls(identity) > before else answer + + +@dataclass(frozen=True, slots=True) +class Streamed: + ids: frozenset[str] + text: str + kinds: tuple[str, ...] + reached_upstream: bool + + +def _streams(stream: Callable[[], _Answer], upstream: Upstream, identity: str, attempts: int) -> Iterator[Streamed]: + """Stream up to ``attempts`` times, stopping after the first answer served without an upstream call.""" + for _ in range(attempts): + before: Final = upstream.calls(identity) + ids, text, kinds = stream() + answer: Final = Streamed(ids, text, kinds, upstream.calls(identity) > before) + yield answer + if not answer.reached_upstream: + return + + +def _never_replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str) -> tuple[Streamed, ...]: + """``EMPTY_STREAMS`` identical streams, every one reaching the upstream and carrying ids of its own.""" + answers: Final = tuple(_streams(stream, upstream, identity, EMPTY_STREAMS)) + assert len(answers) == EMPTY_STREAMS and all(answer.reached_upstream for answer in answers), ( + f"a stream was replayed from the cache: {answers}" + ) + distinct: Final = frozenset[str]().union(*(answer.ids for answer in answers)) + assert sum(len(answer.ids) for answer in answers) == len(distinct), answers + return answers + + +def _replayed(stream: Callable[[], _Answer], upstream: Upstream, identity: str, produced: frozenset[str]) -> Streamed: + """The first of up to ``REPLAY_ATTEMPTS`` identical streams served without an upstream call. + + A streamed request can reach the upstream once per proxy worker before a replay is served: the proxy adds + ``stream_options`` to a stream before keying it only when its own router already resolves the model, and the + worker that did not take the ``/model/new`` call resolves it once its router syncs. The replay carries ids an + upstream answer produced. + """ + answers: Final = tuple(_streams(stream, upstream, identity, REPLAY_ATTEMPTS)) + replay: Final = answers[-1] + assert not replay.reached_upstream, ( + f"no stream was replayed from the cache in {REPLAY_ATTEMPTS} attempts: {answers}" + ) + seen: Final = produced.union(*(answer.ids for answer in answers[:-1])) + assert replay.ids <= seen, (replay.ids, seen) + return replay + + +def test_chat_empty_choices_reach_upstream_again_and_the_real_answer_is_cached(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario, _openai(gateway) as client: + handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices"))) + model: Final = scenario.model(api_base=handle.api_base()) + empty: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + assert empty.choices == [], empty.model_dump_json() + _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1) + rescript(handle, json_response(CHAT)) + real: Final = client.chat.completions.create(model=model, messages=[{"role": "user", "content": text}]) + assert real.id != empty.id and real.choices[0].message.content == "scripted", real.model_dump_json() + _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2) + await_entry(store, real.id) + hit: Final = client.chat.completions.with_raw_response.create( + model=model, messages=[{"role": "user", "content": text}] + ) + served: Final = hit.parse() + assert served.id == real.id and served.choices[0].message.content == "scripted", hit.text + assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers) + assert upstream.calls(handle.scenario_id) == 2 + + +def test_async_chat_spend_rows_record_the_empty_answer_and_the_refill_as_misses(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices"))) + model: Final = scenario.model(api_base=handle.api_base()) + key: Final = scenario.key(models=[model]) + + async def drive() -> tuple[str, str, str]: + async with _async_openai(gateway, key) as client: + messages: Final = [{"role": "user", "content": text}] + empty = await client.chat.completions.create(model=model, messages=messages) + assert empty.choices == [], empty.model_dump_json() + rescript(handle, json_response(CHAT)) + real = await client.chat.completions.create(model=model, messages=messages) + assert real.choices[0].message.content == "scripted", real.model_dump_json() + await_entry(store, real.id) + served = await client.chat.completions.create(model=model, messages=messages) + return empty.id, real.id, served.id + + empty_id, real_id, served_id = asyncio.run(drive()) + assert served_id == real_id != empty_id + assert upstream.calls(handle.scenario_id) == 2 + rows: Final = eventually(lambda: _spend_rows(key), lambda found: len(found) == 3, seconds=70) + by_id: Final = {string_value(row["request_id"]): row for row in rows} + assert set(by_id) >= {empty_id, real_id}, sorted(by_id) + assert by_id[empty_id]["cache_hit"] != "True" and by_id[real_id]["cache_hit"] != "True", rows + hit_row: Final = next(row for request_id, row in by_id.items() if request_id.startswith(f"{real_id}_cache_hit")) + assert hit_row["cache_hit"] == "True" and float(str(hit_row["spend"])) == 0, rows + + +def test_text_completion_empty_choices_reach_upstream_again(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(emptied(TEXT, "choices"))) + model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base()) + body: Final[dict[str, JsonValue]] = {"model": model, "prompt": prompt()} + empty: Final = gateway.request("POST", "/v1/completions", body) + assert choices(empty) == [], empty.text + assert upstream.calls(handle.scenario_id) == 1 + rescript(handle, json_response(TEXT)) + real: Final = gateway.request("POST", "/v1/completions", body) + assert text_of_first_choice(real) == "scripted" and response_id(real) != response_id(empty), real.text + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, response_id(real)) + hit: Final = gateway.request("POST", "/v1/completions", body) + assert response_id(hit) == response_id(real) and text_of_first_choice(hit) == "scripted", hit.text + assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers) + assert upstream.calls(handle.scenario_id) == 2 + + +def test_responses_empty_output_reaches_upstream_again(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(emptied(RESPONSE, "output"))) + model: Final = scenario.model(api_base=handle.api_base()) + + async def drive() -> None: + async with _async_openai(gateway) as client: + empty = await client.responses.create(model=model, input=text) + assert empty.output == [], empty.model_dump_json() + assert upstream.calls(handle.scenario_id) == 1 + rescript(handle, json_response(RESPONSE)) + real = await client.responses.create(model=model, input=text) + assert real.output_text == "scripted", real.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, real.output[0].id) + hit = await client.responses.with_raw_response.create(model=model, input=text) + served = hit.parse() + assert served.output[0].id == real.output[0].id and served.output_text == "scripted", hit.text + assert hit.headers.get("x-litellm-cache-key"), dict(hit.headers) + assert upstream.calls(handle.scenario_id) == 2 + + asyncio.run(drive()) + + +def test_messages_empty_content_reaches_upstream_again(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario, _anthropic(gateway) as client: + handle: Final = scripted(scenario, json_response(emptied(MESSAGE, "content"))) + model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base()) + messages: Final = [{"role": "user", "content": text}] + empty: Final = client.messages.create(model=model, max_tokens=16, messages=messages) + assert empty.content == [], empty.model_dump_json() + _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 1) + rescript(handle, json_response(MESSAGE)) + real: Final = client.messages.create(model=model, max_tokens=16, messages=messages) + assert real.id != empty.id and real.content[0].type == "text", real.model_dump_json() + assert real.content[0].text == "scripted" + _assert_prompt_reached_upstream(upstream, handle.scenario_id, text, 2) + await_entry(store, real.id) + served: Final = client.messages.create(model=model, max_tokens=16, messages=messages) + assert served.id == real.id and served.content[0].type == "text", served.model_dump_json() + assert served.content[0].text == "scripted" + assert upstream.calls(handle.scenario_id) == 2 + + +def test_stale_chat_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(CHAT)) + model: Final = scenario.model(api_base=handle.api_base()) + filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL)) + assert content_of_first_choice(filled) == "scripted", filled.text + stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices") + request: Final = chat_body(model, text) + refilled: Final = eventually( + lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS + ) + assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}" + assert content_of_first_choice(refilled) == "scripted", refilled.text + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, response_id(refilled)) + hit: Final = eventually( + lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS + ) + assert content_of_first_choice(hit) == "scripted", hit.text + assert upstream.calls(handle.scenario_id) == 2 + + +def test_stale_text_completion_entry_with_empty_choices_is_a_miss(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(TEXT)) + model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base()) + short_lived: Final[dict[str, JsonValue]] = {"model": model, "prompt": text, "cache": SHORT_TTL_CACHE_CONTROL} + filled: Final = gateway.request("POST", "/v1/completions", short_lived) + assert text_of_first_choice(filled) == "scripted", filled.text + stale_id: Final = flip_entry_to_empty(store, await_entry(store, response_id(filled)), "choices") + request: Final[dict[str, JsonValue]] = {"model": model, "prompt": text} + refilled: Final = eventually( + lambda: gateway.request("POST", "/v1/completions", request), + lambda answer: response_id(answer) != response_id(filled), + SETTLE_SECONDS, + ) + assert response_id(refilled) != stale_id, f"the stale entry was served: {refilled.text}" + assert text_of_first_choice(refilled) == "scripted", refilled.text + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, response_id(refilled)) + hit: Final = eventually( + lambda: gateway.request("POST", "/v1/completions", request), + lambda answer: response_id(answer) == response_id(refilled), + SETTLE_SECONDS, + ) + assert text_of_first_choice(hit) == "scripted", hit.text + assert upstream.calls(handle.scenario_id) == 2 + + +def test_stale_responses_entry_with_empty_output_is_a_miss(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario, _openai(gateway) as client: + handle: Final = scripted(scenario, json_response(RESPONSE)) + model: Final = scenario.model(api_base=handle.api_base()) + filled: Final = client.responses.create(model=model, input=text, extra_body={"cache": SHORT_TTL_CACHE_CONTROL}) + assert filled.output_text == "scripted", filled.model_dump_json() + flip_entry_to_empty(store, await_entry(store, filled.output[0].id), "output") + refilled: Final = eventually( + lambda: client.responses.create(model=model, input=text), + lambda answer: not answer.output or answer.output[0].id != filled.output[0].id, + SETTLE_SECONDS, + ) + assert refilled.output, f"the stale entry was served: {refilled.model_dump_json()}" + assert refilled.output_text == "scripted", refilled.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, refilled.output[0].id) + hit: Final = eventually( + lambda: client.responses.create(model=model, input=text), + lambda answer: bool(answer.output) and answer.output[0].id == refilled.output[0].id, + SETTLE_SECONDS, + ) + assert hit.output_text == "scripted", hit.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + + +def test_stale_messages_entry_with_empty_content_is_a_miss(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario, _anthropic(gateway) as client: + handle: Final = scripted(scenario, json_response(MESSAGE)) + model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base()) + messages: Final = [{"role": "user", "content": text}] + filled: Final = client.messages.create( + model=model, max_tokens=16, messages=messages, extra_body={"cache": SHORT_TTL_CACHE_CONTROL} + ) + assert filled.content[0].type == "text" and filled.content[0].text == "scripted", filled.model_dump_json() + stale_id: Final = flip_entry_to_empty(store, await_entry(store, filled.id), "content") + refilled: Final = eventually( + lambda: client.messages.create(model=model, max_tokens=16, messages=messages), + lambda answer: answer.id != filled.id, + SETTLE_SECONDS, + ) + assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}" + assert refilled.content[0].type == "text" and refilled.content[0].text == "scripted", refilled.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, refilled.id) + hit: Final = eventually( + lambda: client.messages.create(model=model, max_tokens=16, messages=messages), + lambda answer: answer.id == refilled.id, + SETTLE_SECONDS, + ) + assert hit.content[0].type == "text" and hit.content[0].text == "scripted", hit.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + + +def _streamed_chat(gateway: Gateway, model: str, text: str) -> _Answer: + """The chunk ids, the streamed text and the chunk kinds, after the whole stream was read.""" + with ( + _openai(gateway) as client, + client.chat.completions.create( + model=model, messages=[{"role": "user", "content": text}], stream=True + ) as stream, + ): + chunks: Final = tuple(stream) + ids: Final = frozenset(chunk.id for chunk in chunks) + streamed: Final = "".join( + choice.delta.content or "" for chunk in chunks for choice in chunk.choices + ) # comprehension-ok: flattens chunk choices + return ids, streamed, tuple(chunk.object for chunk in chunks) + + +def test_streamed_chat_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, CHAT_STREAM_WITHOUT_CHOICES) + model: Final = scenario.model(api_base=handle.api_base()) + empty: Final = _never_replayed(partial(_streamed_chat, gateway, model, text), upstream, handle.scenario_id) + assert all(answer.text == "" for answer in empty), empty + + +def _streamed_text(gateway: Gateway, model: str, text: str) -> _Answer: + """The chunk ids, the streamed text and the chunk kinds of a text completion, after the whole stream was read.""" + with _openai(gateway) as client, client.completions.create(model=model, prompt=text, stream=True) as stream: + chunks: Final = tuple(stream) + ids: Final = frozenset(chunk.id for chunk in chunks) + streamed: Final = "".join( + choice.text or "" for chunk in chunks for choice in chunk.choices + ) # comprehension-ok: flattens chunk choices + return ids, streamed, tuple(chunk.object for chunk in chunks) + + +def test_streamed_text_completion_without_choices_is_never_assembled_or_cached(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, TEXT_STREAM_WITHOUT_CHOICES) + model: Final = scenario.model(model=TEXT_MODEL, api_base=handle.api_base()) + empty: Final = _never_replayed(partial(_streamed_text, gateway, model, text), upstream, handle.scenario_id) + assert all(answer.text == "" for answer in empty), empty + + +def test_streamed_chat_with_content_is_replayed_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, CHAT_STREAM) + model: Final = scenario.model(api_base=handle.api_base()) + stream: Final = partial(_streamed_chat, gateway, model, text) + first_ids, first_text, _ = stream() + assert first_text == "streamed response" and len(first_ids) == 1, first_ids + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, next(iter(first_ids))) + replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids) + assert replay.text == "streamed response", replay + + +def _streamed_response(gateway: Gateway, model: str, text: str) -> _Answer: + """The completed response's output item ids, its text and the event kinds, after the whole stream was read.""" + + async def read() -> _Answer: + async with ( + _async_openai(gateway) as client, + await client.responses.create(model=model, input=text, stream=True) as stream, + ): + events: Final = [event async for event in stream] + completed: Final = tuple(event for event in events if event.type == "response.completed") + assert len(completed) == 1, [event.type for event in events] + ids: Final = frozenset(item.id for item in completed[0].response.output) + return ids, completed[0].response.output_text, tuple(event.type for event in events) + + return asyncio.run(read()) + + +def test_streamed_responses_completing_without_output_reach_upstream_again(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, RESPONSE_STREAM_WITHOUT_OUTPUT) + model: Final = scenario.model(api_base=handle.api_base()) + stream: Final = partial(_streamed_response, gateway, model, text) + empty: Final = _never_replayed(stream, upstream, handle.scenario_id) + assert all(answer.ids == frozenset() and answer.text == "" for answer in empty), empty + rescript(handle, RESPONSE_STREAM) + real_ids, real_text, _ = stream() + assert real_text == "streamed response", real_ids + assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the empty stream was replayed" + await_entry(store, next(iter(real_ids))) + replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids) + assert replay.text == "streamed response", replay + + +def test_streamed_responses_with_output_are_replayed_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, RESPONSE_STREAM) + model: Final = scenario.model(api_base=handle.api_base()) + stream: Final = partial(_streamed_response, gateway, model, text) + first_ids, first_text, _ = stream() + assert first_text == "streamed response" and len(first_ids) == 1, first_ids + assert all(item.startswith(f"msg_{handle.scenario_id}-") for item in first_ids), first_ids + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, next(iter(first_ids))) + replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids) + assert replay.text == "streamed response", replay + + +def _streamed_message(gateway: Gateway, model: str, text: str) -> _Answer: + """The message id, the streamed text and the event kinds, after the whole stream was read.""" + + async def read() -> _Answer: + async with _async_anthropic(gateway) as client: + stream: Final = await client.messages.create( + model=model, max_tokens=16, messages=[{"role": "user", "content": text}], stream=True + ) + events: Final = [event async for event in stream] + starts: Final = tuple(event for event in events if event.type == "message_start") + assert len(starts) == 1, [event.type for event in events] + deltas: Final = tuple(event for event in events if event.type == "content_block_delta") + streamed: Final = "".join(delta.delta.text for delta in deltas if delta.delta.type == "text_delta") + return frozenset({starts[0].message.id}), streamed, tuple(event.type for event in events) + + return asyncio.run(read()) + + +def test_streamed_messages_without_content_blocks_reach_upstream_again(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, MESSAGE_STREAM_WITHOUT_CONTENT_BLOCKS) + model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base()) + stream: Final = partial(_streamed_message, gateway, model, text) + empty: Final = _never_replayed(stream, upstream, handle.scenario_id) + assert all("content_block_start" not in answer.kinds and answer.text == "" for answer in empty), empty + rescript(handle, MESSAGE_STREAM) + real_ids, real_text, real_kinds = stream() + assert real_text == "streamed response" and "content_block_start" in real_kinds, real_kinds + assert upstream.calls(handle.scenario_id) == EMPTY_STREAMS + 1, "the stream without content blocks was replayed" + await_entry(store, next(iter(real_ids))) + replay: Final = _replayed(stream, upstream, handle.scenario_id, real_ids) + assert replay.text == "streamed response" and replay.kinds == real_kinds, replay + + +def test_streamed_messages_with_content_are_replayed_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, MESSAGE_STREAM) + model: Final = scenario.model(model=MESSAGES_MODEL, api_base=handle.api_base()) + stream: Final = partial(_streamed_message, gateway, model, text) + first_ids, first_text, first_kinds = stream() + assert first_text == "streamed response" and "content_block_start" in first_kinds, first_kinds + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, next(iter(first_ids))) + replay: Final = _replayed(stream, upstream, handle.scenario_id, first_ids) + assert replay.text == "streamed response" and replay.kinds == first_kinds, replay + + +def test_embeddings_are_still_served_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(EMBEDDING)) + model: Final = scenario.model(model="openai/text-embedding-3-small", api_base=handle.api_base()) + body: Final[dict[str, JsonValue]] = {"model": model, "input": [prompt()]} + first: Final = gateway.post("/v1/embeddings", body) + assert first["object"] == "list" and first["data"], first + assert upstream.calls(handle.scenario_id) == 1 + second: Final = eventually( + lambda: _served_without_upstream( + upstream, handle.scenario_id, lambda: gateway.post("/v1/embeddings", body) + ), + lambda answer: answer is not None, + SETTLE_SECONDS, + ) + assert second is not None and second["data"] == first["data"], second + + +@pytest.mark.parametrize( + "choice", + ( + pytest.param( + {"index": 0, "message": {"role": "assistant", "content": ""}, "finish_reason": "content_filter"}, + id="blocked-empty-content", + ), + pytest.param( + { + "index": 0, + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_1", "type": "function", "function": {"name": "lookup", "arguments": "{}"}} + ], + }, + "finish_reason": "tool_calls", + }, + id="tool-calls-only", + ), + ), +) +def test_chat_answers_with_a_choice_but_no_text_are_still_cached( + gateway: Gateway, choice: dict[str, JsonValue] +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response({**CHAT, "choices": [choice]})) + model: Final = scenario.model(api_base=handle.api_base()) + request: Final = chat_body(model, prompt()) + first: Final = _chat(gateway, request) + assert object_value(choices(first)[0])["finish_reason"] == choice["finish_reason"], first.text + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, response_id(first)) + second: Final = _chat(gateway, request) + assert response_id(second) == response_id(first) and choices(second) == choices(first), second.text + assert upstream.calls(handle.scenario_id) == 1 + + +def test_no_cache_requests_reach_upstream_every_time(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(CHAT)) + model: Final = scenario.model(api_base=handle.api_base()) + request: Final = chat_body(model, prompt(), cache={"no-cache": True}) + ids: Final = tuple(response_id(_chat(gateway, request)) for _ in range(3)) + assert len(set(ids)) == 3, ids + assert upstream.calls(handle.scenario_id) == 3 + + +@pytest.mark.parametrize("status", (500, 429)) +def test_upstream_errors_reach_the_caller_and_are_never_cached(gateway: Gateway, status: int) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + handle: Final = scripted( + scenario, json_response({"error": {"message": "Controlled provider failure", "type": "api_error"}}, status) + ) + model: Final = scenario.model(api_base=handle.api_base()) + request: Final = chat_body(model, prompt()) + first: Final = _chat(gateway, request) + assert first.status_code == status and "Controlled provider failure" in first.text, first.text + after_first: Final = upstream.calls(handle.scenario_id) + assert after_first >= 1 + second: Final = _chat(gateway, request) + assert second.status_code == status and "Controlled provider failure" in second.text, second.text + assert upstream.calls(handle.scenario_id) > after_first, "the error was served from the cache" + + +def test_cache_delete_on_the_entry_key_makes_the_next_request_reach_upstream(gateway: Gateway) -> None: + """``/cache/delete`` clears the Redis entry; a worker's in-memory copy of it serves until the entry's ttl lapses.""" + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + text: Final = prompt() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(CHAT)) + model: Final = scenario.model(api_base=handle.api_base()) + filled: Final = _chat(gateway, chat_body(model, text, cache=SHORT_TTL_CACHE_CONTROL)) + key: Final = await_entry(store, response_id(filled)) + assert gateway.post("/cache/delete", {"keys": [key]}) == {"status": "success"} + assert store.get(key) is None + request: Final = chat_body(model, text) + refilled: Final = eventually( + lambda: _chat(gateway, request), lambda answer: response_id(answer) != response_id(filled), SETTLE_SECONDS + ) + assert content_of_first_choice(refilled) == "scripted", refilled.text + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, response_id(refilled)) + hit: Final = eventually( + lambda: _chat(gateway, request), lambda answer: response_id(answer) == response_id(refilled), SETTLE_SECONDS + ) + assert hit.headers.get("x-litellm-cache-key") == key, dict(hit.headers) + assert upstream.calls(handle.scenario_id) == 2 + + +def _shape(body: Mapping[str, JsonValue], field: str, value: object) -> dict[str, JsonValue]: + if value is MISSING: + return {name: item for name, item in body.items() if name != field} + assert isinstance(value, (str, int)) or value is None + return {**body, field: value} + + +def _request_for(path: str, model: str, text: str) -> dict[str, JsonValue]: + if path == "/v1/responses": + return {"model": model, "input": text} + if path == "/v1/messages": + return message_body(model, text) + return chat_body(model, text) + + +@pytest.mark.parametrize( + ("path", "body", "litellm_model", "field", "value"), + ( + pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", None, id="chat-choices-null"), + pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", MISSING, id="chat-choices-missing"), + pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", "x", id="chat-choices-string"), + pytest.param("/v1/chat/completions", CHAT, CHAT_MODEL, "choices", 5, id="chat-choices-int"), + ), +) +def test_malformed_upstream_answers_are_an_error_and_never_cached( + gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(_shape(body, field, value))) + control: Final = scripted(scenario, json_response(CHAT)) + model: Final = scenario.model(model=litellm_model, api_base=handle.api_base()) + control_model: Final = scenario.model(api_base=control.api_base()) + key: Final = scenario.key(models=[model, control_model]) + request: Final = _request_for(path, model, prompt()) + first: Final = gateway.request("POST", path, request, key=key) + assert first.status_code == 500, first.text + assert "error" in first.text.lower(), first.text + assert upstream.calls(handle.scenario_id) == 1 + second: Final = gateway.request("POST", path, request, key=key) + assert second.status_code == 500, second.text + assert upstream.calls(handle.scenario_id) == 2, "the malformed answer was served from the cache" + unrelated: Final = _chat(gateway, chat_body(control_model, prompt()), key=key) + assert content_of_first_choice(unrelated) == "scripted", unrelated.text + + +@pytest.mark.parametrize( + ("path", "body", "litellm_model", "field", "value"), + ( + pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", None, id="responses-output-null"), + pytest.param("/v1/responses", RESPONSE, CHAT_MODEL, "output", MISSING, id="responses-output-missing"), + pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", None, id="messages-content-null"), + pytest.param("/v1/messages", MESSAGE, MESSAGES_MODEL, "content", MISSING, id="messages-content-missing"), + ), +) +def test_answers_without_the_output_field_reach_upstream_again( + gateway: Gateway, path: str, body: Mapping[str, JsonValue], litellm_model: str, field: str, value: object +) -> None: + upstream: Final = Upstream(gateway.upstream_url) + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(_shape(body, field, value))) + model: Final = scenario.model(model=litellm_model, api_base=handle.api_base()) + request: Final = _request_for(path, model, prompt()) + first: Final = gateway.request("POST", path, request) + assert first.status_code == 200, first.text + assert upstream.calls(handle.scenario_id) == 1 + second: Final = gateway.request("POST", path, request) + assert second.status_code == 200, second.text + assert upstream.calls(handle.scenario_id) == 2, "the answer without output was served from the cache" + + +def test_per_request_ttl_on_an_empty_answer_does_not_pin_it(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(emptied(CHAT, "choices"))) + model: Final = scenario.model(api_base=handle.api_base()) + request: Final = chat_body(model, prompt(), cache={"ttl": 30}) + empty: Final = _chat(gateway, request) + assert choices(empty) == [], empty.text + assert upstream.calls(handle.scenario_id) == 1 + rescript(handle, json_response(CHAT)) + real: Final = _chat(gateway, request) + assert content_of_first_choice(real) == "scripted", real.text + assert upstream.calls(handle.scenario_id) == 2, "the empty answer was pinned for the request's ttl" + key: Final = await_entry(store, response_id(real)) + assert 0 < store.ttl(key) <= 30, store.ttl(key) + + +def test_rerank_results_are_still_served_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(RERANK)) + model: Final = scenario.model(model="cohere/rerank-v4.0", api_base=handle.api_base()) + body: Final[dict[str, JsonValue]] = {"model": model, "query": prompt(), "documents": ["first", "second"]} + first: Final = gateway.post("/v1/rerank", body) + assert string_value(first["id"]).startswith(f"rerank-{handle.scenario_id}"), first + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, string_value(first["id"])) + second: Final = gateway.post("/v1/rerank", body) + assert second["id"] == first["id"] and second["results"] == first["results"], second + assert upstream.calls(handle.scenario_id) == 1 + + +def test_transcriptions_are_still_served_from_the_cache(gateway: Gateway) -> None: + upstream: Final = Upstream(gateway.upstream_url) + store: Final = redis_store() + with gateway.scenario() as scenario: + handle: Final = scripted(scenario, json_response(TRANSCRIPT)) + model: Final = scenario.model(model="openai/gpt-4o-mini-transcribe", api_base=handle.api_base()) + audio: Final = (f"{uuid.uuid4().hex}.wav", b"RIFF" + uuid.uuid4().bytes, "audio/wav") + first: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio}) + assert first.status_code == 200, first.text + transcript: Final = string_value(JSON_OBJECT.validate_json(first.content)["text"]) + assert transcript.startswith(f"scripted {handle.scenario_id}-"), first.text + assert upstream.calls(handle.scenario_id) == 1 + await_entry(store, transcript) + second: Final = gateway.request_multipart("/v1/audio/transcriptions", {"model": model}, {"file": audio}) + assert second.status_code == 200 and JSON_OBJECT.validate_json(second.content)["text"] == transcript, ( + second.text + ) + assert upstream.calls(handle.scenario_id) == 1 diff --git a/tests/integration/run.py b/tests/integration/run.py index df408cfc809..f7b2dead197 100644 --- a/tests/integration/run.py +++ b/tests/integration/run.py @@ -13,7 +13,7 @@ from typing import Final GROUPS: Final = MappingProxyType( { "management": ("management", "authorization", "configuration"), - "accounting": ("pricing", "spend"), + "accounting": ("pricing", "spend", "caching"), "database": ("database",), "providers": ("providers", "routing", "streaming", "messages_endpoint", "translation"), "extensions": ("observability", "compatibility"), diff --git a/tests/integration/sdk/test_response_cache_sync_sdk.py b/tests/integration/sdk/test_response_cache_sync_sdk.py new file mode 100644 index 00000000000..54d78adbaba --- /dev/null +++ b/tests/integration/sdk/test_response_cache_sync_sdk.py @@ -0,0 +1,125 @@ +"""The sync SDK paths of the response cache, which the proxy never reaches. + +``litellm.completion`` and ``litellm.text_completion`` read and write the Redis cache in process. An upstream +answer with no choices is never stored, a stored entry whose choices were emptied is a miss, and the real answer +the refill brings back is the one served afterwards. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator +from typing import Final + +import pytest + +import litellm +from litellm.caching.caching import Cache +from litellm.types.caching import LiteLLMCacheType +from litellm.types.utils import ModelResponse, TextCompletionResponse +from tests.integration.caching.response_cache_case import ( + CHAT, + CHAT_MODEL, + PROVIDER_KEY, + TEXT, + TEXT_MODEL, + Upstream, + await_entry, + emptied, + flip_entry_to_empty, + json_response, + prompt, + redis_store, + rescript, + scripted_in_process, +) + + +@pytest.fixture +def sdk_cache(monkeypatch: pytest.MonkeyPatch) -> Iterator[Cache]: + cache: Final = Cache(type=LiteLLMCacheType.REDIS, host=os.environ["REDIS_HOST"], port=os.environ["REDIS_PORT"]) + monkeypatch.setattr(litellm, "cache", cache) + yield cache + monkeypatch.setattr(litellm, "cache", None) + + +def _completion(api_base: str, text: str) -> ModelResponse: + response: Final = litellm.completion( + model=CHAT_MODEL, messages=[{"role": "user", "content": text}], api_base=api_base, api_key=PROVIDER_KEY + ) + assert isinstance(response, ModelResponse), type(response) + return response + + +def _text_completion(api_base: str, text: str) -> TextCompletionResponse: + response: Final = litellm.text_completion(model=TEXT_MODEL, prompt=text, api_base=api_base, api_key=PROVIDER_KEY) + assert isinstance(response, TextCompletionResponse), type(response) + return response + + +def _content(response: ModelResponse) -> object: + assert len(response.choices) == 1, response.model_dump_json() + return response.choices[0].model_dump()["message"]["content"] + + +def _text(response: TextCompletionResponse) -> object: + assert len(response.choices) == 1, response.model_dump_json() + return response.choices[0].model_dump()["text"] + + +@pytest.mark.usefixtures("sdk_cache") +def test_sync_completion_with_empty_choices_reaches_upstream_again_and_the_real_answer_is_cached() -> None: + upstream: Final = Upstream() + text: Final = prompt() + with scripted_in_process(json_response(emptied(CHAT, "choices"))) as handle: + empty: Final = _completion(handle.api_base(), text) + assert empty.choices == [], empty.model_dump_json() + assert upstream.calls(handle.scenario_id) == 1 + rescript(handle, json_response(CHAT)) + refilled: Final = _completion(handle.api_base(), text) + assert _content(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}" + assert refilled.id != empty.id + assert upstream.calls(handle.scenario_id) == 2 + with redis_store() as store: + await_entry(store, refilled.id) + hit: Final = _completion(handle.api_base(), text) + assert hit.id == refilled.id, hit.model_dump_json() + assert _content(hit) == "scripted" + assert upstream.calls(handle.scenario_id) == 2 + + +@pytest.mark.usefixtures("sdk_cache") +def test_sync_completion_treats_a_stale_entry_with_empty_choices_as_a_miss() -> None: + upstream: Final = Upstream() + text: Final = prompt() + with scripted_in_process(json_response(CHAT)) as handle, redis_store() as store: + first: Final = _completion(handle.api_base(), text) + assert _content(first) == "scripted", first.model_dump_json() + stale_id: Final = flip_entry_to_empty(store, await_entry(store, first.id), "choices") + refilled: Final = _completion(handle.api_base(), text) + assert refilled.id != stale_id, f"the stale entry was served: {refilled.model_dump_json()}" + assert _content(refilled) == "scripted" and refilled.id != first.id, refilled.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + await_entry(store, refilled.id) + hit: Final = _completion(handle.api_base(), text) + assert hit.id == refilled.id, hit.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 + + +@pytest.mark.usefixtures("sdk_cache") +def test_sync_text_completion_with_empty_choices_reaches_upstream_again() -> None: + upstream: Final = Upstream() + text: Final = prompt() + with scripted_in_process(json_response(emptied(TEXT, "choices"))) as handle: + empty: Final = _text_completion(handle.api_base(), text) + assert empty.choices == [], empty.model_dump_json() + assert upstream.calls(handle.scenario_id) == 1 + rescript(handle, json_response(TEXT)) + refilled: Final = _text_completion(handle.api_base(), text) + assert _text(refilled) == "scripted", f"the empty answer was served: {refilled.model_dump_json()}" + assert upstream.calls(handle.scenario_id) == 2 + with redis_store() as store: + await_entry(store, refilled.id) + hit: Final = _text_completion(handle.api_base(), text) + assert hit.id == refilled.id, hit.model_dump_json() + assert upstream.calls(handle.scenario_id) == 2 diff --git a/tests/llm_translation/test_llm_response_utils/test_get_headers.py b/tests/llm_translation/test_llm_response_utils/test_get_headers.py index 380f89bbdd4..60a6a4bebcd 100644 --- a/tests/llm_translation/test_llm_response_utils/test_get_headers.py +++ b/tests/llm_translation/test_llm_response_utils/test_get_headers.py @@ -8,6 +8,7 @@ import pytest from litellm.litellm_core_utils.llm_response_utils.get_headers import ( get_response_headers, _get_llm_provider_headers, + get_provider_request_id, ) @@ -72,3 +73,17 @@ def test_get_llm_provider_headers(): } result = _get_llm_provider_headers(input_headers) assert result == expected_output, "Unexpected output for _get_llm_provider_headers" + + +@pytest.mark.parametrize("header", ("request-id", "Request-Id", "x-request-id", "llm_provider-request-id")) +def test_native_clients_receive_the_provider_request_id(header: str) -> None: + result = get_response_headers({header: "req_test", "unrelated": "value"}) + assert result["request-id"] == "req_test" + assert get_provider_request_id(result) == "req_test" + assert result["llm_provider-unrelated"] == "value" + assert "unrelated" not in result + + +@pytest.mark.parametrize("headers", (None, {}, {"request-id": ""}, {"request-id": 42})) +def test_invalid_provider_request_ids_remain_absent(headers: object) -> None: + assert get_provider_request_id(headers) is None diff --git a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py index 491f8651b52..fe47c1a59f3 100644 --- a/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py +++ b/tests/test_litellm/integrations/clickhouse/test_clickhouse_spend_logger.py @@ -163,6 +163,7 @@ def test_success_row_mapping(): assert row["request_id"] == "chatcmpl-abc123" assert row["response_id"] == "chatcmpl-abc123" assert row["litellm_call_id"] == "gateway-call" + assert row["provider_request_id"] == "" assert row["spend"] == 0.00042 assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (20, 10, 30) assert (row["cache_read_tokens"], row["cache_write_tokens"]) == (5, 7) @@ -186,6 +187,18 @@ def test_success_row_mapping(): assert json.loads(row["metadata"])["user_api_key_alias"] == "my-key" +@pytest.mark.parametrize("header_source", ("response_headers", "additional_headers")) +def test_provider_request_id_stays_separate_from_message_and_gateway_ids(header_source: str) -> None: + headers: Final = {"request-id": "req_native"} + hidden: Final = {"additional_headers": headers} if header_source == "additional_headers" else {} + kwargs: Final = {"response_cost": 0.00042, "response_headers": headers if header_source == "response_headers" else None} + payload: Final = cast(StandardLoggingPayload, _payload(id="msg_native", hidden_params=hidden)) + row: Final = spend_log_row_from_payload(payload, kwargs) + assert (row["provider_request_id"], row["response_id"], row["litellm_call_id"]) == ( + "req_native", "msg_native", "gateway-call" + ) + + @pytest.mark.parametrize("status", ("success", "failure")) @pytest.mark.asyncio async def test_custom_request_metadata_is_redacted_before_clickhouse_logging( diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 1599668839a..abbd44384b1 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -23,17 +23,23 @@ from litellm.caching.caching_handler import ( _is_chat_completion_cached_dict, _should_defer_streaming_cache_hit_callbacks, ) +from litellm.caching import DualCache, InMemoryCache from litellm.caching.caching import LiteLLMCacheType +from litellm.types.caching import CACHED_STREAM_EVENTS_KEY from litellm.types.utils import CallTypes from litellm.types.rerank import RerankResponse from litellm.types.utils import ( + Delta, ModelResponse, + ModelResponseStream, + StreamingChoices, EmbeddingResponse, TextCompletionResponse, TranscriptionResponse, Embedding, ) from litellm.types.llms.openai import ResponsesAPIResponse +from collections.abc import Awaitable, Callable from datetime import timedelta, datetime from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper @@ -2313,3 +2319,355 @@ async def test_response_cache_lookup_and_write_declare_the_llm_response_target(m assert seen == {"get": "llm_response", "set": "llm_response"} assert current_service_target() is None + + +def _completion_logging_obj(call_type: str) -> LiteLLMLogging: + return LiteLLMLogging( + litellm_call_id=str(uuid.uuid4()), + call_type=call_type, + model="gpt-3.5-turbo", + messages=[], + function_id=str(uuid.uuid4()), + stream=False, + start_time=_FIXED_START, + ) + + +_FIXED_START = datetime(2026, 1, 1) + + +def _unique_messages() -> list[dict[str, str]]: + return [{"role": "user", "content": f"no choices {uuid.uuid4()}"}] + + +async def aanthropic_messages(**kwargs: object) -> None: + return None + + +def _responses_api_response_without_output() -> ResponsesAPIResponse: + return ResponsesAPIResponse( + id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response", output=[] + ) + + +def _anthropic_message_without_content() -> dict[str, object]: + return { + "id": f"msg_{uuid.uuid4()}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": [], + } + + +def _responses_api_response_missing_output() -> ResponsesAPIResponse: + return ResponsesAPIResponse.model_construct( + id=f"resp_{uuid.uuid4()}", created_at=0, status="incomplete", model="gpt-4o", object="response" + ) + + +def _anthropic_message_with_null_content() -> dict[str, object]: + return { + "id": f"msg_{uuid.uuid4()}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5", + "content": None, + } + + +def _anthropic_message_missing_content() -> dict[str, object]: + return {"id": f"msg_{uuid.uuid4()}", "type": "message", "role": "assistant", "model": "claude-sonnet-5"} + + +def _anthropic_stream_without_content_blocks() -> dict[str, object]: + return { + CACHED_STREAM_EVENTS_KEY: [ + 'event: message_start\ndata: {"type": "message_start", "message": {"id": "msg_1", "content": []}}\n\n', + 'event: message_delta\ndata: {"type": "message_delta", "delta": {"stop_reason": "end_turn"}}\n\n', + 'event: message_stop\ndata: {"type": "message_stop"}\n\n', + ] + } + + +EmptyResult = ModelResponse | TextCompletionResponse | ResponsesAPIResponse | dict[str, object] +EmptyResponseCase = tuple[EmptyResult, Callable[..., Awaitable[object]], str, dict[str, object]] + + +def _empty_response_cases() -> list[EmptyResponseCase]: + return [ + (litellm.ModelResponse(choices=[]), litellm.acompletion, CallTypes.acompletion.value, {"messages": _unique_messages()}), + ( + litellm.TextCompletionResponse(choices=[]), + litellm.atext_completion, + CallTypes.atext_completion.value, + {"prompt": str(uuid.uuid4())}, + ), + ( + _responses_api_response_without_output(), + aresponses, + CallTypes.aresponses.value, + {"input": str(uuid.uuid4())}, + ), + ( + _responses_api_response_missing_output(), + aresponses, + CallTypes.aresponses.value, + {"input": str(uuid.uuid4())}, + ), + ( + _anthropic_message_without_content(), + aanthropic_messages, + CallTypes.aanthropic_messages.value, + {"messages": _unique_messages(), "max_tokens": 16}, + ), + ( + _anthropic_message_with_null_content(), + aanthropic_messages, + CallTypes.aanthropic_messages.value, + {"messages": _unique_messages(), "max_tokens": 16}, + ), + ( + _anthropic_message_missing_content(), + aanthropic_messages, + CallTypes.aanthropic_messages.value, + {"messages": _unique_messages(), "max_tokens": 16}, + ), + ( + _anthropic_stream_without_content_blocks(), + aanthropic_messages, + CallTypes.aanthropic_messages.value, + {"messages": _unique_messages(), "max_tokens": 16, "stream": True}, + ), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases()) +async def test_async_set_cache_skips_response_without_output( + empty_result: EmptyResult, + original_function: Callable[..., Awaitable[object]], + call_type: str, + kwargs: dict[str, object], +): + setup_cache() + handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START) + + await handler.async_set_cache(result=empty_result, original_function=original_function, kwargs=kwargs) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert await litellm.cache.async_get_cache(**kwargs) is None + lookup = await handler._async_get_cache( + model="gpt-3.5-turbo", + original_function=original_function, + logging_obj=_completion_logging_obj(call_type), + start_time=_FIXED_START, + call_type=call_type, + kwargs=kwargs, + ) + assert lookup.cached_result is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("empty_result, original_function, call_type, kwargs", _empty_response_cases()) +async def test_async_get_cache_treats_stored_response_without_output_as_miss( + empty_result: EmptyResult, + original_function: Callable[..., Awaitable[object]], + call_type: str, + kwargs: dict[str, object], +): + setup_cache() + stored = empty_result.model_dump_json() if hasattr(empty_result, "model_dump_json") else empty_result + await litellm.cache.async_add_cache(stored, **kwargs) + assert await litellm.cache.async_get_cache(**kwargs) is not None + + handler = LLMCachingHandler(original_function=original_function, request_kwargs={}, start_time=_FIXED_START) + lookup = await handler._async_get_cache( + model="gpt-3.5-turbo", + original_function=original_function, + logging_obj=_completion_logging_obj(call_type), + start_time=_FIXED_START, + call_type=call_type, + kwargs=kwargs, + ) + assert lookup.cached_result is None + + +@pytest.mark.asyncio +async def test_async_get_cache_heals_stored_completion_without_choices(): + setup_cache() + handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START) + kwargs = {"messages": _unique_messages()} + poisoned = litellm.ModelResponse( + choices=[], usage=litellm.Usage(prompt_tokens=7, completion_tokens=0, total_tokens=7) + ) + await litellm.cache.async_add_cache(poisoned.model_dump_json(), **kwargs) + assert await litellm.cache.async_get_cache(**kwargs) is not None + + async def lookup(): + return await handler._async_get_cache( + model="gpt-3.5-turbo", + original_function=litellm.acompletion, + logging_obj=_completion_logging_obj(CallTypes.acompletion.value), + start_time=_FIXED_START, + call_type=CallTypes.acompletion.value, + kwargs=kwargs, + ) + + assert (await lookup()).cached_result is None + + await handler.async_set_cache(result=chat_completion_response, original_function=litellm.acompletion, kwargs=kwargs) + await asyncio.gather(*_PENDING_CACHE_WRITES) + healed = (await lookup()).cached_result + assert healed is not None + assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content + + +def test_sync_set_cache_skips_response_without_output(): + setup_cache() + handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START) + kwargs = {"messages": _unique_messages()} + + handler.sync_set_cache(result=litellm.ModelResponse(choices=[]), kwargs=kwargs) + + assert litellm.cache.get_cache(**kwargs) is None + lookup = handler._sync_get_cache( + model="gpt-3.5-turbo", + original_function=completion, + logging_obj=_completion_logging_obj(CallTypes.completion.value), + start_time=_FIXED_START, + call_type=CallTypes.completion.value, + kwargs=kwargs, + ) + assert lookup.cached_result is None + + +def test_sync_get_cache_heals_stored_completion_without_choices(): + setup_cache() + handler = LLMCachingHandler(original_function=completion, request_kwargs={}, start_time=_FIXED_START) + kwargs = {"messages": _unique_messages()} + litellm.cache.add_cache(litellm.ModelResponse(choices=[]).model_dump_json(), **kwargs) + assert litellm.cache.get_cache(**kwargs) is not None + + def lookup(): + return handler._sync_get_cache( + model="gpt-3.5-turbo", + original_function=completion, + logging_obj=_completion_logging_obj(CallTypes.completion.value), + start_time=_FIXED_START, + call_type=CallTypes.completion.value, + kwargs=kwargs, + ) + + assert lookup().cached_result is None + + handler.sync_set_cache(result=chat_completion_response, kwargs=kwargs) + healed = lookup().cached_result + assert healed is not None + assert healed.choices[0].message.content == chat_completion_response.choices[0].message.content + + +@pytest.mark.asyncio +async def test_acompletion_after_a_response_without_choices_calls_the_provider_again(): + setup_cache() + messages = _unique_messages() + + first = await litellm.acompletion( + model="gpt-4o", messages=messages, mock_response=litellm.ModelResponse(choices=[]), caching=True + ) + await asyncio.gather(*_PENDING_CACHE_WRITES) + assert first.choices == [] + + second = await litellm.acompletion(model="gpt-4o", messages=messages, mock_response="hi", caching=True) + assert second.choices[0].message.content == "hi" + assert second._hidden_params.get("cache_hit") is not True + + +def _stream_chunk(choices: list[StreamingChoices]) -> ModelResponseStream: + return ModelResponseStream( + id="chatcmpl-stream", created=0, model="gpt-4o", object="chat.completion.chunk", choices=choices + ) + + +def _closing_chunk_without_output() -> ModelResponseStream: + return _stream_chunk([StreamingChoices(finish_reason="stop", index=0, delta=Delta())]) + + +def _cached_content(cached: object) -> object: + assert cached is not None + stored = cached if isinstance(cached, dict) else json.loads(cached) + return stored["choices"][0]["message"]["content"] + + +def _content_chunk(content: str) -> ModelResponseStream: + return _stream_chunk([StreamingChoices(index=0, delta=Delta(content=content, role="assistant"))]) + + +@pytest.mark.asyncio +async def test_async_streamed_answer_without_output_is_never_cached(): + setup_cache() + kwargs = {"model": "gpt-4o", "messages": _unique_messages()} + handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START) + + await handler._add_streaming_response_to_cache(_closing_chunk_without_output()) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert litellm.cache.get_cache(**kwargs) is None + + +@pytest.mark.asyncio +async def test_async_streamed_answer_with_content_is_cached(): + setup_cache() + kwargs = {"model": "gpt-4o", "messages": _unique_messages()} + handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs=kwargs, start_time=_FIXED_START) + + await handler._add_streaming_response_to_cache(_content_chunk("hi")) + await handler._add_streaming_response_to_cache(_closing_chunk_without_output()) + await asyncio.gather(*_PENDING_CACHE_WRITES) + + assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi" + + +def test_sync_streamed_answer_without_output_is_never_cached(): + setup_cache() + kwargs = {"model": "gpt-4o", "messages": _unique_messages()} + handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START) + + handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output()) + + assert litellm.cache.get_cache(**kwargs) is None + + +def test_sync_streamed_answer_with_content_is_cached(): + setup_cache() + kwargs = {"model": "gpt-4o", "messages": _unique_messages()} + handler = LLMCachingHandler(original_function=completion, request_kwargs=kwargs, start_time=_FIXED_START) + + handler._sync_add_streaming_response_to_cache(_content_chunk("hi")) + handler._sync_add_streaming_response_to_cache(_closing_chunk_without_output()) + + assert _cached_content(litellm.cache.get_cache(**kwargs)) == "hi" + + +@pytest.mark.asyncio +async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_without_output(): + setup_cache() + handler = LLMCachingHandler(original_function=litellm.acompletion, request_kwargs={}, start_time=_FIXED_START) + handler.dual_cache = DualCache(in_memory_cache=InMemoryCache()) + kwargs = {"messages": _unique_messages()} + key = litellm.cache.get_cache_key(**kwargs) + poisoned = litellm.ModelResponse(choices=[]).model_dump_json() + await handler.dual_cache.async_set_cache(key, {"timestamp": _FIXED_START.timestamp(), "response": poisoned}) + assert await handler.dual_cache.async_get_cache(key) is not None + + lookup = await handler._async_get_cache( + model="gpt-3.5-turbo", + original_function=litellm.acompletion, + logging_obj=_completion_logging_obj(CallTypes.acompletion.value), + start_time=_FIXED_START, + call_type=CallTypes.acompletion.value, + kwargs=kwargs, + ) + + assert lookup.cached_result is None + assert await handler.dual_cache.async_get_cache(key) is None diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index b80d75adc0b..2d7e0745039 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -3081,6 +3081,126 @@ class TestBackendEndpointParity: assert destination.endpoint == "https://trace.wandb.ai/otel/v1/traces" +class TestArizeOtlpProtocol: + @staticmethod + def _arize(monkeypatch, endpoint_env, **env): + monkeypatch.delenv("ARIZE_ENDPOINT", raising=False) + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + return destination_for("arize", {"arize_space_id": "s", "arize_api_key": "k", **endpoint_env}) + + def test_http_transport_rewrites_the_grpc_endpoint_to_the_traces_path(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://arize.internal.example/v1/traces" + + def test_http_transport_prefers_the_http_endpoint(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces", + ) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://http.arize.internal.example/v1/traces" + + def test_http_transport_falls_back_to_the_arize_cloud_http_endpoint(self, monkeypatch): + destination = self._arize(monkeypatch, {"arize_otlp_protocol": "http/protobuf"}) + + assert destination.protocol == "otlp_http" + assert destination.endpoint == "https://otlp.arize.com/v1/traces" + + def test_grpc_transport_accepts_the_http_endpoint_as_the_grpc_target(self, monkeypatch): + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "grpc"}, + ARIZE_HTTP_ENDPOINT="https://http.arize.internal.example/v1/traces", + ) + + assert destination.protocol == "otlp_grpc" + assert destination.endpoint == "https://http.arize.internal.example/v1/traces" + + def test_no_transport_var_keeps_the_grpc_env_default(self, monkeypatch): + destination = self._arize(monkeypatch, {}, ARIZE_ENDPOINT="https://arize.internal.example") + + assert destination.protocol == "otlp_grpc" + assert destination.endpoint == "https://arize.internal.example" + + def test_the_http_destination_builds_an_http_exporter_on_the_traces_path(self, monkeypatch): + from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( + OTLPSpanExporter as HTTPSpanExporter, + ) + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + from litellm.integrations.otel.plumbing.providers import _destination_processor + + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "http/protobuf"}, + ARIZE_ENDPOINT="https://arize.internal.example/v1", + ) + processor = _destination_processor(destination) + try: + assert isinstance(processor, BatchSpanProcessor) + assert isinstance(processor.span_exporter, HTTPSpanExporter) + assert processor.span_exporter._endpoint == "https://arize.internal.example/v1/traces" + finally: + processor.shutdown() + + def test_the_grpc_destination_builds_a_grpc_exporter(self, monkeypatch): + from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( + OTLPSpanExporter as GRPCSpanExporter, + ) + from opentelemetry.sdk.trace.export import BatchSpanProcessor + + from litellm.integrations.otel.plumbing.providers import _destination_processor + + destination = self._arize( + monkeypatch, + {"arize_otlp_protocol": "grpc"}, + ARIZE_ENDPOINT="https://arize.internal.example", + ) + processor = _destination_processor(destination) + try: + assert isinstance(processor, BatchSpanProcessor) + assert isinstance(processor.span_exporter, GRPCSpanExporter) + finally: + processor.shutdown() + + def test_a_team_metadata_protocol_reaches_the_destination(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1") + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + is_otel_v2_enabled.cache_clear() + auth = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": { + "arize_space_id": "s", + "arize_api_key": "k", + "arize_otlp_protocol": "http/protobuf", + }, + } + ] + } + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.protocol for d in destinations] == ["otlp_http"] + assert destinations[0].endpoint == "https://arize.internal.example/v1/traces" + + class TestIncompleteCredentials: """Half a credential set builds a non-empty but unusable header dict. Accepting it would suppress the operator's exporter and send the trace where it cannot land.""" diff --git a/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py b/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py index b6a7c2ccb93..b7437b5fd8f 100644 --- a/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py +++ b/tests/unit/integrations/websearch_interception/test_websearch_interception_handler.py @@ -13,7 +13,13 @@ from litellm.integrations.websearch_interception.handler import ( WebSearchInterceptionLogger, ) from litellm.llms.base_llm.search.transformation import SearchResponse -from litellm.proxy._types import LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, ProxyException, UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, + LitellmUserRoles, + ProxyException, + UserAPIKeyAuth, +) from litellm.types.utils import LlmProviders @@ -899,3 +905,184 @@ async def test_pre_request_hook_syncs_forced_tool_choice(): "type": "tool", "name": LITELLM_WEB_SEARCH_TOOL_NAME, } + + +def _single_search_tool_router(search_tool_name): + router = MagicMock() + router.search_tools = [ + { + "search_tool_name": search_tool_name, + "litellm_params": {"search_provider": "tavily", "api_key": "fake-ui-key"}, + } + ] + return router + + +def _virtual_key( + team_id: str | None = None, key_search_tools: list[str] | None = None, api_key: str = "sk-caller" +) -> UserAPIKeyAuth: + token = UserAPIKeyAuth( + api_key=api_key, + user_id="user-1", + team_id=team_id, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + token.via_virtual_key = True + return token + + +@pytest.mark.parametrize( + "general_settings, key_search_tools, team_search_tools, expect_search", + [ + ({}, None, [], True), + ({"search_tool_deny_by_default": True}, ["team-search"], [], False), + ({"search_tool_deny_by_default": True}, ["team-search"], None, False), + ({"search_tool_deny_by_default": True}, [], ["team-search"], False), + ({"search_tool_deny_by_default": True}, ["team-search"], ["team-search"], True), + ], +) +@pytest.mark.asyncio +async def test_execute_search_team_key_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + key_search_tools: list[str] | None, + team_search_tools: list[str] | None, + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="team-search") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + team_object = LiteLLM_TeamTable( + team_id="team-1", + object_permission_id=None if team_search_tools is None else "op-team", + object_permission=( + None + if team_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=team_search_tools) + ), + ) + + monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("team-search")) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_object", AsyncMock(return_value=team_object)) + kwargs = {"metadata": {"user_api_key_auth": _virtual_key("team-1", key_search_tools)}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + mock_asearch.assert_awaited_once() + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_execute_search_key_without_grant_is_denied_under_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="any-search") + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", _single_search_tool_router("any-search")) + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs={"metadata": {"user_api_key_auth": _virtual_key()}}) + + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, caller, expect_search", + [ + ({}, _virtual_key(), True), + ({"search_tool_deny_by_default": False}, _virtual_key(), True), + ({"search_tool_deny_by_default": True}, _virtual_key(key_search_tools=["any"]), False), + ( + {"search_tool_deny_by_default": True}, + UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), + False, + ), + ({"search_tool_deny_by_default": True}, _virtual_key(api_key="litellm_proxy_master_key"), True), + ({"search_tool_deny_by_default": True}, None, True), + ], + ids=["omitted", "false", "virtual-key", "proxy-admin-not-exempt", "master-key-exempt", "sdk-without-proxy-auth"], +) +@pytest.mark.asyncio +async def test_execute_search_unregistered_fallback_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + caller: UserAPIKeyAuth | None, + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"]) + router = MagicMock() + router.search_tools = [] + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + kwargs = {"metadata": {} if caller is None else {"user_api_key_auth": caller}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity" + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, expect_search", + [({}, True), ({"search_tool_deny_by_default": True}, False)], +) +@pytest.mark.asyncio +async def test_execute_search_registered_tool_without_provider_follows_unregistered_fallback_policy( + monkeypatch: pytest.MonkeyPatch, + general_settings: dict[str, bool], + expect_search: bool, +): + import litellm + from litellm.proxy import proxy_server + + logger = WebSearchInterceptionLogger(enabled_providers=["bedrock"], search_tool_name="no-provider") + router = MagicMock() + router.search_tools = [{"search_tool_name": "no-provider", "litellm_params": {}}] + mock_asearch = AsyncMock(return_value=SearchResponse(object="search", results=[])) + + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(litellm, "asearch", mock_asearch) + kwargs = {"metadata": {"user_api_key_auth": _virtual_key(key_search_tools=["no-provider"])}} + + if expect_search: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert mock_asearch.await_args.kwargs["search_provider"] == "perplexity" + else: + with pytest.raises(ProxyException) as exc_info: + await logger._execute_search("what is litellm", kwargs=kwargs) + assert exc_info.value.code == "403" + mock_asearch.assert_not_awaited() diff --git a/tests/unit/litellm_core_utils/test_core_helpers.py b/tests/unit/litellm_core_utils/test_core_helpers.py index 2a6dd347d5f..9f48ea62a72 100644 --- a/tests/unit/litellm_core_utils/test_core_helpers.py +++ b/tests/unit/litellm_core_utils/test_core_helpers.py @@ -1,6 +1,7 @@ """Tests for litellm_core_utils.core_helpers module.""" import logging +from typing import Final import httpx import pytest @@ -16,6 +17,7 @@ from litellm.litellm_core_utils.core_helpers import ( get_provider_response_headers_from_hidden_params, map_finish_reason, normalize_drop_params, + process_response_headers, reconstruct_model_name, redact_nested_match_and_regex_keys, set_provider_response_headers_in_hidden_params, @@ -25,6 +27,14 @@ from litellm.proxy._types import UserAPIKeyAuth from litellm.types.utils import ImageResponse, TranscriptionResponse +@pytest.mark.parametrize("header", ("request-id", "x-request-id", "llm_provider-request-id")) +def test_native_request_id_survives_stream_header_processing(header: str) -> None: + processed: Final = process_response_headers(httpx.Headers({header: "req_native"})) + + assert processed["request-id"] == "req_native" + assert processed[header if header.startswith("llm_provider-") else "llm_provider-" + header] == "req_native" + + class TestBudgetReservationBinding: """The request-end release skips a reservation a cost callback has claimed, so the claim must land on the one dict auth stamped, through whichever metadata field or auth object diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py b/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py index a8a8eba0bf7..8690d2ee68f 100644 --- a/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py +++ b/tests/unit/llms/anthropic/pass_through/messages/test_response_cache.py @@ -160,6 +160,26 @@ async def test_streaming_cache_is_not_shared_with_non_streaming(local_cache, req assert non_streaming["content"][0]["text"] == "ALPHA" +@pytest.mark.asyncio +async def test_stream_without_content_blocks_is_not_cached(local_cache, request_kwargs, monkeypatch): + empty_events = [STREAM_EVENTS[0], STREAM_EVENTS[4], STREAM_EVENTS[5]] + fake_handler = _CountingHandler( + [_byte_stream(empty_events), _byte_stream(STREAM_EVENTS), _byte_stream([b"event: never_used\n\n"])] + ) + monkeypatch.setattr(handler, "anthropic_messages_handler", fake_handler) + + empty = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + await asyncio.sleep(0) + refilled = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + await asyncio.sleep(0) + replayed = await _collect(await litellm.anthropic_messages(**request_kwargs, stream=True)) + + assert empty == empty_events + assert refilled == STREAM_EVENTS + assert replayed == STREAM_EVENTS + assert len(fake_handler.calls) == 2 + + @pytest.mark.asyncio async def test_failed_stream_is_not_cached(local_cache, request_kwargs, monkeypatch): error_events = STREAM_EVENTS[:3] + [ diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 93d18727f22..2d59ab337d4 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -895,6 +895,7 @@ async def test_async_anthropic_messages_handler_streaming_forwards_provider_resp additional_headers = result._hidden_params["additional_headers"] assert additional_headers["llm_provider-x-amzn-requestid"] == "amzn-req-123" assert additional_headers["llm_provider-x-amzn-trace-id"] == "Root=1-abc-def" + assert mock_logging_obj.model_call_details["response_headers"] == dict(upstream_response.headers) collected = b"".join([chunk async for chunk in result]) assert b"message_start" in collected @@ -1601,8 +1602,7 @@ async def test_async_anthropic_messages_handler_forwards_request_timeout(monkeyp expected_response = {"id": "msg_1", "content": []} mock_config.transform_anthropic_messages_response = Mock(return_value=expected_response) - ok_response = Mock() - ok_response.raise_for_status = Mock(return_value=None) + ok_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) mock_client = AsyncMock(spec=AsyncHTTPHandler) mock_client.post = AsyncMock(return_value=ok_response) @@ -2072,8 +2072,7 @@ async def test_async_anthropic_messages_handler_passes_api_key_to_agentic_hooks( captured_kwargs.update(call_kwargs) return sentinel_response - mock_httpx_response = Mock() - mock_httpx_response.status_code = 200 + mock_httpx_response = httpx.Response(200, request=httpx.Request("POST", "https://api.anthropic.com/v1/messages")) with ( patch.object( diff --git a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py index 9ce23c442f6..e855d8cd346 100644 --- a/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py +++ b/tests/unit/proxy/auth/test_auth_checks_object_access_and_lookup.py @@ -4,7 +4,7 @@ import json import re import sys import time -from collections.abc import Iterator, Mapping +from collections.abc import Awaitable, Iterator, Mapping from types import SimpleNamespace from typing import TYPE_CHECKING, Final, Literal, Optional from unittest.mock import AsyncMock, MagicMock, patch @@ -61,6 +61,7 @@ from litellm.proxy.auth.auth_checks import ( _check_agent_caller_model_access, _virtual_key_max_budget_check, _virtual_key_soft_budget_check, + common_checks, get_key_object, get_user_object, invalidate_team_member_spend_state, @@ -74,6 +75,7 @@ from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting from litellm.constants import ( DEFAULT_MANAGEMENT_OBJECT_IN_MEMORY_CACHE_TTL, END_USER_RESTRICTED_REGISTRY_MAX_SIZE, + LITELLM_PROXY_MASTER_KEY_ALIAS, PROXY_DB_LOOKUP_MAX_CONCURRENCY, REGISTRY_ERROR_NEGATIVE_CACHE_TTL, TAG_REGISTRY_MAX_SIZE, @@ -90,6 +92,7 @@ from litellm.proxy.common_utils.user_api_key_cache import ( UserApiKeyCache, end_user_cache_key, end_user_restricted_registry_cache_key, + object_permission_cache_key, tag_cache_key, tag_registry_cache_key, ) @@ -1646,8 +1649,9 @@ async def test_vector_store_access_check_with_permissions(): ) mock_prisma_client = MagicMock() - mock_permissions = MagicMock() - mock_permissions.vector_stores = ["store-1", "store-2"] + mock_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="perm-123", vector_stores=["store-1", "store-2"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=mock_permissions) mock_vector_store_registry = MagicMock() @@ -1688,12 +1692,12 @@ async def test_vector_store_access_check_with_team_permissions(): request_body = {} valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None) - team_object = MagicMock() - team_object.object_permission_id = "team-permission" + team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission") mock_prisma_client = MagicMock() - team_permissions = MagicMock() - team_permissions.vector_stores = ["team-store-allowed"] + team_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="team-permission", vector_stores=["team-store-allowed"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions) mock_vector_store_registry = MagicMock() @@ -1755,12 +1759,12 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query( } valid_token = UserAPIKeyAuth(token="team-test-token", object_permission_id=None) - team_object = MagicMock() - team_object.object_permission_id = "team-permission" + team_object = LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission") mock_prisma_client = MagicMock() - team_permissions = MagicMock() - team_permissions.vector_stores = ["KBALLOWED123"] + team_permissions = LiteLLM_ObjectPermissionTable( + object_permission_id="team-permission", vector_stores=["KBALLOWED123"] + ) mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock(return_value=team_permissions) with ( @@ -1786,6 +1790,592 @@ async def test_vector_store_access_check_enforces_team_allowlist_for_rag_query( assert exc_info.value.type == expected_error_type +_KEY_DENIED: Final = ProxyErrorTypes.key_vector_store_access_denied +_TEAM_DENIED: Final = ProxyErrorTypes.team_vector_store_access_denied +_USER_DENIED: Final = ProxyErrorTypes.user_vector_store_access_denied + + +def _virtual_key( + object_permission_id: str | None = None, + api_key: str = "sk-standalone", + user_role: LitellmUserRoles | None = None, + team_id: str | None = None, +) -> UserAPIKeyAuth: + key: Final = UserAPIKeyAuth( + api_key=api_key, + user_id="key-owner", + user_role=user_role, + team_id=team_id, + object_permission_id=object_permission_id, + ) + key.via_virtual_key = True + return key + + +def _permission_row( + object_permission_id: str, permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None +) -> LiteLLM_ObjectPermissionTable | None: + if isinstance(permission, SimpleNamespace): + return LiteLLM_ObjectPermissionTable( + object_permission_id=object_permission_id, vector_stores=permission.vector_stores + ) + return permission + + +async def _common_checks_for_rag_query( + general_settings: Mapping[str, object], + valid_token: UserAPIKeyAuth, + key_permission: SimpleNamespace | None, + team_object: LiteLLM_TeamTable | None = None, + team_permission: SimpleNamespace | None = None, + user_object: LiteLLM_UserTable | None = None, + user_permission: SimpleNamespace | LiteLLM_ObjectPermissionTable | None = None, + request_body: Mapping[str, object] | None = None, + user_api_key_cache: UserApiKeyCache | None = None, + vector_store_registry: VectorStoreRegistry | None = None, +) -> bool: + permissions: Final = { + "key-permission": _permission_row("key-permission", key_permission), + "team-permission": _permission_row("team-permission", team_permission), + "user-permission": _permission_row("user-permission", user_permission), + } + mock_prisma_client: Final = MagicMock() + mock_prisma_client.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: permissions.get(where["object_permission_id"]) + ) + mock_prisma_client.db.litellm_teammembership.find_unique = AsyncMock(return_value=None) + body: Final = ( + dict(request_body) + if request_body is not None + else { + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + } + ) + with ( + patch( # test-quality-ok: production auth reads these module globals; no dependency injection seam exists + "litellm.proxy.proxy_server.prisma_client", mock_prisma_client + ), + patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists + "litellm.vector_store_registry", vector_store_registry + ), + patch( # test-quality-ok: production auth reads this module global; no dependency injection seam exists + "litellm.proxy.proxy_server.user_api_key_cache", + UserApiKeyCache() if user_api_key_cache is None else user_api_key_cache, + ), + ): + return await common_checks( + request_body=body, + team_object=team_object, + user_object=user_object, + end_user_object=None, + global_proxy_spend=None, + general_settings=dict(general_settings), + route="/v1/rag/query", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=valid_token, + request=MagicMock(spec=Request), + ) + + +async def _assert_rag_query_outcome(request: Awaitable[bool], denied_by: ProxyErrorTypes | None) -> None: + if denied_by is None: + assert await request is True + return + with pytest.raises(ProxyException) as exc_info: + await request + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == (denied_by, "vector_store", "401") + + +async def _team_key_rag_query( + general_settings: Mapping[str, object], + key_vector_stores: list[str] | None, + team_vector_stores: list[str] | None, + user_object: LiteLLM_UserTable | None = None, +) -> bool: + return await _common_checks_for_rag_query( + general_settings, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission", team_id="team-1"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + LiteLLM_TeamTable( + team_id="team-1", object_permission_id=None if team_vector_stores is None else "team-permission" + ), + None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores), + user_object, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "denied_by"), + [ + ({}, None), + ({"vector_store_deny_by_default": False}, None), + ({"vector_store_deny_by_default": True}, _KEY_DENIED), + ], + ids=["flag-omitted", "flag-false", "flag-true"], +) +async def test_standalone_key_without_vector_store_permission_follows_deny_by_default( + general_settings: Mapping[str, object], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome(_common_checks_for_rag_query(general_settings, _virtual_key(), None), denied_by) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "vector_stores", "denied_by"), + [ + (True, [], _KEY_DENIED), + (True, ["KBSTOREA"], None), + (True, ["KBSTOREB"], _KEY_DENIED), + (True, None, _KEY_DENIED), + (False, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["enabled-empty", "enabled-contains", "enabled-excludes", "enabled-null", "disabled-excludes"], +) +async def test_standalone_key_vector_store_permission_record_under_deny_by_default( + deny_by_default: bool, vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=vector_stores), + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [False, True], ids=["flag-false", "flag-true"]) +async def test_master_key_rag_query_is_unchanged_by_deny_by_default(deny_by_default: bool): + master_key: Final = _virtual_key(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query({"vector_store_deny_by_default": deny_by_default}, master_key, None), None + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "vector_stores", "denied_by"), + [ + ({}, None, None), + ({"vector_store_deny_by_default": False}, None, None), + ({"vector_store_deny_by_default": False}, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["flag-omitted-no-record", "flag-false-no-record", "flag-false-excludes"], +) +async def test_proxy_admin_virtual_key_keeps_flag_off_vector_store_behavior( + general_settings: Mapping[str, object], vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + admin_key: Final = _virtual_key( + object_permission_id=None if vector_stores is None else "key-permission", user_role=LitellmUserRoles.PROXY_ADMIN + ) + key_permission: Final = None if vector_stores is None else SimpleNamespace(vector_stores=vector_stores) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query(general_settings, admin_key, key_permission), denied_by + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("key_vector_stores", "team_vector_stores", "denied_by"), + [ + (["KBSTOREA"], ["KBSTOREA"], None), + (None, ["KBSTOREA"], _KEY_DENIED), + ([], ["KBSTOREA"], _KEY_DENIED), + (["KBSTOREA"], None, _TEAM_DENIED), + (["KBSTOREA"], [], _TEAM_DENIED), + (["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED), + (["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED), + ], + ids=["both-grant", "key-no-record", "key-empty", "team-no-record", "team-empty", "key-excludes", "team-excludes"], +) +async def test_team_key_requires_key_and_team_grant_under_deny_by_default( + key_vector_stores: list[str] | None, team_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _team_key_rag_query({"vector_store_deny_by_default": True}, key_vector_stores, team_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "key_vector_stores", "team_vector_stores", "denied_by"), + [ + ({}, [], ["KBSTOREA"], None), + ({"vector_store_deny_by_default": False}, [], ["KBSTOREA"], None), + ({"vector_store_deny_by_default": False}, None, None, None), + ({"vector_store_deny_by_default": False}, ["KBSTOREB"], ["KBSTOREA"], _KEY_DENIED), + ({"vector_store_deny_by_default": False}, ["KBSTOREA"], ["KBSTOREB"], _TEAM_DENIED), + ], + ids=["omitted-key-empty", "false-key-empty", "false-no-records", "false-key-excludes", "false-team-excludes"], +) +async def test_team_key_keeps_legacy_vector_store_behavior_when_flag_off( + general_settings: Mapping[str, object], + key_vector_stores: list[str] | None, + team_vector_stores: list[str] | None, + denied_by: ProxyErrorTypes | None, +): + await _assert_rag_query_outcome( + _team_key_rag_query(general_settings, key_vector_stores, team_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +async def test_user_owned_team_key_without_key_grant_is_denied_despite_team_membership(): + team_member: Final = LiteLLM_UserTable( + user_id="key-owner", teams=["team-1"], user_role=LitellmUserRoles.INTERNAL_USER + ) + + await _assert_rag_query_outcome( + _team_key_rag_query({"vector_store_deny_by_default": True}, [], ["KBSTOREA"], team_member), _KEY_DENIED + ) + + +def _user_row(user_vector_stores: list[str] | None, teams: tuple[str, ...] = ()) -> LiteLLM_UserTable: + return LiteLLM_UserTable( + user_id="user-1", + teams=list(teams), + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission_id=None if user_vector_stores is None else "user-permission", + ) + + +async def _keyless_rag_query( + general_settings: Mapping[str, object], + user_vector_stores: list[str] | None, + team_vector_stores: list[str] | None = None, + team_id: str | None = None, + user_permission: LiteLLM_ObjectPermissionTable | None = None, +) -> bool: + return await _common_checks_for_rag_query( + general_settings, + UserAPIKeyAuth(user_id="user-1", team_id=team_id, user_role=LitellmUserRoles.INTERNAL_USER), + None, + None + if team_id is None + else LiteLLM_TeamTable( + team_id=team_id, object_permission_id=None if team_vector_stores is None else "team-permission" + ), + None if team_vector_stores is None else SimpleNamespace(vector_stores=team_vector_stores), + _user_row(user_vector_stores, teams=() if team_id is None else (team_id,)), + user_permission or (None if user_vector_stores is None else SimpleNamespace(vector_stores=user_vector_stores)), + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_vector_stores", "denied_by"), + [(["KBSTOREA"], None), (["KBSTOREB"], _USER_DENIED), ([], _USER_DENIED), (None, _USER_DENIED)], + ids=["user-grants", "user-excludes", "user-empty", "user-no-record"], +) +async def test_keyless_user_without_team_needs_user_grant_under_deny_by_default( + user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores), denied_by + ) + + +@pytest.mark.asyncio +async def test_keyless_user_null_vector_store_list_grants_nothing_under_deny_by_default(): + null_list: Final = LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=None) + + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, [], user_permission=null_list), _USER_DENIED + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("general_settings", "user_vector_stores"), + [({}, ["KBSTOREB"]), ({"vector_store_deny_by_default": False}, ["KBSTOREB"]), ({}, None)], + ids=["omitted-user-excludes", "false-user-excludes", "omitted-user-no-record"], +) +async def test_keyless_user_keeps_legacy_vector_store_behavior_when_flag_off( + general_settings: Mapping[str, object], user_vector_stores: list[str] | None +): + await _assert_rag_query_outcome(_keyless_rag_query(general_settings, user_vector_stores), None) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_vector_stores", "user_vector_stores", "denied_by"), + [ + (["KBSTOREA"], [], None), + (["KBSTOREB"], ["KBSTOREA"], _TEAM_DENIED), + ([], ["KBSTOREA"], _TEAM_DENIED), + (None, ["KBSTOREA"], _TEAM_DENIED), + ], + ids=["team-grants-user-empty", "team-excludes-user-grants", "team-empty-user-grants", "team-no-record-user-grants"], +) +async def test_keyless_team_member_needs_only_team_grant_under_deny_by_default( + team_vector_stores: list[str] | None, user_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _keyless_rag_query({"vector_store_deny_by_default": True}, user_vector_stores, team_vector_stores, "team-1"), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_id", "denied_by"), + [("team-1", None), (None, None)], + ids=["session-on-team-uses-team-grant", "session-without-team-uses-user-grant"], +) +async def test_session_token_is_not_a_virtual_key_under_deny_by_default( + team_id: str | None, denied_by: ProxyErrorTypes | None +): + session: Final = UserAPIKeyAuth( + api_key="hashed-session-token", + user_id="user-1", + team_id=team_id, + user_role=LitellmUserRoles.INTERNAL_USER, + is_session_token=True, + ) + session.via_virtual_key = True + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + session, + None, + None if team_id is None else LiteLLM_TeamTable(team_id=team_id, object_permission_id="team-permission"), + SimpleNamespace(vector_stores=["KBSTOREA"]), + _user_row(["KBSTOREA"] if team_id is None else []), + SimpleNamespace(vector_stores=["KBSTOREA"] if team_id is None else []), + ), + denied_by, + ) + + +@pytest.mark.asyncio +async def test_user_owned_standalone_key_cannot_use_owner_grants_under_deny_by_default(): + owner: Final = LiteLLM_UserTable( + user_id="key-owner", user_role=LitellmUserRoles.INTERNAL_USER, object_permission_id="user-permission" + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(), + None, + user_object=owner, + user_permission=SimpleNamespace(vector_stores=["KBSTOREA"]), + ), + _KEY_DENIED, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "key_vector_stores", "denied_by"), + [ + (True, ["KBSTOREA", "KBSTOREB"], None), + (True, ["KBSTOREA"], _KEY_DENIED), + (True, ["KBSTOREB"], _KEY_DENIED), + (False, ["KBSTOREA"], _KEY_DENIED), + ], + ids=["enabled-grants-both", "enabled-missing-second", "enabled-missing-first", "disabled-missing-second"], +) +async def test_every_requested_vector_store_needs_a_grant( + deny_by_default: bool, key_vector_stores: list[str], denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=key_vector_stores), + request_body={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in these KBs?"}], + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREB"]}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + vector_store_registry=VectorStoreRegistry(), + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "valid_token", + [_virtual_key(), UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER)], + ids=["standalone-key", "keyless-user"], +) +async def test_request_without_vector_stores_is_unaffected_by_deny_by_default(valid_token: UserAPIKeyAuth): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + valid_token, + None, + user_object=_user_row(None), + request_body={"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, + vector_store_registry=VectorStoreRegistry(), + ), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_body", + [ + {"query": "what is in this KB?", "vector_store_id": "KBSTOREA", "vector_store_ids": ["KBSTOREA"]}, + { + "model": "gpt-4o-mini", + "input": "what is in this KB?", + "tools": [{"type": "file_search", "vector_store_ids": ["KBSTOREA"]}], + }, + ], + ids=["vector-store-search-route", "responses-file-search"], +) +@pytest.mark.parametrize( + ("deny_by_default", "key_vector_stores", "denied_by"), + [(True, None, _KEY_DENIED), (True, ["KBSTOREA"], None), (False, None, None)], + ids=["enabled-no-grant", "enabled-grant", "disabled-no-grant"], +) +async def test_deny_by_default_reads_requested_vector_stores_without_a_registry( + request_body: Mapping[str, object], + deny_by_default: bool, + key_vector_stores: list[str] | None, + denied_by: ProxyErrorTypes | None, +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + request_body=request_body, + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "key_vector_stores", "denied_by"), + [ + ({"tools": [1]}, None, None), + ({"tools": 1}, None, None), + ({"tools": [{"type": "file_search", "vector_store_ids": None}]}, None, None), + ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREA"], None), + ({"tools": [1, {"type": "file_search", "vector_store_ids": ["KBSTOREA"]}]}, ["KBSTOREB"], _KEY_DENIED), + ], + ids=["int-tool", "non-list-tools", "null-tool-ids", "int-tool-beside-granted", "int-tool-beside-ungranted"], +) +async def test_deny_by_default_ignores_tools_that_name_no_vector_store( + request_body: Mapping[str, object], key_vector_stores: list[str] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(object_permission_id=None if key_vector_stores is None else "key-permission"), + None if key_vector_stores is None else SimpleNamespace(vector_stores=key_vector_stores), + request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body}, + ), + denied_by, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_body", + [ + {"tools": [{"type": "file_search", "vector_store_ids": "KBSTOREA"}]}, + {"tools": [{"type": "file_search", "vector_store_ids": [1]}]}, + {"vector_store_ids": "KBSTOREA"}, + ], + ids=["string-tool-ids", "int-tool-id", "string-top-level-ids"], +) +async def test_deny_by_default_rejects_malformed_vector_store_ids_as_bad_request(request_body: Mapping[str, object]): + with pytest.raises(ProxyException) as exc_info: + await _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + _virtual_key(object_permission_id="key-permission"), + SimpleNamespace(vector_stores=["KBSTOREA"]), + request_body={"model": "gpt-4o-mini", "input": "what is in this KB?", **request_body}, + ) + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + "invalid_request_error", + "vector_store_ids", + "400", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("request_body", "denied_by"), + [ + ({"model": "gpt-4o-mini", "messages": [{"role": "user", "content": "hello"}]}, None), + (None, _KEY_DENIED), + ], + ids=["no-vector-store", "ungranted-vector-store"], +) +@pytest.mark.parametrize("flag_value", [None, "enabled"], ids=["null", "string"]) +async def test_invalid_deny_by_default_value_only_denies_vector_store_requests( + flag_value: object, request_body: Mapping[str, object] | None, denied_by: ProxyErrorTypes | None +): + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": flag_value}, _virtual_key(), None, request_body=request_body + ), + denied_by, + ) + + +@pytest.mark.asyncio +async def test_keyless_user_grant_is_read_through_the_object_permission_cache(): + cache: Final = UserApiKeyCache(default_in_memory_ttl=60) + await cache.async_set_cache( + key=object_permission_cache_key("user-permission"), + value=LiteLLM_ObjectPermissionTable(object_permission_id="user-permission", vector_stores=["KBSTOREA"]), + model_type=LiteLLM_ObjectPermissionTable, + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": True}, + UserAPIKeyAuth(user_id="user-1", user_role=LitellmUserRoles.INTERNAL_USER), + None, + user_object=_user_row(["KBSTOREA"]), + user_api_key_cache=cache, + ), + None, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deny_by_default", [True, False], ids=["strict", "default"]) +async def test_key_and_team_grants_are_read_through_the_object_permission_cache(deny_by_default: bool): + cache: Final = UserApiKeyCache(default_in_memory_ttl=60) + for object_permission_id, vector_stores in (("key-permission", ["KBSTOREA"]), ("team-permission", ["KBSTOREB"])): + await cache.async_set_cache( + key=object_permission_cache_key(object_permission_id), + value=LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, vector_stores=vector_stores), + model_type=LiteLLM_ObjectPermissionTable, + ) + + await _assert_rag_query_outcome( + _common_checks_for_rag_query( + {"vector_store_deny_by_default": deny_by_default}, + _virtual_key(object_permission_id="key-permission", team_id="team-1"), + None, + LiteLLM_TeamTable(team_id="team-1", object_permission_id="team-permission"), + None, + user_api_key_cache=cache, + ), + _TEAM_DENIED, + ) + + def test_can_object_call_model_with_alias(): """Test that can_object_call_model works with model aliases""" from litellm import Router @@ -10346,7 +10936,9 @@ async def test_authoritative_access_group_reads_writer_despite_stale_allow_cache from litellm.proxy._types import LiteLLM_AccessGroupTable from litellm.proxy.auth.auth_checks import get_access_object - stale: Final = LiteLLM_AccessGroupTable(access_group_id="group", access_group_name="Policy", access_model_names=["old"]) + stale: Final = LiteLLM_AccessGroupTable( + access_group_id="group", access_group_name="Policy", access_model_names=["old"] + ) current: Final = stale.model_copy(update={"access_model_names": ["new"] if allowed else []}) client: Final = MagicMock() client.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(return_value=current) @@ -10498,9 +11090,12 @@ async def test_authoritative_key_cannot_keep_grants_when_permission_is_unavailab from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache database: Final = MagicMock() - database.get_data = AsyncMock(return_value=UserAPIKeyAuth( - object_permission_id="grant", object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"]) - )) + database.get_data = AsyncMock( + return_value=UserAPIKeyAuth( + object_permission_id="grant", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="grant", agents=["allowed"]), + ) + ) database.writer_db.litellm_objectpermissiontable.find_unique = AsyncMock( return_value=None, side_effect=None if missing else RuntimeError("writer unavailable") ) @@ -10523,7 +11118,9 @@ async def test_authoritative_group_grants_propagate_policy_outages( database: Final = MagicMock() database.db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable")) - database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock(side_effect=RuntimeError("database unavailable")) + database.writer_db.litellm_accessgrouptable.find_unique = AsyncMock( + side_effect=RuntimeError("database unavailable") + ) monkeypatch.setattr(proxy_server, "prisma_client", database) monkeypatch.setattr(proxy_server, "user_api_key_cache", UserApiKeyCache()) if strict: diff --git a/tests/unit/proxy/auth/test_fallback_model_access.py b/tests/unit/proxy/auth/test_fallback_model_access.py index 4cbf4474596..24fcd4671c8 100644 --- a/tests/unit/proxy/auth/test_fallback_model_access.py +++ b/tests/unit/proxy/auth/test_fallback_model_access.py @@ -1,12 +1,13 @@ import pytest from litellm import Router -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import LiteLLM_ObjectPermissionTable, UserAPIKeyAuth from litellm.proxy.auth.fallback_model_access import ( RouterFallbackAccessCheck, is_model_authorized_for_token, router_fallback_access_check, ) +from litellm.search import asearch def _router() -> Router: @@ -105,3 +106,69 @@ async def test_proxy_check_reads_enforce_fallback_model_access_from_general_sett ) is expected ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("check", [ENFORCED, NOT_ENFORCED], ids=["enforced", "not-enforced"]) +async def test_search_tool_fallback_target_follows_the_key_search_tool_grant( + monkeypatch: pytest.MonkeyPatch, check: RouterFallbackAccessCheck +): + monkeypatch.setattr("litellm.proxy.proxy_server.general_settings", {}) + router = Router( + model_list=[], + search_tools=[ + {"search_tool_name": name, "litellm_params": {"search_provider": "tavily", "api_key": "k"}} + for name in ("search-a", "search-b") + ], + ) + request_kwargs = { + "original_generic_function": asearch, + "litellm_metadata": { + "user_api_key_auth": UserAPIKeyAuth( + api_key="hashed", + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-key", search_tools=["search-a"] + ), + ) + }, + } + + assert await check(model="search-a", request_kwargs=request_kwargs, llm_router=router) + assert not await check(model="search-b", request_kwargs=request_kwargs, llm_router=router) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "request_kwargs, expected", + [ + ({"original_generic_function": asearch}, True), + ({}, False), + ], + ids=["search-request", "completion-request"], +) +async def test_fallback_named_like_both_a_model_and_a_search_tool_follows_the_request_kind( + request_kwargs: dict, expected: bool +): + router = Router( + model_list=[ + {"model_name": "shared-name", "litellm_params": {"model": "openai/secret", "api_key": "k"}}, + ], + search_tools=[ + {"search_tool_name": "shared-name", "litellm_params": {"search_provider": "tavily", "api_key": "k"}}, + ], + ) + key = UserAPIKeyAuth( + api_key="hashed", + models=["open-model"], + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=["shared-name"]), + ) + + allowed = await ENFORCED( + model="shared-name", + request_kwargs={**request_kwargs, "metadata": {"user_api_key_auth": key}}, + llm_router=router, + ) + + assert allowed is expected diff --git a/tests/unit/proxy/auth/test_search_tool_deny_by_default.py b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py new file mode 100644 index 00000000000..cd285e0386f --- /dev/null +++ b/tests/unit/proxy/auth/test_search_tool_deny_by_default.py @@ -0,0 +1,270 @@ +from collections.abc import Mapping +from typing import Final, TypedDict +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.constants import LITELLM_PROXY_MASTER_KEY_ALIAS +from litellm.proxy import proxy_server +from litellm.proxy._types import ( + UI_TEAM_ID, + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTable, + LiteLLM_UserTable, + LitellmUserRoles, + ProxyErrorTypes, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.auth_checks import ( + TeamObjectLoader, + can_caller_call_search_tool, + check_unregistered_search_fallback, +) +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key + +_DENY_ON: Final = {"search_tool_deny_by_default": True} +_KEY: Final = ProxyErrorTypes.key_search_tool_access_denied +_TEAM: Final = ProxyErrorTypes.team_search_tool_access_denied +_USER: Final = ProxyErrorTypes.user_search_tool_access_denied +_LEGACY: Final = ProxyErrorTypes.key_model_access_denied +_TEAM_LOAD_FAILS: Final = "team-load-fails" +_NO_ROW: Final = "no-row" + + +class CallerFields(TypedDict, total=False): + virtual_key: bool + key_tools: list[str] | None | str + team_id: str | None + user_id: str | None + user_role: LitellmUserRoles | None + api_key: str + + +def _grant(search_tools: list[str] | None, object_permission_id: str = "op") -> LiteLLM_ObjectPermissionTable: + return LiteLLM_ObjectPermissionTable(object_permission_id=object_permission_id, search_tools=search_tools) + + +def _caller( + virtual_key: bool = True, + key_tools: list[str] | None | str = _NO_ROW, + team_id: str | None = None, + user_id: str | None = "user-1", + user_role: LitellmUserRoles | None = LitellmUserRoles.INTERNAL_USER, + api_key: str = "sk-caller", +) -> UserAPIKeyAuth: + token: Final = UserAPIKeyAuth( + api_key=api_key, + user_id=user_id, + user_role=user_role, + team_id=team_id, + object_permission_id=None if key_tools == _NO_ROW else "op-key", + object_permission=None if key_tools == _NO_ROW else _grant(key_tools, "op-key"), + ) + token.via_virtual_key = virtual_key + return token + + +def _team_loader(team_tools: list[str] | None | str) -> TeamObjectLoader: + async def load() -> LiteLLM_TeamTable | None: + if team_tools == _TEAM_LOAD_FAILS: + raise ProxyException(message="team lookup failed", type="auth_error", param="team_id", code=404) + if team_tools == _NO_ROW: + return LiteLLM_TeamTable(team_id="team-1") + return LiteLLM_TeamTable( + team_id="team-1", object_permission_id="op-team", object_permission=_grant(team_tools, "op-team") + ) + + return load + + +async def _no_team() -> None: + return None + + +@pytest.fixture +def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache: + user_api_key_cache: Final = UserApiKeyCache() + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + return user_api_key_cache + + +def _cache_user(cache: UserApiKeyCache, user_tools: list[str] | None) -> None: + cache.set_cache(key="user-1", value=LiteLLM_UserTable(user_id="user-1", object_permission_id="op-user")) + cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant(user_tools, "op-user")) + + +async def _denied_by( + general_settings: Mapping[str, object], caller: UserAPIKeyAuth, load_team: TeamObjectLoader = _no_team +) -> ProxyErrorTypes | None: + try: + await can_caller_call_search_tool("search-a", caller, general_settings, load_team) + except ProxyException as e: + denial = e + else: + return None + assert (denial.code, denial.param) == ("403", "search_tool_name") + return ProxyErrorTypes(denial.type) + + +@pytest.mark.parametrize( + "general_settings", + [{}, {"search_tool_deny_by_default": False}], + ids=["omitted", "false"], +) +@pytest.mark.parametrize( + "caller, team_tools, expected", + [ + pytest.param(CallerFields(), None, None, id="no grants anywhere"), + pytest.param(CallerFields(key_tools=[]), None, None, id="empty key list"), + pytest.param(CallerFields(team_id="team-1"), [], None, id="empty team list"), + pytest.param(CallerFields(key_tools=["search-b"]), None, _LEGACY, id="key allowlist excludes"), + pytest.param(CallerFields(team_id="team-1"), ["search-b"], _LEGACY, id="team allowlist excludes"), + pytest.param(CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, id="both allow"), + ], +) +@pytest.mark.asyncio +async def test_search_tool_access_is_unchanged_without_search_tool_deny_by_default( + cache: UserApiKeyCache, + general_settings: dict[str, bool], + caller: CallerFields, + team_tools: list[str] | None, + expected: ProxyErrorTypes | None, +): + _cache_user(cache, ["search-b"]) + load_team: Final = _no_team if team_tools is None else _team_loader(team_tools) + assert await _denied_by(general_settings, _caller(**caller), load_team) == expected + + +@pytest.mark.parametrize( + "caller, team_tools, user_tools, expected", + [ + pytest.param(CallerFields(), None, ["search-a"], _KEY, id="standalone key, no permission row"), + pytest.param(CallerFields(key_tools=None), None, None, _KEY, id="standalone key, search_tools null"), + pytest.param(CallerFields(key_tools=[]), None, None, _KEY, id="standalone key, empty grant"), + pytest.param(CallerFields(key_tools=["search-b"]), None, None, _KEY, id="standalone key, grants another tool"), + pytest.param(CallerFields(key_tools=["search-a"]), None, [], None, id="standalone key, key grant is enough"), + pytest.param(CallerFields(key_tools=[]), None, ["search-a"], _KEY, id="standalone key, user cannot stand in"), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), ["search-a"], None, None, id="team key, both" + ), + pytest.param(CallerFields(key_tools=[], team_id="team-1"), ["search-a"], None, _KEY, id="team key, empty key"), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), [], None, _TEAM, id="team key, empty team" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), None, None, _TEAM, id="team key, team null" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), _NO_ROW, None, _TEAM, id="team key, team has no row" + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), + _TEAM_LOAD_FAILS, + None, + _TEAM, + id="team key, team fails to load", + ), + pytest.param( + CallerFields(key_tools=["search-a"], team_id="team-1"), + ["search-b"], + ["search-a"], + _TEAM, + id="team key, user cannot stand in for team", + ), + pytest.param( + CallerFields(virtual_key=False, team_id="team-1"), ["search-a"], [], None, id="keyless team member" + ), + pytest.param( + CallerFields(virtual_key=False, team_id="team-1"), [], ["search-a"], _TEAM, id="keyless, empty team" + ), + pytest.param(CallerFields(virtual_key=False), None, ["search-a"], None, id="keyless user, user grants"), + pytest.param(CallerFields(virtual_key=False), None, None, _USER, id="keyless user, search_tools null"), + pytest.param(CallerFields(virtual_key=False), None, [], _USER, id="keyless user, empty grant"), + pytest.param( + CallerFields(virtual_key=False, user_id="user-2"), None, None, _USER, id="keyless user fails to load" + ), + pytest.param(CallerFields(virtual_key=False, user_id=None), None, None, _USER, id="keyless caller, no user"), + pytest.param( + CallerFields(user_role=LitellmUserRoles.PROXY_ADMIN), + None, + None, + _KEY, + id="proxy admin virtual key not exempt", + ), + pytest.param(CallerFields(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, None, None, id="master key exempt"), + pytest.param( + CallerFields(team_id=UI_TEAM_ID, virtual_key=False), None, None, None, id="dashboard session exempt" + ), + ], +) +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_requires_every_owning_identity_to_grant( + cache: UserApiKeyCache, + caller: CallerFields, + team_tools: list[str] | str | None, + user_tools: list[str] | None, + expected: ProxyErrorTypes | None, +): + if user_tools is not None: + _cache_user(cache, user_tools) + elif caller.get("user_id", "user-1") == "user-1": + _cache_user(cache, None) + load_team: Final = _no_team if team_tools is None and "team_id" not in caller else _team_loader(team_tools) + assert await _denied_by(_DENY_ON, _caller(**caller), load_team) == expected + + +@pytest.mark.parametrize("value", ["true", "enabled", 1], ids=["string-true", "string", "int"]) +@pytest.mark.asyncio +async def test_non_boolean_search_tool_deny_by_default_enables_the_policy(cache: UserApiKeyCache, value: object): + assert await _denied_by({"search_tool_deny_by_default": value}, _caller(key_tools=[])) == _KEY + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_denies_when_no_database_is_connected( + cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(proxy_server, "prisma_client", None) + assert await _denied_by(_DENY_ON, _caller(key_tools=["search-a"])) == _KEY + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_reads_the_user_grant_on_every_call(cache: UserApiKeyCache): + caller: Final = _caller(virtual_key=False) + _cache_user(cache, ["search-a"]) + assert await _denied_by(_DENY_ON, caller) is None + cache.set_cache(key=object_permission_cache_key("op-user"), value=_grant([], "op-user")) + assert await _denied_by(_DENY_ON, caller) == _USER + + +@pytest.mark.asyncio +async def test_search_tool_deny_by_default_does_not_load_the_user_when_off( + cache: UserApiKeyCache, monkeypatch: pytest.MonkeyPatch +): + get_user_object: Final = AsyncMock() + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_user_object", get_user_object) + assert await _denied_by({}, _caller(virtual_key=False)) is None + get_user_object.assert_not_awaited() + + +@pytest.mark.parametrize( + "general_settings, caller, expected", + [ + pytest.param({}, _caller(), None, id="flag off"), + pytest.param(_DENY_ON, _caller(), _KEY, id="virtual key"), + pytest.param(_DENY_ON, _caller(virtual_key=False, team_id="team-1"), _TEAM, id="keyless team member"), + pytest.param(_DENY_ON, _caller(virtual_key=False), _USER, id="keyless user"), + pytest.param(_DENY_ON, _caller(user_role=LitellmUserRoles.PROXY_ADMIN), _KEY, id="proxy admin key"), + pytest.param(_DENY_ON, _caller(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS), None, id="master key"), + ], +) +def test_unregistered_search_fallback_follows_search_tool_deny_by_default( + general_settings: Mapping[str, object], caller: UserAPIKeyAuth, expected: ProxyErrorTypes | None +): + if expected is None: + assert check_unregistered_search_fallback(caller, general_settings) is True + return + with pytest.raises(ProxyException) as exc_info: + check_unregistered_search_fallback(caller, general_settings) + assert (exc_info.value.type, exc_info.value.code) == (expected, "403") diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index d4d11527994..5de1d9300fe 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -12,6 +12,7 @@ from functools import partial from pathlib import Path from textwrap import dedent from types import SimpleNamespace +from typing import Final from unittest.mock import ANY, AsyncMock, MagicMock, patch @@ -21,11 +22,13 @@ from fastapi import HTTPException, status import litellm import litellm.proxy.proxy_server from litellm.caching.dual_cache import DualCache +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy._types import ( LiteLLMRoutes, LiteLLM_JWTAuth, LiteLLM_BudgetTable, LiteLLM_EndUserTable, + LiteLLM_ObjectPermissionTable, LiteLLM_OrganizationTable, LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, @@ -5304,6 +5307,222 @@ async def test_centralized_common_checks_tolerates_db_errors_when_fetching_conte setattr(_proxy_server_mod, k, v) +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("deny_by_default", "team_lookup_error", "denied"), + [ + (False, RuntimeError("team cache unavailable"), False), + (True, RuntimeError("team cache unavailable"), True), + (True, HTTPException(status_code=404, detail="team read failed"), True), + ], + ids=["flag-off-lookup-swallowed", "flag-on-lookup-swallowed", "flag-on-team-rebuilt-from-token"], +) +async def test_team_key_vector_store_access_when_team_cannot_be_resolved( + monkeypatch: pytest.MonkeyPatch, deny_by_default: bool, team_lookup_error: Exception, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth( + api_key="sk-team-key", team_id="team-1", team_models=["gpt-4o-mini"], object_permission_id="key-permission" + ) + token.via_virtual_key = True + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: LiteLLM_ObjectPermissionTable( + object_permission_id=where["object_permission_id"], vector_stores=["KBSTOREA"] + ) + if where["object_permission_id"] == "key-permission" + else None + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())), + "general_settings": {"vector_store_deny_by_default": deny_by_default}, + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_team_object", AsyncMock(side_effect=team_lookup_error) + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.team_vector_store_access_denied, + "vector_store", + "401", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("team_lookup", "denied"), + [ + ({"team-1": "team-1-grants-a"}, False), + (RuntimeError("team cache unavailable"), True), + ({"team-1": "team-1-grants-none", "team-2": "team-2-grants-a"}, True), + ], + ids=["resolved-team-grants", "team-lookup-swallowed-no-personal-fallback", "other-member-team-grants-ignored"], +) +async def test_keyless_team_member_vector_store_access_uses_only_the_resolved_team( + monkeypatch: pytest.MonkeyPatch, team_lookup: dict[str, str] | Exception, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth( + user_id="user-1", + team_id="team-1", + team_models=["gpt-4o-mini"], + user_role=LitellmUserRoles.INTERNAL_USER, + ) + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + grants: Final = { + "team-1-grants-a": ["KBSTOREA"], + "team-1-grants-none": [], + "team-2-grants-a": ["KBSTOREA"], + "user-grants-a": ["KBSTOREA"], + } + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: LiteLLM_ObjectPermissionTable( + object_permission_id=where["object_permission_id"], vector_stores=grants[where["object_permission_id"]] + ) + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "proxy_logging_obj": MagicMock(service_logging_obj=MagicMock(async_service_success_hook=AsyncMock())), + "general_settings": {"vector_store_deny_by_default": True}, + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + + async def get_team(team_id: str, **_: object) -> LiteLLM_TeamTableCachedObj: + if isinstance(team_lookup, Exception): + raise team_lookup + return LiteLLM_TeamTableCachedObj(team_id=team_id, models=["gpt-4o-mini"], object_permission_id=team_lookup[team_id]) + + monkeypatch.setattr("litellm.proxy.auth.user_api_key_auth.get_team_object", get_team) + monkeypatch.setattr("litellm.proxy.auth.auth_checks.get_team_membership", AsyncMock(return_value=None)) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + AsyncMock( + return_value=LiteLLM_UserTable( + user_id="user-1", teams=["team-1", "team-2"], object_permission_id="user-grants-a" + ) + ), + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.team_vector_store_access_denied, + "vector_store", + "401", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("user_permission_id", "denied"), + [("admin-grants-a", False), (None, True)], + ids=["admin-personal-grant", "admin-without-grant"], +) +async def test_keyless_proxy_admin_keeps_personal_vector_store_grants_under_deny_by_default( + monkeypatch: pytest.MonkeyPatch, user_permission_id: str | None, denied: bool +): + from fastapi import Request + from starlette.datastructures import URL + + token: Final = UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN) + request: Final = Request(scope={"type": "http"}) + request._url = URL(url="/v1/rag/query") + database: Final = MagicMock() + database.db.litellm_objectpermissiontable.find_unique = AsyncMock( + side_effect=lambda where: SimpleNamespace( + dict=lambda: {"object_permission_id": where["object_permission_id"], "vector_stores": ["KBSTOREA"]}, + vector_stores=["KBSTOREA"], + ) + ) + attrs: Final = { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": database, + "general_settings": {"vector_store_deny_by_default": True}, + "user_api_key_cache": UserApiKeyCache(), + "proxy_logging_obj": MagicMock( + service_logging_obj=MagicMock( + async_service_success_hook=AsyncMock(), async_service_failure_hook=AsyncMock() + ) + ), + } + for name, value in attrs.items(): + monkeypatch.setattr(litellm.proxy.proxy_server, name, value) + monkeypatch.setattr(litellm, "vector_store_registry", None) + monkeypatch.setattr( + "litellm.proxy.auth.user_api_key_auth.get_user_object", + AsyncMock( + return_value=LiteLLM_UserTable( + user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN, object_permission_id=user_permission_id + ) + ), + ) + + checks: Final = _run_centralized_common_checks( + user_api_key_auth_obj=token, + request=request, + request_data={ + "model": "gpt-4o-mini", + "messages": [{"role": "user", "content": "what is in this KB?"}], + "retrieval_config": {"vector_store_id": "KBSTOREA", "custom_llm_provider": "bedrock"}, + }, + route="/v1/rag/query", + ) + if not denied: + await checks + return + with pytest.raises(ProxyException) as exc_info: + await checks + assert (exc_info.value.type, exc_info.value.param, exc_info.value.code) == ( + ProxyErrorTypes.user_vector_store_access_denied, + "vector_store", + "401", + ) + + @pytest.mark.asyncio async def test_centralized_common_checks_propagates_end_user_budget_error(): """Regression: ``get_end_user_object`` raises ``litellm.BudgetExceededError`` diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index 33ae986a511..d10a96e2aef 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -121,6 +121,137 @@ def test_arize_sampling_rate_rejected_on_non_arize_callback(): assert callback_config_error("arize", {"arize_success_sampling_rate": "0.5"}) is None +class TestArizeOtlpProtocol: + def test_protocol_on_a_non_arize_callback_is_rejected(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + for callback_name in ["langfuse", "langfuse_otel", "datadog", None]: + error = callback_config_error(callback_name, {"arize_otlp_protocol": "grpc"}) + assert error is not None and "applies to the arize callback only" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_unknown_protocols_are_rejected(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + for bad in ["otlp_http", "HTTP/PROTOBUF", "http_json", "", "None"]: + error = callback_config_error("arize", {"arize_otlp_protocol": bad}) + assert error is not None and "arize_otlp_protocol" in error and "http/protobuf" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_protocol_is_rejected_while_otel_v2_is_off(self, monkeypatch): + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) + assert error is not None and "LITELLM_OTEL_V2" in error + finally: + is_otel_v2_enabled.cache_clear() + + def test_both_protocols_are_accepted_with_otel_v2_on(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + assert callback_config_error("arize", {"arize_otlp_protocol": "grpc"}) is None + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_protocol_is_not_a_family_credential(self): + stored = [{"arize_api_key": "k1"}] + assert cross_entry_family_error({"arize_otlp_protocol": "grpc"}, stored) is None + assert cross_entry_family_error({"arize_otlp_protocol": "http/protobuf"}, stored) is None + + def test_protocol_on_a_key_logging_entry_is_validated(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + invalid_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "carrier-pigeon"}, + } + ] + } + error = logging_metadata_config_error(invalid_metadata) + assert error is not None and "arize_otlp_protocol" in error + + valid_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + assert logging_metadata_config_error(valid_metadata) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_failure_only_arize_callbacks_reject_the_protocol(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + error = callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "failure") + assert error is not None and "arize_otlp_protocol" in error and "failure" in error + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success") is None + assert ( + callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}, "success_and_failure") is None + ) + assert callback_config_error("arize", {"arize_otlp_protocol": "http/protobuf"}) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_failure_only_key_logging_entry_rejects_the_protocol(self, monkeypatch): + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + is_otel_v2_enabled.cache_clear() + try: + failure_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "failure", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + error = logging_metadata_config_error(failure_metadata) + assert error is not None and "arize_otlp_protocol" in error + + success_metadata = { + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_otlp_protocol": "http/protobuf"}, + } + ] + } + assert logging_metadata_config_error(success_metadata) is None + finally: + is_otel_v2_enabled.cache_clear() + + def test_arize_sampling_rates_are_not_family_credentials(): """The rates choose what the Arize family exports, not where it sends, so an entry that repeats or adds a rate next to a stored Arize entry is not the diff --git a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py index 582f6e82f11..49bc2d0f64e 100644 --- a/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py +++ b/tests/unit/proxy/management_endpoints/search_endpoints/test_search_tool_management.py @@ -1,14 +1,16 @@ import contextlib import json -from types import SimpleNamespace +from collections.abc import Callable from datetime import datetime +from types import SimpleNamespace from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from fastapi.testclient import TestClient - +# Import proxy_server module first to ensure it's initialized +import litellm.proxy.proxy_server as ps from litellm.proxy._types import ( LiteLLM_ObjectPermissionTable, LiteLLM_TeamTable, @@ -16,9 +18,6 @@ from litellm.proxy._types import ( UserAPIKeyAuth, ) -# Import proxy_server module first to ensure it's initialized -import litellm.proxy.proxy_server as ps - # Now we can safely import app from litellm.proxy.proxy_server import app from litellm.types.search import SearchToolInfoResponse @@ -116,9 +115,7 @@ async def test_list_search_tools_config_only(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - return_value={"search_tools": config_tools} - ) + mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools}) mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth @@ -192,9 +189,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - return_value={"search_tools": config_tools} - ) + mock_proxy_config.get_config = AsyncMock(return_value={"search_tools": config_tools}) mock_proxy_config.parse_search_tools = MagicMock(return_value=config_tools) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth @@ -215,11 +210,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): # Verify DB tool is present db_tool = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "existing-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "existing-tool"), None, ) assert db_tool is not None @@ -232,11 +223,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): # Verify unique config tool is present config_tool = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "unique-config-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "unique-config-tool"), None, ) assert config_tool is not None @@ -247,8 +234,7 @@ async def test_list_search_tools_filters_duplicate_config_tools(monkeypatch): ( t for t in data["search_tools"] - if t["search_tool_name"] == "existing-tool" - and t["is_from_config"] is True + if t["search_tool_name"] == "existing-tool" and t["is_from_config"] is True ), None, ) @@ -323,11 +309,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test datetime conversion for tool 1 tool1 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "datetime-test-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "datetime-test-tool"), None, ) assert tool1 is not None @@ -341,11 +323,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test None handling for tool 2 tool2 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "null-datetime-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "null-datetime-tool"), None, ) assert tool2 is not None @@ -357,11 +335,7 @@ async def test_list_search_tools_datetime_conversion(monkeypatch): # Test string passthrough for tool 3 tool3 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "string-datetime-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "string-datetime-tool"), None, ) assert tool3 is not None @@ -401,9 +375,7 @@ async def test_list_search_tools_config_error_handling(monkeypatch): with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): # Mock proxy_config to raise an error mock_proxy_config = MagicMock() - mock_proxy_config.get_config = AsyncMock( - side_effect=Exception("Config error") - ) + mock_proxy_config.get_config = AsyncMock(side_effect=Exception("Config error")) with patch("litellm.proxy.proxy_server.proxy_config", mock_proxy_config): # Mock auth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth @@ -422,13 +394,8 @@ async def test_list_search_tools_config_error_handling(monkeypatch): assert len(data["search_tools"]) == 1 assert data["search_tools"][0]["search_tool_name"] == "db-tool-1" # Verify masking of sensitive values - assert ( - data["search_tools"][0]["litellm_params"]["api_key"] - != "sk-test" - ) - assert ( - "****" in data["search_tools"][0]["litellm_params"]["api_key"] - ) + assert data["search_tools"][0]["litellm_params"]["api_key"] != "sk-test" + assert "****" in data["search_tools"][0]["litellm_params"]["api_key"] finally: app.dependency_overrides.pop(user_api_key_auth, None) @@ -543,31 +510,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch): # Test tool 1: api_key should be masked tool1 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "perplexity-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "perplexity-tool"), None, ) assert tool1 is not None - assert ( - tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef" - ) + assert tool1["litellm_params"]["api_key"] != "pplx-sk-9876567890abcdef" assert "****" in tool1["litellm_params"]["api_key"] assert tool1["litellm_params"]["search_provider"] == "perplexity" - assert ( - tool1["litellm_params"]["api_base"] - == "https://api.perplexity.ai" - ) + assert tool1["litellm_params"]["api_base"] == "https://api.perplexity.ai" # Test tool 2: api_key should be masked tool2 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tavily-tool" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tavily-tool"), None, ) assert tool2 is not None @@ -577,29 +531,18 @@ async def test_list_search_tools_db_masking_sensitive_values(monkeypatch): # Test tool 3: access_token and secret_key should be masked tool3 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tool-with-token" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-token"), None, ) assert tool3 is not None - assert ( - tool3["litellm_params"]["access_token"] - != "token-abcdefghijklmnop" - ) + assert tool3["litellm_params"]["access_token"] != "token-abcdefghijklmnop" assert "****" in tool3["litellm_params"]["access_token"] assert tool3["litellm_params"]["secret_key"] != "secret-xyz123" assert "****" in tool3["litellm_params"]["secret_key"] # Test tool 4: non-sensitive fields should remain unmasked tool4 = next( - ( - t - for t in data["search_tools"] - if t["search_tool_name"] == "tool-with-non-sensitive" - ), + (t for t in data["search_tools"] if t["search_tool_name"] == "tool-with-non-sensitive"), None, ) assert tool4 is not None @@ -615,6 +558,7 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error(): """`SearchToolRegistry.get_all_search_tools_from_db` self-heals across one ClientNotConnectedError via call_with_db_reconnect_retry.""" import prisma + from litellm.proxy.search_endpoints.search_tool_registry import ( SearchToolRegistry, ) @@ -628,25 +572,18 @@ async def test_get_all_search_tools_from_db_retries_on_transport_error(): return [] mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock( - side_effect=_flaky_find_many - ) + mock_prisma_client.db.litellm_searchtoolstable.find_many = AsyncMock(side_effect=_flaky_find_many) mock_prisma_client.attempt_db_reconnect = AsyncMock(return_value=True) mock_prisma_client._db_auth_reconnect_timeout_seconds = 2.0 mock_prisma_client._db_auth_reconnect_lock_timeout_seconds = 0.1 - result = await SearchToolRegistry.get_all_search_tools_from_db( - prisma_client=mock_prisma_client - ) + result = await SearchToolRegistry.get_all_search_tools_from_db(prisma_client=mock_prisma_client) assert result == [] assert len(invocations) == 2 mock_prisma_client.attempt_db_reconnect.assert_awaited_once() reconnect_kwargs = mock_prisma_client.attempt_db_reconnect.await_args.kwargs - assert ( - reconnect_kwargs["reason"] - == "get_all_search_tools_from_db_lookup_failure" - ) + assert reconnect_kwargs["reason"] == "get_all_search_tools_from_db_lookup_failure" @contextlib.contextmanager @@ -749,9 +686,7 @@ async def test_list_search_tools_scoped_to_key_object_permission(): @pytest.mark.asyncio async def test_list_search_tools_unrestricted_internal_user_sees_all(): """An internal user with no search_tools allowlist is unrestricted and sees every tool.""" - unrestricted_user = UserAPIKeyAuth( - user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user" - ) + unrestricted_user = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="internal_user") with ( _mock_search_tool_backend(_scoping_db_tools()), @@ -791,9 +726,7 @@ async def test_list_search_tools_scoped_to_team_object_permission(): response = TestClient(app).get("/search_tools/list") assert response.status_code == 200 - assert [t["search_tool_name"] for t in response.json()["search_tools"]] == [ - "db-tool-2" - ] + assert [t["search_tool_name"] for t in response.json()["search_tools"]] == ["db-tool-2"] @pytest.mark.asyncio @@ -1061,9 +994,15 @@ def _live_router_and_db(db_rows: list): fake_router.search_tools = list(db_rows) with contextlib.ExitStack() as stack: - stack.enter_context(patch("litellm.proxy.proxy_server.prisma_client", MagicMock())) # test-quality-ok: proxy globals are the only seam; see the module note above - stack.enter_context(patch("litellm.proxy.proxy_server.proxy_config", proxy_config)) # test-quality-ok: proxy globals are the only seam; see the module note above - stack.enter_context(patch("litellm.proxy.proxy_server.llm_router", fake_router)) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()) + ) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.proxy_config", proxy_config) + ) # test-quality-ok: proxy globals are the only seam; see the module note above + stack.enter_context( + patch("litellm.proxy.proxy_server.llm_router", fake_router) + ) # test-quality-ok: proxy globals are the only seam; see the module note above stack.enter_context( patch( # test-quality-ok: proxy globals are the only seam; see the module note above "litellm.proxy.search_endpoints.search_tool_management.SEARCH_TOOL_REGISTRY", @@ -1152,6 +1091,171 @@ async def test_create_search_tool_survives_a_failing_router_refresh(): assert response.json()["search_tool_name"] == "tavily-search" +def _search_caller( + virtual_key: bool, + key_search_tools: list[str] | None = None, + team_id: str | None = None, + user_role: LitellmUserRoles = LitellmUserRoles.INTERNAL_USER, + api_key: str = "sk-caller", +) -> UserAPIKeyAuth: + caller = UserAPIKeyAuth( + api_key=api_key, + user_role=user_role, + user_id="internal_user", + team_id=team_id, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + caller.via_virtual_key = virtual_key + return caller + + +@pytest.fixture +def search_permission_cache(monkeypatch: pytest.MonkeyPatch) -> Callable[[list[str] | None], None]: + from litellm.proxy._types import LiteLLM_UserTable + from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache, object_permission_cache_key + + cache = UserApiKeyCache() + monkeypatch.setattr(ps, "user_api_key_cache", cache) + monkeypatch.setattr(ps, "prisma_client", MagicMock()) + + def grant_user(search_tools: list[str] | None) -> None: + cache.set_cache( + key="internal_user", value=LiteLLM_UserTable(user_id="internal_user", object_permission_id="op-user") + ) + cache.set_cache( + key=object_permission_cache_key("op-user"), + value=LiteLLM_ObjectPermissionTable(object_permission_id="op-user", search_tools=search_tools), + ) + + return grant_user + + +@pytest.mark.parametrize( + "general_settings, user_search_tools, expected", + [ + ({}, None, ["db-tool-1", "db-tool-2"]), + ({"search_tool_deny_by_default": False}, [], ["db-tool-1", "db-tool-2"]), + ({"search_tool_deny_by_default": True}, None, []), + ({"search_tool_deny_by_default": True}, [], []), + ({"search_tool_deny_by_default": True}, ["db-tool-2"], ["db-tool-2"]), + ], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_keyless_user_follows_search_tool_deny_by_default( + search_permission_cache: Callable[[list[str] | None], None], + general_settings: dict[str, bool], + user_search_tools: list[str] | None, + expected: list[str], +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + search_permission_cache(user_search_tools) + team_lookup = AsyncMock() + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=False), + team_lookup, + general_settings, + ) + + assert [t["search_tool_name"] for t in visible] == expected + team_lookup.assert_not_awaited() + + +@pytest.mark.parametrize( + "key_search_tools, expected", + [(None, []), ([], []), (["db-tool-1"], ["db-tool-1"])], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_standalone_key_needs_its_own_grant( + search_permission_cache: Callable[[list[str] | None], None], key_search_tools: list[str] | None, expected: list[str] +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + search_permission_cache(["db-tool-1", "db-tool-2"]) + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=True, key_search_tools=key_search_tools), + AsyncMock(), + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + +@pytest.mark.parametrize( + "key_search_tools, team_search_tools, expected", + [ + (["db-tool-1"], [], []), + ([], ["db-tool-1"], []), + (["db-tool-1", "db-tool-2"], ["db-tool-1"], ["db-tool-1"]), + ], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_team_key_needs_key_and_team_grants( + search_permission_cache: Callable[[list[str] | None], None], + key_search_tools: list[str], + team_search_tools: list[str], + expected: list[str], +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + team_lookup = AsyncMock( + return_value=LiteLLM_TeamTable( + team_id="team-1", + object_permission_id="op-team", + object_permission=LiteLLM_ObjectPermissionTable( + object_permission_id="op-team", search_tools=team_search_tools + ), + ) + ) + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + _search_caller(virtual_key=True, key_search_tools=key_search_tools, team_id="team-1"), + team_lookup, + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + +@pytest.mark.parametrize( + "caller, expected", + [ + ( + _search_caller( + virtual_key=False, user_role=LitellmUserRoles.PROXY_ADMIN, api_key="litellm_proxy_master_key" + ), + ["db-tool-1", "db-tool-2"], + ), + (_search_caller(virtual_key=True, user_role=LitellmUserRoles.PROXY_ADMIN), []), + ], + ids=["master-key-sees-every-tool", "admin-virtual-key-is-filtered"], +) +@pytest.mark.asyncio +async def test_filter_visible_search_tools_admin_under_search_tool_deny_by_default( + search_permission_cache: Callable[[list[str] | None], None], caller: UserAPIKeyAuth, expected: list[str] +): + from litellm.proxy.search_endpoints.search_tool_management import _filter_visible_search_tools + + visible = await _filter_visible_search_tools( + _search_tool_responses("db-tool-1", "db-tool-2"), + caller, + AsyncMock(), + {"search_tool_deny_by_default": True}, + ) + + assert [t["search_tool_name"] for t in visible] == expected + + class _StoredSearchToolRow(SimpleNamespace): def __iter__(self): return iter(self.__dict__.items()) diff --git a/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py index b64877f7816..34fa164fd6d 100644 --- a/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_callback_management_endpoints.py @@ -361,6 +361,70 @@ class TestNewRelicTeamCallbackValidation: is_otel_v2_enabled.cache_clear() +class TestArizeOtlpProtocolValidation: + def test_valid_protocols_are_accepted(self): + from litellm.proxy._types import AddTeamCallback + + for protocol in ("grpc", "http/protobuf"): + data = AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_api_key": "k", "arize_space_id": "s", "arize_otlp_protocol": protocol}, + ) + assert data.callback_vars["arize_otlp_protocol"] == protocol + + def test_an_invalid_protocol_is_rejected_at_model_validation(self): + from pydantic import ValidationError + + from litellm.proxy._types import AddTeamCallback + + with pytest.raises(ValidationError, match="arize_otlp_protocol"): + AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"arize_otlp_protocol": "otlp_grpc"}, + ) + + def test_a_failure_only_team_callback_rejects_the_protocol(self, monkeypatch): + from fastapi import HTTPException + + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy._types import AddTeamCallback + from litellm.proxy.management_endpoints.team_callback_endpoints import _validate_team_callback + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + try: + with pytest.raises(HTTPException) as exc: + _validate_team_callback( + AddTeamCallback( + callback_name="arize", + callback_type="failure", + callback_vars={ + "arize_api_key": "k", + "arize_space_id": "s", + "arize_otlp_protocol": "http/protobuf", + }, + ) + ) + assert exc.value.status_code == 400 + assert "arize_otlp_protocol" in str(exc.value.detail) + finally: + is_otel_v2_enabled.cache_clear() + + def test_the_raw_otlp_env_var_name_stays_rejected(self): + from pydantic import ValidationError + + from litellm.proxy._types import AddTeamCallback + + with pytest.raises(ValidationError, match="Invalid callback variable"): + AddTeamCallback( + callback_name="arize", + callback_type="success", + callback_vars={"OTEL_EXPORTER_OTLP_PROTOCOL": "http/protobuf"}, + ) + + class TestNewRelicKeyLoggingValidation: """Key-level logging is written through key metadata, not /team/callback.""" diff --git a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py index 7150ba892df..912622da4bd 100644 --- a/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_internal_user_endpoints.py @@ -4066,6 +4066,34 @@ async def test_user_update_invalidates_the_cached_entitlement(mocker): } +@pytest.mark.asyncio +async def test_user_update_broadcasts_the_entitlement_invalidation_to_other_workers(mocker: MockerFixture): + from litellm.proxy.management_endpoints.internal_user_endpoints import ( + _update_single_user_helper, + ) + + _object_permission_mocks(mocker) + cache: Final = mocker.MagicMock() + cache.async_delete_cache = mocker.AsyncMock() + mocker.patch("litellm.proxy.proxy_server.user_api_key_cache", cache) # test-quality-ok: substitute the cache dependency + broadcast: Final = mocker.patch( # test-quality-ok: observe the Redis publication boundary + "litellm.proxy.common_utils.auth_cache_invalidation_pubsub.publish_auth_cache_invalidation", + new_callable=mocker.AsyncMock, + ) + + await _update_single_user_helper( + user_request=UpdateUserRequest(user_id="target-user", object_permission={"vector_stores": []}), + user_api_key_dict=UserAPIKeyAuth(user_id="admin-1", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + + broadcast_keys: Final = {call.kwargs["cache_key"] for call in broadcast.await_args_list} + assert broadcast_keys == { + "object_permission_id:perm-new", + "user_object_permission_id:target-user", + "target-user", + } + + @pytest.mark.asyncio async def test_admin_can_clear_a_users_mcp_entitlement(mocker): """An explicit empty object_permission means "no object permission", so it must unlink. diff --git a/tests/unit/proxy/management_endpoints/test_team_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_endpoints.py index 6f6dbe1e954..6c5447fce64 100644 --- a/tests/unit/proxy/management_endpoints/test_team_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_endpoints.py @@ -8417,6 +8417,7 @@ async def test_update_team_org_scoped_tpm_rpm_bypasses_user_limit( # Mock team update mock_updated_team = MagicMock(spec=LiteLLM_TeamTable) mock_updated_team.team_id = "org-team-update-bypass-123" + mock_updated_team.object_permission_id = None mock_updated_team.tpm_limit = 10000 mock_updated_team.rpm_limit = 1000 mock_updated_team.access_group_ids = None @@ -8570,6 +8571,7 @@ async def test_update_team_guardrails_with_org_id( # Mock team update mock_updated_team = MagicMock(spec=LiteLLM_TeamTable) mock_updated_team.team_id = "team-guardrails-123" + mock_updated_team.object_permission_id = None mock_updated_team.organization_id = "test-org-guardrails" mock_updated_team.metadata = { "guardrails": ["aporia-pre-call", "aporia-post-call"] diff --git a/tests/unit/proxy/proxy_server/test_proxy_config.py b/tests/unit/proxy/proxy_server/test_proxy_config.py index 2f5eb81bec8..0b58e5d141f 100644 --- a/tests/unit/proxy/proxy_server/test_proxy_config.py +++ b/tests/unit/proxy/proxy_server/test_proxy_config.py @@ -26,7 +26,7 @@ import pytest from pydantic import JsonValue, TypeAdapter, ValidationError import litellm -from litellm.proxy._types import CommonProxyErrors +from litellm.proxy._types import CommonProxyErrors, ConfigGeneralSettings from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper from litellm.proxy.proxy_server import ( ProxyConfig, @@ -2329,6 +2329,40 @@ async def test_load_config_logs_disabled_budget_reservation_once(tmp_path, monke assert [record.levelno for record in records] == ([logging.INFO] if setting == "true" else []) +@pytest.mark.asyncio +@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"]) +@pytest.mark.parametrize(("yaml_value", "expected"), [("true", True), ("false", False)]) +async def test_load_config_yaml_deny_by_default_is_boolean( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str, expected: bool +): + config_file: Final = tmp_path / "deny_by_default.yaml" + config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n {setting}: {yaml_value}\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + _, _, general_settings = await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + assert general_settings[setting] is expected + assert getattr(ConfigGeneralSettings.model_validate(dict(general_settings)), setting) is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("setting", ["vector_store_deny_by_default", "search_tool_deny_by_default"]) +@pytest.mark.parametrize("yaml_value", ["", "enabled"], ids=["null", "string"]) +async def test_load_config_rejects_non_boolean_deny_by_default( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, setting: str, yaml_value: str +): + config_file: Final = tmp_path / "deny_by_default.yaml" + config_file.write_text(f"model_list: []\nlitellm_settings: {{}}\ngeneral_settings:\n {setting}: {yaml_value}\n") + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setattr("litellm.proxy.proxy_server.store_model_in_db", False) + monkeypatch.delenv("LITELLM_CONFIG_BUCKET_NAME", raising=False) + + with pytest.raises(ValidationError, match=setting): + await ProxyConfig().load_config(router=None, config_file_path=str(config_file)) + + @pytest.mark.asyncio async def test_ProxyConfig_load_config_resolves_router_settings_plugins(tmp_path, monkeypatch): """Regression: router_settings.plugins dotted-path strings must be resolved to @@ -3616,9 +3650,7 @@ def test_ProxyConfig__decrypt_and_set_db_env_variables_cannot_enable_mcp_stdio(m assert os.environ.get("LITELLM_ENABLE_MCP_STDIO") is None -def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag( - monkeypatch, caplog -): +def test_ProxyConfig__decrypt_and_set_db_env_variables_warns_once_about_the_ignored_mcp_stdio_flag(monkeypatch, caplog): monkeypatch.setattr( "litellm.proxy.proxy_server.decrypt_value_helper", lambda value, key, return_original_value=False: value, diff --git a/tests/unit/proxy/search_endpoints/test_endpoints.py b/tests/unit/proxy/search_endpoints/test_endpoints.py index bd6460e3dfb..e8212b2a8b1 100644 --- a/tests/unit/proxy/search_endpoints/test_endpoints.py +++ b/tests/unit/proxy/search_endpoints/test_endpoints.py @@ -1,12 +1,142 @@ +from collections.abc import Iterator +from typing import Final from unittest.mock import AsyncMock, MagicMock import orjson import pytest +import respx +from fastapi import FastAPI +from fastapi.testclient import TestClient +import litellm +from litellm import Router from litellm.proxy import proxy_server -from litellm.proxy._types import UserAPIKeyAuth +from litellm.proxy._types import ( + LiteLLM_ObjectPermissionTable, + LiteLLM_TeamTableCachedObj, + LitellmUserRoles, + ProxyException, + UserAPIKeyAuth, +) +from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.proxy.common_utils.user_api_key_cache import UserApiKeyCache from litellm.proxy.route_llm_request import ProxyMissingRequiredParamError -from litellm.proxy.search_endpoints.endpoints import search +from litellm.proxy.search_endpoints.endpoints import router, search + +TAVILY_SEARCH_URL: Final = "https://api.tavily.com/search" +TAVILY_RESULT: Final = {"title": "LiteLLM", "url": "https://docs.litellm.ai", "content": "LLM gateway"} + + +def _search_router() -> Router: + return Router( + model_list=[], + search_tools=[ + {"search_tool_name": "search-a", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}}, + ], + num_retries=0, + ) + + +def _client(caller: UserAPIKeyAuth) -> TestClient: + app: Final = FastAPI() + app.include_router(router) + app.add_exception_handler(ProxyException, proxy_server.openai_exception_handler) + app.dependency_overrides[user_api_key_auth] = lambda: caller + return TestClient(app, raise_server_exceptions=False) + + +def _team_key(key_search_tools: list[str]) -> UserAPIKeyAuth: + caller: Final = UserAPIKeyAuth( + api_key="sk-team-key", + user_role=LitellmUserRoles.INTERNAL_USER, + user_id="user-1", + team_id="team-1", + object_permission_id="op-key", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools), + ) + caller.via_virtual_key = True + return caller + + +@pytest.fixture +def tavily(monkeypatch: pytest.MonkeyPatch) -> Iterator[respx.Route]: + monkeypatch.setattr( # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary. + litellm, + "disable_aiohttp_transport", + True, + ) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as mock: + yield mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]}) + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture +def cache(monkeypatch: pytest.MonkeyPatch) -> UserApiKeyCache: + user_api_key_cache: Final = UserApiKeyCache() + monkeypatch.setattr(proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(proxy_server, "prisma_client", MagicMock()) + monkeypatch.setattr(proxy_server, "llm_router", _search_router()) + return user_api_key_cache + + +def _cache_team(cache: UserApiKeyCache, search_tools: list[str]) -> None: + team: Final = LiteLLM_TeamTableCachedObj( + team_id="team-1", + object_permission_id="op-team", + object_permission=LiteLLM_ObjectPermissionTable(object_permission_id="op-team", search_tools=search_tools), + ) + cache.set_cache(key="team_id:team-1", value=team) + + +@pytest.mark.parametrize( + "general_settings, key_search_tools, team_search_tools, expected_status", + [ + ({}, [], [], 200), + ({"search_tool_deny_by_default": False}, [], [], 200), + ({"search_tool_deny_by_default": True}, ["search-a"], [], 403), + ({"search_tool_deny_by_default": True}, [], ["search-a"], 403), + ({"search_tool_deny_by_default": True}, ["search-a"], ["search-b"], 403), + ({"search_tool_deny_by_default": True}, ["search-a"], ["search-a"], 200), + ], +) +@pytest.mark.parametrize("path", ["/v1/search/search-a", "/search/search-a"]) +def test_direct_search_team_key_follows_search_tool_deny_by_default( + monkeypatch: pytest.MonkeyPatch, + cache: UserApiKeyCache, + tavily: respx.Route, + path: str, + general_settings: dict[str, bool], + key_search_tools: list[str], + team_search_tools: list[str], + expected_status: int, +): + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + _cache_team(cache, team_search_tools) + + response: Final = _client(_team_key(key_search_tools)).post(path, json={"query": "what is litellm"}) + + assert response.status_code == expected_status, response.text + if expected_status == 200: + assert response.json()["results"][0]["url"] == TAVILY_RESULT["url"] + assert tavily.call_count == 1 + else: + assert "search-a" in response.text + assert tavily.call_count == 0 + + +def test_direct_search_body_tool_name_is_denied_under_search_tool_deny_by_default(monkeypatch, cache, tavily): + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + caller: Final = UserAPIKeyAuth(api_key="sk-standalone", user_role=LitellmUserRoles.INTERNAL_USER) + caller.via_virtual_key = True + + response: Final = _client(caller).post( + "/v1/search", json={"search_tool_name": "search-a", "query": "what is litellm"} + ) + + assert response.status_code == 403, response.text + assert response.json()["error"]["type"] == "key_search_tool_access_denied" + assert tavily.call_count == 0 def _json_request(body: dict[str, object]) -> MagicMock: @@ -52,3 +182,104 @@ async def test_search_with_only_a_query_falls_back_to_the_proxy_default_model(mo router.asearch.assert_awaited_once() assert router.asearch.await_args.kwargs["query"] == "litellm" assert router.asearch.await_args.kwargs["model"] == "perplexity-search" + + +def _standalone_key(key_search_tools: list[str] | None) -> UserAPIKeyAuth: + caller: Final = UserAPIKeyAuth( + api_key="sk-standalone", + user_role=LitellmUserRoles.INTERNAL_USER, + object_permission_id=None if key_search_tools is None else "op-key", + object_permission=( + None + if key_search_tools is None + else LiteLLM_ObjectPermissionTable(object_permission_id="op-key", search_tools=key_search_tools) + ), + ) + caller.via_virtual_key = True + return caller + + +@pytest.mark.parametrize( + "general_settings, body", + [ + ({"search_tool_deny_by_default": True}, {"model": "search-a", "query": "what is litellm"}), + ( + {"search_tool_deny_by_default": True, "completion_model": "search-a"}, + {"query": "what is litellm"}, + ), + ], + ids=["body-model", "completion-model"], +) +@pytest.mark.parametrize("key_search_tools, expected_status", [(None, 403), (["search-a"], 200)]) +def test_direct_search_authorizes_the_tool_resolved_from_model_settings( + monkeypatch, cache, tavily, general_settings, body, key_search_tools, expected_status +): + monkeypatch.setattr(proxy_server, "general_settings", general_settings) + + response: Final = _client(_standalone_key(key_search_tools)).post("/v1/search", json=body) + + assert response.status_code == expected_status, response.text + assert tavily.call_count == (1 if expected_status == 200 else 0) + + +@pytest.mark.parametrize( + "key_search_tools, expected_status, fallback_calls", + [(["search-a"], 500, 0), (["search-a", "search-b"], 200, 1)], +) +def test_router_search_fallback_target_must_be_granted( + monkeypatch, cache, key_search_tools, expected_status, fallback_calls +): + from litellm.proxy.auth.fallback_model_access import router_fallback_access_check + + monkeypatch.setattr(proxy_server, "general_settings", {"search_tool_deny_by_default": True}) + monkeypatch.setattr( # test-quality-ok: respx needs HTTPX enabled to fake the provider HTTP boundary. + litellm, + "disable_aiohttp_transport", + True, + ) + monkeypatch.setattr( + proxy_server, + "llm_router", + Router( + model_list=[], + search_tools=[ + {"search_tool_name": "search-a", "litellm_params": {"search_provider": "exa_ai", "api_key": "fake"}}, + {"search_tool_name": "search-b", "litellm_params": {"search_provider": "tavily", "api_key": "fake"}}, + ], + fallbacks=[{"search-a": ["search-b"]}], + fallback_access_check=router_fallback_access_check, + num_retries=0, + ), + ) + litellm.in_memory_llm_clients_cache.flush_cache() + with respx.mock(assert_all_called=False) as mock: + failing_tool: Final = mock.post(url__regex=r"https://api\.exa\.ai/.*").respond(500, json={"error": "down"}) + fallback_tool: Final = mock.post(TAVILY_SEARCH_URL).respond(200, json={"results": [TAVILY_RESULT]}) + response: Final = _client(_standalone_key(key_search_tools)).post( + "/v1/search/search-a", json={"query": "what is litellm"} + ) + litellm.in_memory_llm_clients_cache.flush_cache() + + assert response.status_code == expected_status, response.text + assert failing_tool.call_count == 1 + assert fallback_tool.call_count == fallback_calls + + +@pytest.mark.parametrize( + "requested_tool, completion_model, expected_status", + [("search-b", "search-a", 403), ("search-a", "search-b", 200)], + ids=["requested-tool-ungranted", "completion-model-ungranted"], +) +def test_direct_search_authorizes_the_requested_tool_over_completion_model( + monkeypatch, cache, tavily, requested_tool, completion_model, expected_status +): + monkeypatch.setattr( + proxy_server, "general_settings", {"search_tool_deny_by_default": True, "completion_model": completion_model} + ) + + response: Final = _client(_standalone_key(["search-a"])).post( + f"/v1/search/{requested_tool}", json={"query": "what is litellm"} + ) + + assert response.status_code == expected_status, response.text + assert tavily.call_count == (1 if expected_status == 200 else 0) diff --git a/tests/unit/proxy/test_credential_slot_registry.py b/tests/unit/proxy/test_credential_slot_registry.py index 02268dba361..044b00b33bb 100644 --- a/tests/unit/proxy/test_credential_slot_registry.py +++ b/tests/unit/proxy/test_credential_slot_registry.py @@ -89,6 +89,7 @@ CALLBACK_PARAM_CLASSIFICATION: Final[Mapping[str, Classification]] = MappingProx "arize_space_id": NotSecret("space identifier"), "arize_success_sampling_rate": NotSecret("sampling rate"), "arize_error_sampling_rate": NotSecret("sampling rate"), + "arize_otlp_protocol": NotSecret("transport protocol enum"), "posthog_api_key": Unplanted(), "posthog_api_url": NotSecret("sink endpoint URL"), "wandb_api_key": Unplanted(), diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index db2aec10fdb..e604604025a 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -8590,3 +8590,38 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback(): team_callback_settings_obj=None, ) assert under_other.callback_vars == {"langfuse_host": "https://cloud.langfuse.com"} + + +def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monkeypatch): + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + monkeypatch.setenv("ARIZE_ENDPOINT", "https://arize.internal.example/v1") + monkeypatch.delenv("ARIZE_HTTP_ENDPOINT", raising=False) + is_otel_v2_enabled.cache_clear() + try: + auth = UserAPIKeyAuth( + api_key="hashed-key", + metadata={ + "logging": [ + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": { + "arize_space_id": "s", + "arize_api_key": "k", + "arize_otlp_protocol": "http/protobuf", + }, + } + ] + }, + team_metadata={}, + ) + + destinations = resolve_tenant_otel_destinations(auth) + + assert [d.protocol for d in destinations] == ["otlp_http"] + assert destinations[0].endpoint == "https://arize.internal.example/v1/traces" + finally: + is_otel_v2_enabled.cache_clear() diff --git a/tests/unit/responses/test_streaming_iterator.py b/tests/unit/responses/test_streaming_iterator.py index 2f6dccb37f3..9e9d221ce84 100644 --- a/tests/unit/responses/test_streaming_iterator.py +++ b/tests/unit/responses/test_streaming_iterator.py @@ -646,7 +646,15 @@ def test_stream_cache_write_completes_when_asyncio_run_closes_the_loop(monkeypat status="completed", model="test-model", object="response", - output=[], + output=[ + { + "type": "message", + "id": "msg_lit6184", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "cached", "annotations": []}], + } + ], ), ) monkeypatch.setattr(litellm, "cache", _SlowWriteCache()) @@ -1261,6 +1269,25 @@ def test_billed_terminal_response_copies_when_estimating_and_leaves_the_original assert response.usage is None +def test_persist_completed_response_to_cache_skips_a_response_without_output(monkeypatch): + logging_obj: Final = _logging_obj_stub() + caching_handler: Final = Mock() + caching_handler.request_kwargs = {"stream": True} + logging_obj._llm_caching_handler = caching_handler + iterator: Final = _make_iterator(sse_events=[], logging_obj=logging_obj) + iterator.completed_response = ResponseCompletedEvent.model_construct( + type="response.completed", + response=ResponsesAPIResponse.model_construct(id="resp_empty", output=[], usage=None), + ) + cache: Final = Mock() + monkeypatch.setattr(litellm, "cache", cache) + + iterator._persist_completed_response_to_cache(is_async=False) + + cache.add_cache.assert_not_called() + caching_handler._should_store_result_in_cache.assert_not_called() + + def test_persist_completed_response_to_cache_survives_an_unserializable_response(monkeypatch): bad_response: Final = ResponsesAPIResponse.model_construct(id="r", output=[object()], usage=None) with pytest.raises(PydanticSerializationError): diff --git a/tests/unit/rust_bridge/messages/test_route_host.py b/tests/unit/rust_bridge/messages/test_route_host.py index 1be42e2249d..7b15553a055 100644 --- a/tests/unit/rust_bridge/messages/test_route_host.py +++ b/tests/unit/rust_bridge/messages/test_route_host.py @@ -161,4 +161,4 @@ def test_stream_hidden_params_projects_upstream_headers_the_way_the_python_handl assert isinstance(additional, dict) assert additional["llm_provider-request-id"] == "req_upstream_123" assert additional["x-ratelimit-remaining-requests"] == "41" - assert "request-id" not in additional + assert additional["request-id"] == additional["llm_provider-request-id"] diff --git a/tests/unit/rust_bridge/test_callbacks_legacy_python.py b/tests/unit/rust_bridge/test_callbacks_legacy_python.py index 045dcb52ce9..c78fbb3f246 100644 --- a/tests/unit/rust_bridge/test_callbacks_legacy_python.py +++ b/tests/unit/rust_bridge/test_callbacks_legacy_python.py @@ -34,6 +34,14 @@ def _supplied_logger() -> Logging: ) +def test_native_stream_headers_reach_spend_callbacks() -> None: + logger: Final = _supplied_logger() + legacy.stream_opened(logger, {"additional_headers": {"llm_provider-request-id": "req_native"}}) + + assert logger.stream is True + assert logger.model_call_details["response_headers"] == {"llm_provider-request-id": "req_native"} + + def test_setup_reuses_a_supplied_logger() -> None: supplied: Final = _supplied_logger() result: Final = setup( @@ -159,9 +167,7 @@ def test_failure_handler_of_an_internal_call_leaves_the_outer_budget_reservation pending.close() -CONTRACT_PATH: Final = ( - Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" -) +CONTRACT_PATH: Final = Path(__file__).parents[3] / "litellm-rust/crates/callbacks-legacy-python/python_contract.json" def test_the_rust_contract_matches_the_shim_signatures() -> None: diff --git a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx index 19020d92066..6ee092f030d 100644 --- a/ui/litellm-dashboard/src/components/callback_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/callback_info_helpers.tsx @@ -20,6 +20,7 @@ interface CallbackConfig { supports_key_team_logging: boolean; dynamic_params: Record; dynamic_param_options?: Record; + success_event_params?: string[]; description: string; } @@ -34,7 +35,12 @@ export const CALLBACK_CONFIGS: CallbackConfig[] = [ arize_space_id: "password", arize_success_sampling_rate: "number", arize_error_sampling_rate: "number", + arize_otlp_protocol: "select", }, + dynamic_param_options: { + arize_otlp_protocol: ["grpc", "http/protobuf"], + }, + success_event_params: ["arize_otlp_protocol"], description: "Arize Logging Integration", }, { diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts index 27c46a528e7..f33f86cd280 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts @@ -26,6 +26,8 @@ function makeTrace(scene: Scenario, index: number, now: number) { parent_span_id: null, span_id: spanId(0), spend: null, + spend_log_request_id: null, + spend_match: null, start_offset_ms: 0, status: "ok", type: "agent", @@ -53,6 +55,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { input_tokens: 520 + index * 41, output_tokens: 48 + index * 7, spend: 0.003 + index * 0.0002, + spend_match: "matched", }; const trace: Trace = { summary: { @@ -71,6 +74,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { service: "demo-agents", span_count: toolCount + 2, spend: model.spend, + priced_calls: 1, start_time: iso(now - (index + 1) * 35 * 60_000), status: scene.failed ? "error" : "ok", tool_calls: toolCount, @@ -85,6 +89,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { llm_calls: 1, tool_calls: toolCount, spend: model.spend, + priced_calls: 1, }, ], spans: [base, ...tools, model], diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts index 866f1042b73..44eac42755c 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/lensDemoLongTrace.ts @@ -85,6 +85,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { input_tokens: models.reduce((sum, span) => sum + span.input_tokens, 0), output_tokens: models.reduce((sum, span) => sum + span.output_tokens, 0), spend: models.reduce((sum, span) => sum + (span.spend ?? 0), 0), + priced_calls: models.length, }; const finalDetail = run.details.at(-1)!; const history = caseDetails.filter((_, index) => index % 3 === 2); @@ -102,6 +103,7 @@ export function withReleaseCases(run: { trace: Trace; details: SpanDetail[] }) { llm_calls: caseCount, tool_calls: caseCount, spend: summary.spend - (final.spend ?? 0), + priced_calls: caseCount, }, ], }, diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx index 16afa455be6..33fb073d0a9 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx @@ -10,8 +10,8 @@ import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; import type { TraceHandoff } from "../../api"; +import { runCost } from "../../list/AgentTracesTable"; import { traceRefOf, traceShareUrl } from "../../routing"; -import { formatCost } from "../../list/AgentTracesTable"; import { IdChip } from "../../ui/IdChip"; import { SpanIcon } from "../../ui/SpanIcon"; import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework"; @@ -89,6 +89,7 @@ interface RunHeaderProps { export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) { const { summary } = trace; const failed = summary.status === "error"; + const cost = runCost(summary); return (
@@ -127,7 +128,10 @@ export function RunHeader({ trace, handoff, onBack, embedded }: RunHeaderProps) - + {summary.error_count > 0 && }
diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx index fb9285d0777..cf2c32ff5c7 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/DetailPane.integration.test.tsx @@ -13,10 +13,31 @@ import type { Span, SpanDetail, SpanErrorPage, Trace } from "../../types"; vi.mock("../../../../networking", () => ({ agentTraceSpanCall: vi.fn(), agentTraceSpanErrorCall: vi.fn(), + uiSpendLogsCall: vi.fn(), getProxyBaseUrl: () => "http://proxy.test/", })); -import { agentTraceSpanCall, agentTraceSpanErrorCall } from "../../../../networking"; +vi.mock("../../../../logs/detail", () => ({ + LogDetailsDrawer: ({ + open, + logEntry, + onClose, + backTo, + }: { + open: boolean; + logEntry: { request_id: string } | null; + onClose: () => void; + backTo?: { label: string }; + }) => + open && logEntry ? ( +
+ {logEntry.request_id} + {backTo && } +
+ ) : null, +})); + +import { agentTraceSpanCall, agentTraceSpanErrorCall, uiSpendLogsCall } from "../../../../networking"; type SpanFields = Partial & Pick; @@ -37,6 +58,8 @@ const span = (overrides: SpanFields): Span => ({ output_tokens: 0, litellm_request_id: null, spend: null, + spend_log_request_id: null, + spend_match: null, ...overrides, }); @@ -49,6 +72,9 @@ const llmFields: SpanFields = { input_tokens: 659, output_tokens: 60, litellm_request_id: "chatcmpl-abc", + spend: 0.0002, + spend_log_request_id: "req_7f3a9c2e1b44", + spend_match: "matched", }; const failedToolFields: SpanFields = { span_id: "tool1", @@ -244,13 +270,47 @@ describe("DetailPane", () => { expect(input).toHaveTextContent("acme-404"); }); - it("shows the LiteLLM request facts on the Request tab", async () => { + it("links a priced LLM step to its spend log in the header and opens it by that request id", async () => { const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); renderPane(spanRow(llm)); + const link = screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" }); + expect(link).toHaveTextContent("LiteLLM Spend Log$0.0002"); + await user.click(link); + expect(await screen.findByRole("dialog", { name: "Request log" })).toHaveTextContent("req_7f3a9c2e1b44"); + expect(vi.mocked(uiSpendLogsCall).mock.calls[0][0].params).toEqual({ request_id: "req_7f3a9c2e1b44" }); + }); + + it("returns from the spend log to the same agent trace step", async () => { + const user = userEvent.setup(); + vi.mocked(uiSpendLogsCall).mockResolvedValue({ data: [{ request_id: "req_7f3a9c2e1b44" }] }); + renderPane(spanRow(llm)); + await user.click(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })); + const drawer = await screen.findByRole("dialog", { name: "Request log" }); + await user.click(within(drawer).getByRole("button", { name: "Back to Lens trace" })); + await waitFor(() => expect(screen.queryByRole("dialog", { name: "Request log" })).not.toBeInTheDocument()); + expect(screen.getByRole("button", { name: "Open LiteLLM spend log req_7f3a9c2e1b44" })).toBeVisible(); + }); + + it.each([ + { spend_match: "no_call_id" as const, reason: "This step records no usable call identifier or transport context" }, + { spend_match: "no_spend_log" as const, reason: "No matching spend log belongs to this run" }, + { spend_match: "incomplete_evidence" as const, reason: "This step does not account for every request in the call" }, + ])("says why an unpriced LLM step has no cost ($spend_match)", async ({ spend_match, reason }) => { + const user = userEvent.setup(); + const unpriced: SpanFields = { ...llmFields, spend: null, spend_log_request_id: null, spend_match }; + renderPane(spanRow(span(unpriced))); + expect(screen.getByTitle(reason)).toHaveTextContent("Cost not matched"); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); await user.click(screen.getByRole("tab", { name: "Request" })); - expect(await screen.findByText("chatcmpl-abc")).toBeInTheDocument(); - expect(screen.getByText("659")).toBeInTheDocument(); - expect(screen.getByRole("button", { name: /Open request log/ })).toBeInTheDocument(); + expect(await screen.findByText(reason, { selector: "p" })).toBeInTheDocument(); + }); + + it("keeps the Cost fact for a priced step that has no single spend log to link", () => { + const pricedFromRetries: SpanFields = { ...llmFields, spend: 0.0005, spend_log_request_id: null }; + renderPane(spanRow(span(pricedFromRetries))); + expect(screen.queryByRole("button", { name: /Open LiteLLM spend log/ })).not.toBeInTheDocument(); + expect(screen.getByText("Cost").parentElement).toHaveTextContent("Cost$0.0005"); }); it("summarizes a ×N group with its failure pattern", () => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx index 4757c55ab80..240ea149199 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/PaneHeader.tsx @@ -16,11 +16,12 @@ interface PaneHeaderProps { idValue?: string; facts: readonly (readonly [label: string, value: string])[]; actions?: React.ReactNode; + links?: React.ReactNode; onClose: () => void; } /** Selected step identity: tile, name, id, then its time and usage on one quiet line. */ -export function PaneHeader({ type, model, failed, title, idValue, facts, actions, onClose }: PaneHeaderProps) { +export function PaneHeader({ type, model, failed, title, idValue, facts, actions, links, onClose }: PaneHeaderProps) { return (
@@ -40,15 +41,18 @@ export function PaneHeader({ type, model, failed, title, idValue, facts, actions
- {facts.length > 0 && ( -
- {facts.map(([label, value]) => ( -
-
{label}
-
{value}
-
- ))} -
+ {(facts.length > 0 || links) && ( +
+
+ {facts.map(([label, value]) => ( +
+
{label}
+
{value}
+
+ ))} +
+ {links} +
)} ); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx index 855a7060336..cd36d2686ef 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/RequestTab.tsx @@ -1,19 +1,14 @@ "use client"; -import { ArrowUpRight } from "lucide-react"; -import { useState } from "react"; - import CopyButton from "@/components/shared/CopyButton"; -import { Button } from "@/components/ui/button"; -import { LogDetailsDrawer } from "../../../../logs/detail"; import { fieldEntries } from "../content/payload"; import { formatCost } from "../../list/AgentTracesTable"; import type { Span } from "../../types"; import { fmtMs, fmtTok } from "../../utils"; -import { useSpanRequestLog } from "../useSpanRequestLog"; import { FieldTree } from "../content/FieldTree"; import { DetailGroup } from "./DetailGroup"; +import { SpendLogLink, unmatchedReason } from "./SpendLogLink"; interface RequestTabProps { span: Span; @@ -22,17 +17,7 @@ interface RequestTabProps { traceStartMs: number; } -/** Usage and the linked LiteLLM request for an LLM span; "Open request log" opens the request drawer over the run. */ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) { - const [drawerOpen, setDrawerOpen] = useState(false); - const logQuery = useSpanRequestLog( - accessToken, - span.litellm_request_id, - traceStartMs + span.start_offset_ms, - drawerOpen, - ); - const logNotFound = drawerOpen && logQuery.isSuccess && logQuery.data === null; - if (span.type !== "llm") { return
This span is not a model request.
; } @@ -51,30 +36,22 @@ export function RequestTab({ span, accessToken, traceStartMs }: RequestTabProps) - - {span.litellm_request_id ? ( -
+ +
+ {span.spend_log_request_id && (
- +
- +
- - {logNotFound &&

No request log found for this call.

} + )} +
+
- ) : ( -

Not linked to a LiteLLM request

- )} + {unmatchedReason(span) &&

{unmatchedReason(span)}

} +
- setDrawerOpen(false)} - logEntry={logQuery.data ?? null} - accessToken={accessToken} - />
); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx index cfa92f67577..d74d7ad8ccd 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/span/SpanPane.tsx @@ -12,6 +12,7 @@ import { ContentTab, useSpanDetail } from "../content/ContentTab"; import { AttributesTab } from "./AttributesTab"; import { PaneHeader } from "./PaneHeader"; import { RequestTab } from "./RequestTab"; +import { SpendLogLink } from "./SpendLogLink"; const TAB_LABELS: Record = { content: "Content", request: "Request", attributes: "Attributes" }; @@ -26,7 +27,7 @@ const spanFacts = (span: Span): readonly Fact[] => { const tokens = span.input_tokens + span.output_tokens; const optional: readonly (Fact | null)[] = [ tokens > 0 ? ["Tokens", fmtTok(tokens)] : null, - span.spend != null ? ["Cost", formatCost(span.spend)] : null, + span.spend != null && span.spend_log_request_id == null ? ["Cost", formatCost(span.spend)] : null, span.type === "llm" && span.model ? ["Step", span.name] : null, ]; return [["Duration", fmtMs(span.duration_ms)], ...optional.filter((fact): fact is Fact => fact !== null)]; @@ -42,6 +43,7 @@ export function SpanPane({ }: SpanTabProps & { trace: Trace; span: Span; accessToken: string; onClose: () => void }) { const { trace_id: traceId, trace_ref: traceRef, start_time: startTime } = trace.summary; const handoff = useTracesApi(accessToken).handoff(traceId, span.span_id, traceRef); + const traceStartMs = Date.parse(startTime); const detailQuery = useSpanDetail(accessToken, traceId, tab === "attributes" ? span.span_id : null, traceRef); return (