diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 02b48a3be3b..98f36607fff 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4083,6 +4083,7 @@ dependencies = [ name = "litellm-inference-responses" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "futures-util", "litellm-auth", diff --git a/litellm-rust/crates/inference-responses/AGENTS.md b/litellm-rust/crates/inference-responses/AGENTS.md index 1dd199024b0..d7b58badf19 100644 --- a/litellm-rust/crates/inference-responses/AGENTS.md +++ b/litellm-rust/crates/inference-responses/AGENTS.md @@ -2,3 +2,4 @@ - This crate owns Responses API call orchestration. Payloads belong in `litellm-llms-types`, transformations in `llms/src/base_llm/responses` and `llms/src//responses` - The HTTP route returns `litellm_host::call::CallOutput`: a completed response, or a stream head and chunks, cached through `litellm_inference::caching::execute_streaming` - WebSocket sessions (`src/websocket.rs`) stay separate from the HTTP call driver because a connection can accept multiple requests while receiving events +- `src/ids.rs` owns the LiteLLM-managed ids handed back to callers (`resp_`, `encitem_`, `litellm_enc:`, `cntr_`) and must stay fully compatible with `litellm/responses/utils.py` diff --git a/litellm-rust/crates/inference-responses/Cargo.toml b/litellm-rust/crates/inference-responses/Cargo.toml index 255518b9c54..793483ee76d 100644 --- a/litellm-rust/crates/inference-responses/Cargo.toml +++ b/litellm-rust/crates/inference-responses/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-responses/src/ids.rs b/litellm-rust/crates/inference-responses/src/ids.rs new file mode 100644 index 00000000000..b51aea1559e --- /dev/null +++ b/litellm-rust/crates/inference-responses/src/ids.rs @@ -0,0 +1,499 @@ +//! LiteLLM-managed identifiers on Responses payloads, fully compatible with `litellm/responses/utils.py` +//! +//! LiteLLM rewrites some ids in a Responses API result before handing it back, so that a later +//! request carrying those ids can be routed to the provider and deployment that produced them, +//! and then restored to what the provider originally issued. The native and Python paths serve +//! the same traffic, so an id minted by either one must decode on the other. +//! +//! Each managed id is a fixed prefix followed by standard, padded base64 of a `;`-separated +//! plaintext: +//! +//! | Id | Applied to | Decoded plaintext | +//! |---|---|---| +//! | `resp_` | the response `id` | `litellm:custom_llm_provider:{provider};model_id:{model_id};response_id:{id}` | +//! | `encitem_` | the `id` of an output item with `encrypted_content` | `litellm:model_id:{model_id};item_id:{id}` | +//! | `litellm_enc:;{content}` | that item's `encrypted_content` | `` is `model_id:{model_id}`, and the original content follows the `;` unencoded | +//! | `cntr_` | `container_id` on output items, `code_interpreter_call`, and message annotations | `litellm:custom_llm_provider:{provider};model_id:{model_id};container_id:{id}` | +//! +//! For example, OpenAI's `resp_0a1b2c3d4e5f` served without a deployment id becomes +//! `resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm`, +//! which decodes to `litellm:custom_llm_provider:openai;model_id:None;response_id:resp_0a1b2c3d4e5f`. +//! +//! How they fit together: +//! - `resp_` lets a follow-up's `previous_response_id` find the provider and deployment again. +//! [`encode_response`] stamps it on every response, unless the id is already managed. +//! - `encitem_` and `litellm_enc:` carry encrypted-content affinity: encrypted reasoning only +//! decrypts under the API key that produced it, so the router pins replays to that deployment. +//! They are stamped only when the router sets `encrypted_content_affinity_enabled` and a +//! deployment `model_id` is known (see [`IdScope`]). Items without an `id`, such as Codex's, +//! carry the deployment in the wrapped content alone. +//! - `cntr_` lets container calls find the provider and deployment that own a container. A +//! container id that already names a provider is left as it is. +//! - On the way in, [`restore_input`] turns `encitem_` ids and `litellm_enc:` content in the +//! request input back into the provider's own bytes before the request is sent. The routing +//! decision itself stays in Python's `EncryptedContentAffinityCheck`. +//! +//! Python's quirks are kept on purpose: +//! - A missing provider or `model_id` is written as `None` in `resp_` ids (an f-string of +//! `None`), but as an empty string in `cntr_` ids. [`decode_container_id`] reads both as absent. +//! - Decoding follows Python's non-strict `base64.b64decode`: characters outside the alphabet +//! are skipped, trailing bits are allowed, and padding stripped in transit is added back. +//! - A `resp_` id counts as managed when it decodes to at least three `;`-separated parts. + +use base64::{ + Engine, alphabet, + engine::{GeneralPurpose, general_purpose::PAD}, +}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use serde_json::{Map, Value}; + +const RESPONSE_PREFIX: &str = "resp_"; +const ITEM_PREFIX: &str = "encitem_"; +const CONTAINER_PREFIX: &str = "cntr_"; +const WRAPPED_CONTENT_PREFIX: &str = "litellm_enc:"; + +/// Python's non-strict `base64.b64decode`: characters outside the alphabet are skipped and +/// non-zero trailing bits are accepted +const LENIENT: GeneralPurpose = GeneralPurpose::new( + &alphabet::STANDARD, + PAD.with_decode_allow_trailing_bits(true), +); + +/// Where a response came from, as encoded into the ids LiteLLM hands back +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IdScope { + pub provider: Option, + pub model_id: Option, + pub encrypted_content_affinity: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DecodedContainerId { + pub provider: Option, + pub model_id: Option, + pub container_id: String, +} + +/// Rewrites the response id, encrypted-content items and container ids the way the Python +/// Responses path does before returning to the caller. An already managed id is left untouched +pub fn encode_response(response: &mut ResponsesApiResponse, scope: &IdScope) { + if is_managed_response_id(&response.id) { + return; + } + response.id = response_id(scope, &response.id); + if let Some(model_id) = scope + .model_id + .as_deref() + .filter(|_| scope.encrypted_content_affinity) + .filter(|model_id| !model_id.is_empty()) + { + response + .output + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|item| encode_encrypted_item(item, model_id)); + } + response + .output + .iter_mut() + .for_each(|item| encode_containers(item, scope)); +} + +/// Restores encrypted-content item ids and contents that LiteLLM encoded on an earlier response, +/// so the provider receives what it originally issued +pub fn restore_input(input: &mut Value) { + let Some(items) = input.as_array_mut() else { + return; + }; + for item in items.iter_mut().filter_map(Value::as_object_mut) { + if let Some(Value::String(id)) = item.get_mut("id") + && let Some((_, original)) = decode_encrypted_item_id(id) + { + *id = original; + } + if let Some(Value::String(content)) = item.get_mut("encrypted_content") + && let Some(original) = unwrap_encrypted_content(content) + { + *content = original; + } + } +} + +pub fn response_id(scope: &IdScope, id: &str) -> String { + let assembled = format!( + "litellm:custom_llm_provider:{};model_id:{};response_id:{id}", + python_str(scope.provider.as_deref()), + python_str(scope.model_id.as_deref()), + ); + format!("{RESPONSE_PREFIX}{}", LENIENT.encode(assembled)) +} + +pub fn is_managed_response_id(id: &str) -> bool { + decode_text(&id.replace(RESPONSE_PREFIX, "")) + .is_some_and(|decoded| decoded.split(';').count() >= 3) +} + +pub fn encrypted_item_id(model_id: &str, id: &str) -> String { + format!( + "{ITEM_PREFIX}{}", + LENIENT.encode(format!("litellm:model_id:{model_id};item_id:{id}")) + ) +} + +pub fn decode_encrypted_item_id(id: &str) -> Option<(String, String)> { + let decoded = decode_text(&padded(id.strip_prefix(ITEM_PREFIX)?))?; + let (model_id, item_id) = decoded.split_once(';')?; + Some(( + model_id.replace("litellm:model_id:", ""), + item_id.replace("item_id:", ""), + )) +} + +pub fn wrap_encrypted_content(content: &str, model_id: &str) -> String { + format!( + "{WRAPPED_CONTENT_PREFIX}{};{content}", + LENIENT.encode(format!("model_id:{model_id}")) + ) +} + +/// The original content of a wrapped `encrypted_content`, or `None` when it is not wrapped +pub fn unwrap_encrypted_content(content: &str) -> Option { + if !content.starts_with(WRAPPED_CONTENT_PREFIX) { + return None; + } + let (metadata, original) = content.split_once(';')?; + decode_text(&padded(&metadata.replace(WRAPPED_CONTENT_PREFIX, "")))?; + Some(original.to_owned()) +} + +pub fn container_id(scope: &IdScope, id: &str) -> String { + let assembled = format!( + "litellm:custom_llm_provider:{};model_id:{};container_id:{id}", + scope.provider.as_deref().unwrap_or_default(), + scope.model_id.as_deref().unwrap_or_default(), + ); + format!("{CONTAINER_PREFIX}{}", LENIENT.encode(assembled)) +} + +pub fn decode_container_id(id: &str) -> DecodedContainerId { + let unmanaged = || DecodedContainerId { + provider: None, + model_id: None, + container_id: id.to_owned(), + }; + if !id.starts_with(CONTAINER_PREFIX) { + return unmanaged(); + } + let Some(decoded) = decode_text(&id.replace(CONTAINER_PREFIX, "")) else { + return unmanaged(); + }; + let parsed = decoded + .strip_prefix("litellm:custom_llm_provider:") + .and_then(|rest| rest.split_once(';')) + .and_then(|(provider, rest)| { + let (model_id, rest) = rest.strip_prefix("model_id:")?.split_once(';')?; + let container_id = rest.strip_prefix("container_id:")?; + (!container_id.is_empty() && !container_id.contains('\n')).then_some(( + provider, + model_id, + container_id, + )) + }); + let Some((provider, model_id, container_id)) = parsed else { + return unmanaged(); + }; + let present = |value: &str| (!matches!(value, "" | "None")).then(|| value.to_owned()); + DecodedContainerId { + provider: present(provider), + model_id: present(model_id), + container_id: container_id.to_owned(), + } +} + +fn encode_encrypted_item(item: &mut Map, model_id: &str) { + let Some(Value::String(content)) = item.get_mut("encrypted_content") else { + return; + }; + if content.is_empty() { + return; + } + *content = wrap_encrypted_content(content, model_id); + if let Some(Value::String(id)) = item.get_mut("id") + && !id.is_empty() + { + *id = encrypted_item_id(model_id, id); + } +} + +fn encode_containers(item: &mut Value, scope: &IdScope) { + let Some(item) = item.as_object_mut() else { + return; + }; + encode_container_field(item, scope); + if let Some(nested) = item + .get_mut("code_interpreter_call") + .and_then(Value::as_object_mut) + { + encode_container_field(nested, scope); + } + if item.get("type").and_then(Value::as_str) != Some("message") { + return; + } + let Some(content) = item.get_mut("content").and_then(Value::as_array_mut) else { + return; + }; + content + .iter_mut() + .filter_map(|part| part.get_mut("annotations").and_then(Value::as_array_mut)) + .flatten() + .for_each(|annotation| encode_containers(annotation, scope)); +} + +fn encode_container_field(item: &mut Map, scope: &IdScope) { + if let Some(Value::String(id)) = item.get_mut("container_id") + && decode_container_id(id).provider.is_none() + { + *id = container_id(scope, id); + } +} + +fn python_str(value: Option<&str>) -> &str { + value.unwrap_or("None") +} + +fn padded(value: &str) -> String { + let missing = value.len() % 4; + let padding = if missing == 0 { 0 } else { 4 - missing }; + format!("{value}{}", "=".repeat(padding)) +} + +fn decode_text(value: &str) -> Option { + let alphabet: String = value + .chars() + .filter(|character| { + character.is_ascii_alphanumeric() || matches!(character, '+' | '/' | '=') + }) + .collect(); + String::from_utf8(LENIENT.decode(alphabet).ok()?).ok() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn scope(provider: Option<&str>, model_id: Option<&str>, affinity: bool) -> IdScope { + IdScope { + provider: provider.map(str::to_owned), + model_id: model_id.map(str::to_owned), + encrypted_content_affinity: affinity, + } + } + + fn response(id: &str, output: Value) -> ResponsesApiResponse { + serde_json::from_value(json!({"id": id, "model": "gpt-6.1-sol", "output": output})).unwrap() + } + + #[rstest] + #[case::openai_without_model_id( + Some("openai"), + None, + "resp_0a1b2c3d4e5f", + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOnJlc3BfMGExYjJjM2Q0ZTVm" + )] + #[case::openrouter_with_model_id( + Some("openrouter"), + Some("deploy-1"), + "gen-123", + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5yb3V0ZXI7bW9kZWxfaWQ6ZGVwbG95LTE7cmVzcG9uc2VfaWQ6Z2VuLTEyMw==" + )] + fn response_ids_match_python( + #[case] provider: Option<&str>, + #[case] model_id: Option<&str>, + #[case] id: &str, + #[case] expected: &str, + ) { + assert_eq!(response_id(&scope(provider, model_id, false), id), expected); + } + + #[rstest] + #[case::raw_short_id("resp_0a1b2c3d4e5f", false)] + #[case::raw_long_id("resp_68a1b2c3d4e5f6a7b8c9d0e1f2a3b4c5d6e7f8a9b0c1d2e3", false)] + #[case::two_segments("resp_YTti", false)] + #[case::three_segments("resp_YTtiO2M=", true)] + #[case::managed( + "resp_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpOb25lO3Jlc3BvbnNlX2lkOng=", + true + )] + fn managed_response_ids_match_python(#[case] id: &str, #[case] expected: bool) { + assert_eq!(is_managed_response_id(id), expected); + } + + #[test] + fn encrypted_items_match_python() { + assert_eq!( + encrypted_item_id("deploy-1", "rs_0a1b"), + "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=" + ); + assert_eq!( + wrap_encrypted_content("gAAAA", "deploy-1"), + "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA" + ); + } + + #[rstest] + #[case::padded("encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", Some(("deploy-1", "rs_0a1b")))] + #[case::padding_stripped_in_transit("encitem_bGl0ZWxsbTptb2RlbF9pZDptO2l0ZW1faWQ6cnNfMQ", Some(("m", "rs_1")))] + #[case::not_base64("encitem_notbase64!", None)] + #[case::unmanaged("rs_0a1b", None)] + fn encrypted_item_ids_decode_like_python( + #[case] id: &str, + #[case] expected: Option<(&str, &str)>, + ) { + assert_eq!( + decode_encrypted_item_id(id), + expected.map(|(model_id, item_id)| (model_id.to_owned(), item_id.to_owned())) + ); + } + + #[rstest] + #[case::unpadded_metadata("litellm_enc:bW9kZWxfaWQ6bQ;gAAAA", Some("gAAAA"))] + #[case::no_separator("litellm_enc:nosemicolon", None)] + #[case::unwrapped("gAAAA", None)] + fn encrypted_content_unwraps_like_python( + #[case] content: &str, + #[case] expected: Option<&str>, + ) { + assert_eq!(unwrap_encrypted_content(content).as_deref(), expected); + } + + #[rstest] + #[case::with_model_id( + Some("deploy-1"), + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDpkZXBsb3ktMTtjb250YWluZXJfaWQ6Y250cl9hYmM=" + )] + #[case::without_model_id( + None, + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfYWJj" + )] + fn container_ids_match_python(#[case] model_id: Option<&str>, #[case] expected: &str) { + assert_eq!( + container_id(&scope(Some("openai"), model_id, false), "cntr_abc"), + expected + ); + } + + #[rstest] + #[case::raw("cntr_abc", None, None, "cntr_abc")] + #[case::managed( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOm9wZW5haTttb2RlbF9pZDptO2NvbnRhaW5lcl9pZDpjbnRyXzE=", + Some("openai"), + Some("m"), + "cntr_1" + )] + #[case::empty_segments( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOjttb2RlbF9pZDo7Y29udGFpbmVyX2lkOmNudHJfMQ==", + None, + None, + "cntr_1" + )] + #[case::python_none_provider( + "cntr_bGl0ZWxsbTpjdXN0b21fbGxtX3Byb3ZpZGVyOk5vbmU7bW9kZWxfaWQ6O2NvbnRhaW5lcl9pZDp4", + None, + None, + "x" + )] + fn container_ids_decode_like_python( + #[case] id: &str, + #[case] provider: Option<&str>, + #[case] model_id: Option<&str>, + #[case] container_id: &str, + ) { + assert_eq!( + decode_container_id(id), + DecodedContainerId { + provider: provider.map(str::to_owned), + model_id: model_id.map(str::to_owned), + container_id: container_id.to_owned(), + } + ); + } + + #[test] + fn encodes_response_items_under_affinity() { + let mut encoded = response( + "resp_1", + json!([ + {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []}, + {"type": "reasoning", "encrypted_content": "gBBBB", "summary": []}, + {"type": "code_interpreter_call", "id": "ci_1", "container_id": "cntr_abc"}, + {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [ + {"type": "container_file_citation", "container_id": "cntr_abc"} + ]}]} + ]), + ); + let scope = scope(Some("openai"), Some("deploy-1"), true); + encode_response(&mut encoded, &scope); + let container = container_id(&scope, "cntr_abc"); + assert_eq!(encoded.id, response_id(&scope, "resp_1")); + assert_eq!( + Value::Array(encoded.output), + json!([ + { + "type": "reasoning", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA", + "summary": [] + }, + {"type": "reasoning", "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gBBBB", "summary": []}, + {"type": "code_interpreter_call", "id": "ci_1", "container_id": container}, + {"type": "message", "id": "msg_1", "content": [{"type": "output_text", "text": "", "annotations": [ + {"type": "container_file_citation", "container_id": container} + ]}]} + ]) + ); + } + + #[rstest] + #[case::affinity_disabled(scope(Some("openai"), Some("deploy-1"), false))] + #[case::no_model_id(scope(Some("openai"), None, true))] + fn leaves_encrypted_items_without_affinity(#[case] scope: IdScope) { + let output = json!([{"type": "reasoning", "id": "rs_1", "encrypted_content": "gAAAA", "summary": []}]); + let mut encoded = response("resp_1", output.clone()); + encode_response(&mut encoded, &scope); + assert_eq!(Value::Array(encoded.output), output); + } + + #[test] + fn leaves_managed_responses_untouched() { + let scope = scope(Some("openai"), Some("deploy-1"), true); + let id = response_id(&scope, "resp_1"); + let output = json!([{"type": "code_interpreter_call", "container_id": "cntr_abc"}]); + let mut encoded = response(&id, output.clone()); + encode_response(&mut encoded, &scope); + assert_eq!((encoded.id, Value::Array(encoded.output)), (id, output)); + } + + #[test] + fn restores_encrypted_items_in_input() { + let mut input = json!([ + { + "type": "reasoning", + "id": "encitem_bGl0ZWxsbTptb2RlbF9pZDpkZXBsb3ktMTtpdGVtX2lkOnJzXzBhMWI=", + "encrypted_content": "litellm_enc:bW9kZWxfaWQ6ZGVwbG95LTE=;gAAAA", + "summary": [] + }, + {"role": "user", "content": "hi"} + ]); + restore_input(&mut input); + assert_eq!( + input, + json!([ + {"type": "reasoning", "id": "rs_0a1b", "encrypted_content": "gAAAA", "summary": []}, + {"role": "user", "content": "hi"} + ]) + ); + } +} diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index 431982457f6..11939823b05 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -1,6 +1,7 @@ use litellm_host::observation::ObservationSender; pub use litellm_inference::RouteError as Error; +pub mod ids; pub mod route; pub mod types; pub mod websocket; diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 8261e66406b..9c430ba7868 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,22 +1,41 @@ use std::convert::Infallible; use crate::routes::inference::InferenceHost; -use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; -use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall}; +use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; +use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py}; +use litellm_inference_responses::{ + Error, + ids::{self, IdScope}, + route::Responses, + types::ResponsesCall, +}; use pyo3::{ gc::{PyTraverseError, PyVisit}, prelude::*, types::PyDict, }; -pub(super) struct ResponsesPythonHost(pub InferenceHost); +pub(super) struct ResponsesPythonHost { + host: InferenceHost, + ids: IdScope, +} + +impl ResponsesPythonHost { + pub fn new(host: InferenceHost) -> Self { + Self { + host, + ids: IdScope::default(), + } + } +} pub(super) fn project( host: &mut InferenceHost, py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> PyResult { - let call = host.project(py, arguments, "input")?; + let mut call = host.project(py, arguments, "input")?; + ids::restore_input(&mut call.input); let optional_params = call .params .into_iter() @@ -50,6 +69,39 @@ pub(super) fn project( }) } +/// The provider and deployment Python encodes into the ids it returns: the resolved provider and +/// `litellm_metadata.model_info.id` +fn id_scope( + host: &InferenceHost, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + call: &ResponsesCall, +) -> PyResult { + let metadata = host.argument(py, arguments, "litellm_metadata")?; + let encrypted_content_affinity = match &metadata { + Some(metadata) => metadata + .call_method1("get", ("encrypted_content_affinity_enabled",))? + .is_truthy()?, + None => false, + }; + let metadata = metadata + .map(|value| from_py::(&value)) + .transpose()? + .unwrap_or_default(); + let model_id = match metadata.pointer("/model_info/id") { + None | Some(serde_json::Value::Null) => None, + Some(serde_json::Value::String(id)) => Some(id.clone()), + Some(other) => Some(other.to_string()), + }; + Ok(IdScope { + provider: get_custom_llm_provider(&call.model, call.custom_llm_provider.as_deref()) + .map(|resolved| resolved.custom_llm_provider.to_owned()) + .or_else(|| call.custom_llm_provider.clone()), + model_id, + encrypted_content_affinity, + }) +} + impl PythonBinding for ResponsesPythonHost { type Protocol = Responses; type Failure = PyErr; @@ -59,7 +111,8 @@ impl PythonBinding for ResponsesPythonHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> Result> { - let call = project(&mut self.0, py, arguments).map_err(InvokeError::Python)?; + let call = project(&mut self.host, py, arguments).map_err(InvokeError::Python)?; + self.ids = id_scope(&self.host, py, arguments, &call).map_err(InvokeError::Python)?; if call .optional_params .get("stream") @@ -77,7 +130,9 @@ impl PythonBinding for ResponsesPythonHost { py: Python<'_>, response: ::Response, ) -> PyResult> { - self.0.response(py, &response) + let mut response = response; + ids::encode_response(&mut response, &self.ids); + self.host.response(py, &response) } fn encode_stream_head( @@ -103,7 +158,7 @@ impl PythonBinding for ResponsesPythonHost { } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { - self.0.error(py, error) + self.host.error(py, error) } fn host_error(error: &PyErr) -> Error { Error::InvalidRequest(error.to_string().into()) @@ -122,9 +177,9 @@ impl PythonHostCalls for ResponsesPythonHost { impl PythonOwned for ResponsesPythonHost { fn close(&mut self, _: Python<'_>) { - self.0.close(); + self.host.close(); } fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { - self.0.traverse(visit) + self.host.traverse(visit) } } diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 984f1e74df2..9caeae6c3da 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -62,7 +62,7 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py )); } let host = InferenceHost::new(call.view()?, ROUTE_HOST_MODULE); - run_inference::(py, call, asynchronous, ResponsesPythonHost(host)) + run_inference::(py, call, asynchronous, ResponsesPythonHost::new(host)) } #[pyfunction] diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 0c8efcd36c1..0ed5cf0d01d 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -44,6 +44,16 @@ _COMPARED_LOG_FIELDS: Final = ( "status", ) +TEXT_MESSAGE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + "id": "msg_0a1b2c3d4e5f", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Paris"}], + } +) + TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( { "id": "resp_0a1b2c3d4e5f", @@ -61,13 +71,7 @@ TEXT_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( "type": "reasoning", "summary": [], }, - { - "id": "msg_0a1b2c3d4e5f", - "type": "message", - "status": "completed", - "role": "assistant", - "content": [{"type": "output_text", "annotations": [], "logprobs": [], "text": "Paris"}], - }, + dict(TEXT_MESSAGE), ], "parallel_tool_calls": True, "previous_response_id": None, @@ -122,6 +126,36 @@ WEATHER_TOOL: Final[Mapping[str, JsonValue]] = MappingProxyType( ) +ENCRYPTED_REASONING_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( + { + **TEXT_RESPONSE, + "output": [ + {"id": "rs_0a1b2c3d4e5f", "type": "reasoning", "summary": [], "encrypted_content": "gAAAAB0a1b2c3d"}, + { + "id": "ci_0a1b2c3d4e5f", + "type": "code_interpreter_call", + "status": "completed", + "code": "print(1)", + "container_id": "cntr_0a1b2c3d4e5f", + "outputs": [], + }, + dict(TEXT_MESSAGE), + ], + } +) + +AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ + { + "type": "reasoning", + "id": ResponsesAPIRequestUtils._build_encrypted_item_id("deploy-1", "rs_0a1b2c3d4e5f"), # pyright: ignore[reportPrivateUsage] # the encoder Python applies on the way out + "summary": [], + "encrypted_content": ResponsesAPIRequestUtils.wrap_encrypted_content_with_model_id( + "gAAAAB0a1b2c3d", "deploy-1" + ), + }, + {"role": "user", "content": "And Germany?"}, +] + REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ {"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None}, {"role": "user", "content": "And Germany?"}, @@ -157,30 +191,28 @@ class Case: yield pytest.param(self, aspect, id=f"{self.name}-{aspect}", marks=marks) -ID_ENCODING_GAP: Final = MappingProxyType( - {"response": "native responses return the provider's raw response id instead of LiteLLM's encoded resp_ id"} +DEPLOYMENT_METADATA: Final = MappingProxyType( + {"model_info": {"id": "deploy-1"}, "encrypted_content_affinity_enabled": True} ) -def _gaps(**request_gaps: str) -> Mapping[str, str]: - return MappingProxyType({**ID_ENCODING_GAP, **request_gaps}) +def _gaps(**gaps: str) -> Mapping[str, str]: + return MappingProxyType(gaps) CASES: Final = ( - Case("plain_text", {"input": "What is the capital of France?"}, gaps=_gaps()), + Case("plain_text", {"input": "What is the capital of France?"}), Case( "instructions_and_input_items", { "instructions": "Answer in one word", "input": [{"role": "user", "content": [{"type": "input_text", "text": "Capital of France?"}]}], }, - gaps=_gaps(), ), Case( "function_tool", {"input": "Weather in Paris?", "tools": [dict(WEATHER_TOOL)], "tool_choice": "auto"}, upstream=FUNCTION_CALL_RESPONSE, - gaps=_gaps(), ), Case( "encoded_previous_response_id", @@ -190,7 +222,16 @@ CASES: Final = ( custom_llm_provider="openai", model_id=None, response_id="resp_6f5e4d3c2b1a" ), }, - gaps=_gaps(), + ), + Case( + "deployment_scoped_ids", + {"input": "hi", "litellm_metadata": dict(DEPLOYMENT_METADATA)}, + upstream=ENCRYPTED_REASONING_RESPONSE, + ), + Case( + "affinity_encoded_reasoning_replay", + {"input": AFFINITY_REPLAY_INPUT, "litellm_metadata": dict(DEPLOYMENT_METADATA)}, + upstream=ENCRYPTED_REASONING_RESPONSE, ), Case( "warm_up_probe_below_minimum_max_output_tokens", @@ -215,7 +256,6 @@ CASES: Final = ( ], "tools": [{**WEATHER_TOOL, "cache_control": {"type": "ephemeral"}}], }, - gaps=_gaps(), ), Case( "reasoning_replay_without_status",