diff --git a/litellm-rust/README.md b/litellm-rust/README.md index c5470d6845f..eb2f459c2b3 100644 --- a/litellm-rust/README.md +++ b/litellm-rust/README.md @@ -133,7 +133,7 @@ fixtures. These generic proofs complement, rather than replace, native OCR private proof tests The standard-library-only tests in -`crates/python-interop/tests/callback_patterns.rs` define small inline Python +`crates/python-interop/tests/synthetic/patterns.rs` define small inline Python callbacks, with Rust controlling invocation, ownership and assertions. They compare Python-reference and Rust-retained calls using both direct and awaited invocation. They model the behavior @@ -147,7 +147,7 @@ Existing synthetic lifecycle cases also cover streams, context and cancellation Run this matrix without LiteLLM, vendor SDKs, credentials or services: ```bash -cargo test --manifest-path litellm-rust/Cargo.toml -p litellm-python-interop --test callback_patterns +cargo test --manifest-path litellm-rust/Cargo.toml -p litellm-python-interop --test synthetic ``` These are behavioral models, not tests of vendor authentication, delivery or @@ -155,6 +155,26 @@ production dispatcher policy. The optional component and integration fixtures exercise existing LiteLLM implementations with fake transports and credentials as supplementary coverage; run them with `make test-rust-python` +The interop tests have two explicit Cargo targets, each rooted in its directory's +`mod.rs`: `tests/synthetic/` for custom, minimal Python implementations and +`tests/integration/` for real LiteLLM components with fake transports. The latter +are component-level compatibility tests, not complete SDK or proxy route tests. +Shared Rust fixtures live in `tests/support/`, and Python scenarios live in +`tests/fixtures/` + +Use `#[fixture]` composition for setup: initialize Python once, but create a fresh +scenario scope and owner factory for each case. Use named `#[case::behavior]` +entries for scenarios and `#[values(Backend::Python, Backend::PreparedCall)]` for +the invocation matrix. Keep copying and ownership controls alongside the behavior +they distinguish. Register new Rust modules in the appropriate `mod.rs`; Cargo +test autodiscovery is disabled so new files cannot silently become a third group + +Run `--test synthetic` for the standard-library-only group. Integration cases +remain explicitly ignored without the repository Python environment; +`make test-rust-python` configures that environment and runs both groups with +`--include-ignored`. To inspect the groups without running them, use +`cargo test --manifest-path litellm-rust/Cargo.toml -p litellm-python-interop --tests -- --list` + The callback lifecycle scenarios use `#[serial(python_interpreter)]` to isolate CPython GC and interpreter-wide LiteLLM settings under `cargo test`. Compatible tests in the same binary use diff --git a/litellm-rust/crates/python-interop/Cargo.toml b/litellm-rust/crates/python-interop/Cargo.toml index 3e7d87dc760..5b8cf986a27 100644 --- a/litellm-rust/crates/python-interop/Cargo.toml +++ b/litellm-rust/crates/python-interop/Cargo.toml @@ -4,6 +4,15 @@ version = "0.1.0" edition.workspace = true license.workspace = true repository.workspace = true +autotests = false + +[[test]] +name = "synthetic" +path = "tests/synthetic/mod.rs" + +[[test]] +name = "integration" +path = "tests/integration/mod.rs" [dependencies] pyo3.workspace = true diff --git a/litellm-rust/crates/python-interop/tests/integration/lifecycle.rs b/litellm-rust/crates/python-interop/tests/integration/lifecycle.rs new file mode 100644 index 00000000000..e4600e107ff --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/integration/lifecycle.rs @@ -0,0 +1,96 @@ +use pyo3::prelude::*; +use pyo3::types::PyDict; +use rstest::{fixture, rstest}; +use serial_test::serial; + +use crate::support::Backend; +use crate::support::python::run_fixture; +use crate::support::scenarios::{run_scenario_fixture, scenario_scope}; + +#[fixture] +fn component_scope(scenario_scope: Py) -> Py { + Python::attach(|py| { + run_fixture( + py, + scenario_scope.bind(py), + include_str!("../fixtures/callback_components.py"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/callback_components.py" + ), + ) + .unwrap(); + }); + scenario_scope +} + +#[fixture] +fn integration_scope(scenario_scope: Py) -> Py { + Python::attach(|py| { + run_fixture( + py, + scenario_scope.bind(py), + include_str!("../fixtures/callback_integrations.py"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/callback_integrations.py" + ), + ) + .unwrap(); + }); + scenario_scope +} + +#[rstest] +#[case::identity_and_ignored_returns("pre_call_identity_and_ignored_returns")] +#[case::mutations_visible_to_later_callbacks("pre_call_mutations_visible_to_later_callbacks")] +#[case::mutation_survives_failure("pre_call_mutation_survives_failure")] +#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] +#[serial(python_interpreter)] +fn pre_call_contract( + component_scope: Py, + #[case] scenario: &str, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, +) -> PyResult<()> { + run_scenario_fixture(component_scope, scenario, backend) +} + +#[rstest] +#[case::real_post_call_logging("real_post_call_logging")] +#[case::real_post_call_dict_response("real_post_call_dict_response")] +#[case::real_sync_logging("real_sync_logging")] +#[case::real_sync_logging_hook_failure("real_sync_logging_hook_failure")] +#[case::real_sync_failure_chain("real_sync_failure_chain")] +#[case::real_async_failure_chain("real_async_failure_chain")] +#[case::real_async_logging("real_async_logging")] +#[case::real_copy_boundaries("real_copy_boundaries")] +#[case::real_logging_worker("real_logging_worker")] +#[case::real_sync_stream_copies("real_sync_stream_copies")] +#[case::real_stream_completion("real_stream_completion")] +#[case::real_stream_close("real_stream_close")] +#[case::real_stream_cancellation("real_stream_cancellation")] +#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] +#[serial(python_interpreter)] +fn component_contract( + component_scope: Py, + #[case] scenario: &str, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, +) -> PyResult<()> { + run_scenario_fixture(component_scope, scenario, backend) +} + +#[rstest] +#[case::real_logging_queue_copy_control("real_logging_queue_copy_control")] +#[case::real_crowdstrike_translator_identity("real_crowdstrike_translator_identity")] +#[case::real_rubrik_block_lifecycle("real_rubrik_block_lifecycle")] +#[case::real_parallel_guardrail_sharing_and_exception_order("real_parallel_guardrail_snapshots")] +#[case::real_purview_sync_background_and_active_loop("real_purview_sync_background")] +#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] +#[serial(python_interpreter)] +fn integration_contract( + integration_scope: Py, + #[case] scenario: &str, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, +) -> PyResult<()> { + run_scenario_fixture(integration_scope, scenario, backend) +} diff --git a/litellm-rust/crates/python-interop/tests/integration/mod.rs b/litellm-rust/crates/python-interop/tests/integration/mod.rs new file mode 100644 index 00000000000..6213338de2d --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/integration/mod.rs @@ -0,0 +1,5 @@ +#[path = "../support/mod.rs"] +mod support; + +mod lifecycle; +mod ocr; diff --git a/litellm-rust/crates/python-interop/tests/integration/ocr.rs b/litellm-rust/crates/python-interop/tests/integration/ocr.rs new file mode 100644 index 00000000000..8c38ef61006 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/integration/ocr.rs @@ -0,0 +1,175 @@ +use litellm_python_interop::{InvocationMode, InvocationOutcome, PreparedCall}; +use pyo3::prelude::*; +use pyo3::types::{PyDict, PyList, PyTuple}; +use rstest::rstest; +use serial_test::serial; + +use crate::support::python::{InitializedPython, initialized_python, item, scope}; + +fn prepare_pre_call( + py: Python<'_>, + logger: &Bound<'_, PyAny>, + view: &Bound<'_, PyDict>, +) -> PyResult { + let keywords = PyDict::new(py); + keywords.set_item("input", "OCR document processing")?; + keywords.set_item("api_key", py.None())?; + keywords.set_item("additional_args", view)?; + Ok(PreparedCall::new( + InvocationMode::Direct, + logger.getattr("pre_call")?.unbind(), + PyTuple::empty(py).unbind(), + Some(keywords.unbind()), + )) +} + +#[rstest] +#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] +#[serial(python_interpreter)] +fn real_ocr_logging_preserves_execution_roots_and_continues_after_error( + initialized_python: &InitializedPython, +) -> PyResult<()> { + let _ = initialized_python; + Python::attach(|py| { + let globals = scope( + py, + c" +from litellm.integrations.custom_logger import CustomLogger + +class Retain(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + order.append('retain') + self.view = kwargs['additional_args'] + self.headers = self.view['headers'] + self.body = self.view['complete_input_dict'] + self.snapshot = (self.headers['X-Trace'], self.body['document']['value']) + return {'ignored_replacement': True} + +class MutateThenFail(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + order.append('mutate_then_fail') + view = kwargs['additional_args'] + view['headers']['X-Trace'] = 'mutated' + view['complete_input_dict']['document']['value'] = 'mutated' + view['headers'] = {'X-Trace': 'replacement'} + view['complete_input_dict'] = {'replacement': True} + raise RuntimeError('expected callback failure') + +class Observe(CustomLogger): + def log_pre_api_call(self, model, messages, kwargs): + order.append('observe') + self.view = kwargs['additional_args'] + self.snapshot = ( + tuple(sorted(self.view['headers'].items())), + self.view['complete_input_dict'].get('replacement'), + 'document' in self.view['complete_input_dict'], + ) + +", + )?; + let order = PyList::empty(py); + globals.set_item("order", &order)?; + let first = item(&globals, "Retain").call0()?; + let last = item(&globals, "Observe").call0()?; + let document = PyDict::new(py); + document.set_item("value", "original")?; + let headers = PyDict::new(py); + headers.set_item("X-Trace", "original")?; + let body = PyDict::new(py); + body.set_item("document", &document)?; + body.set_item("alias", &document)?; + let view = PyDict::new(py); + view.set_item("headers", &headers)?; + view.set_item("complete_input_dict", &body)?; + view.set_item("api_base", "https://example.invalid/ocr")?; + let keywords = PyDict::new(py); + keywords.set_item("model", "test")?; + keywords.set_item("messages", PyList::empty(py))?; + keywords.set_item("stream", false)?; + keywords.set_item("call_type", "ocr")?; + keywords.set_item( + "start_time", + py.import("datetime")? + .getattr("datetime")? + .call_method0("now")?, + )?; + keywords.set_item("litellm_call_id", "retained-test")?; + keywords.set_item("function_id", "retained-test")?; + keywords.set_item( + "dynamic_input_callbacks", + PyList::new( + py, + [&first, &item(&globals, "MutateThenFail").call0()?, &last], + )?, + )?; + let logger = py + .import("litellm.litellm_core_utils.litellm_logging")? + .getattr("Logging")? + .call((), Some(&keywords))?; + let invocation = prepare_pre_call(py, &logger, &view)?; + match invocation.invoke(py)? { + InvocationOutcome::Returned(value) => assert!(value.is_none(py)), + InvocationOutcome::Awaitable(_) => { + panic!("direct binding produced an awaitable outcome") + } + } + drop(invocation); + assert!(headers.is(first.getattr("headers")?)); + assert!(body.is(first.getattr("body")?)); + assert!(view.is(last.getattr("view")?)); + assert_eq!(item(&headers, "X-Trace").extract::()?, "mutated"); + assert_eq!( + order.extract::>()?, + ["retain", "mutate_then_fail", "observe"] + ); + assert_eq!( + first.getattr("snapshot")?.extract::<(String, String)>()?, + ("original".to_owned(), "original".to_owned()) + ); + assert_eq!( + last.getattr("snapshot")? + .extract::<(Vec<(String, String)>, bool, bool)>()?, + ( + vec![("X-Trace".to_owned(), "replacement".to_owned())], + true, + false + ) + ); + assert!(first.getattr("view")?.is(last.getattr("view")?)); + assert!(first.getattr("body")?.get_item("document")?.is(&document)); + assert!(first.getattr("body")?.get_item("alias")?.is(&document)); + assert_eq!(item(&document, "value").extract::()?, "mutated"); + assert_eq!( + last.getattr("view")? + .get_item("headers")? + .get_item("X-Trace")? + .extract::()?, + "replacement" + ); + let replacement = PyDict::new(py); + replacement.set_item("replacement", true)?; + assert!( + last.getattr("view")? + .get_item("complete_input_dict")? + .eq(replacement)? + ); + document.set_item("value", "after invocation")?; + assert_eq!( + first + .getattr("body")? + .get_item("document")? + .get_item("value")? + .extract::()?, + "after invocation" + ); + drop((headers, body, view)); + assert_eq!( + first + .getattr("headers")? + .get_item("X-Trace")? + .extract::()?, + "mutated" + ); + Ok(()) + }) +} diff --git a/litellm-rust/crates/python-interop/tests/support/mod.rs b/litellm-rust/crates/python-interop/tests/support/mod.rs index c0a088c7544..5f39a4df4f8 100644 --- a/litellm-rust/crates/python-interop/tests/support/mod.rs +++ b/litellm-rust/crates/python-interop/tests/support/mod.rs @@ -1 +1,9 @@ +mod callback_owner; pub mod python; +pub mod scenarios; + +#[derive(Clone, Copy, Debug)] +pub enum Backend { + Python, + PreparedCall, +} diff --git a/litellm-rust/crates/python-interop/tests/support/scenarios.rs b/litellm-rust/crates/python-interop/tests/support/scenarios.rs new file mode 100644 index 00000000000..e59e3ff1873 --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/support/scenarios.rs @@ -0,0 +1,54 @@ +use pyo3::prelude::*; +use pyo3::types::PyDict; +use rstest::fixture; + +use super::python::{InitializedPython, initialized_python, run_fixture}; +use super::{Backend, callback_owner}; + +#[fixture] +pub fn scenario_scope(initialized_python: &InitializedPython) -> Py { + initialized_python.attach(|py| { + let globals = PyDict::new(py); + globals + .set_item( + "factory", + Py::new(py, callback_owner::OwnerFactory::default()).unwrap(), + ) + .unwrap(); + globals + .set_item( + "AWAIT_ADAPTER_FILENAME", + litellm_python_interop::AWAIT_ADAPTER_FILENAME + .to_str() + .unwrap(), + ) + .unwrap(); + run_fixture( + py, + &globals, + include_str!("../fixtures/callback_lifecycle.py"), + concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/callback_lifecycle.py" + ), + ) + .unwrap(); + globals.unbind() + }) +} + +pub fn run_scenario_fixture( + scenario_scope: Py, + scenario: &str, + backend: Backend, +) -> PyResult<()> { + Python::attach(|py| { + let globals = scenario_scope.bind(py); + globals.get_item("run_scenario")?.unwrap().call1(( + scenario, + matches!(backend, Backend::PreparedCall), + globals.get_item("factory")?.unwrap(), + ))?; + Ok(()) + }) +} diff --git a/litellm-rust/crates/python-interop/tests/callback_controls.rs b/litellm-rust/crates/python-interop/tests/synthetic/controls.rs similarity index 94% rename from litellm-rust/crates/python-interop/tests/callback_controls.rs rename to litellm-rust/crates/python-interop/tests/synthetic/controls.rs index 7261c84ecf3..56fb36283a5 100644 --- a/litellm-rust/crates/python-interop/tests/callback_controls.rs +++ b/litellm-rust/crates/python-interop/tests/synthetic/controls.rs @@ -6,10 +6,8 @@ use pyo3::types::{PyDict, PyTuple}; use rstest::rstest; use serial_test::serial; -#[path = "support/mod.rs"] -mod support; - -use support::python::{InitializedPython, initialized_python, item, scope}; +use crate::support::Backend; +use crate::support::python::{InitializedPython, initialized_python, item, scope}; enum ControlCall { Reference(Py), @@ -22,10 +20,10 @@ impl ControlCall { callback: Bound<'_, PyAny>, args: Bound<'_, PyTuple>, kwargs: Option>, - retained: bool, + backend: Backend, mode: InvocationMode, ) -> PyResult { - if retained { + if matches!(backend, Backend::PreparedCall) { return Ok(Self::Prepared(PreparedCall::new( mode, callback.unbind(), @@ -76,7 +74,7 @@ fn argument_copy_boundaries_determine_identity_and_mutation_visibility( #[case] transform: &CStr, #[case] expected_identity: (bool, bool, bool), #[case] live_stages: (u8, u8, u8), - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, #[values(InvocationMode::Direct, InvocationMode::Await)] mode: InvocationMode, ) -> PyResult<()> { initialized_python.attach(|py| { @@ -110,7 +108,7 @@ async def observe_async(value, *, alias): ), transformed.get_item(0)?.cast_into::()?, Some(transformed.get_item(1)?.cast_into::()?), - retained, + backend, mode, )?; let original = item(&globals, "original"); @@ -163,7 +161,7 @@ fn result_copy_boundaries_determine_root_and_nested_identity( initialized_python: &InitializedPython, #[case] transform: Option<&CStr>, #[case] expected_identity: (bool, bool), - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, #[values(InvocationMode::Direct, InvocationMode::Await)] mode: InvocationMode, ) -> PyResult<()> { initialized_python.attach(|py| { @@ -194,7 +192,7 @@ async def callback_async(): ), PyTuple::empty(py), None, - retained, + backend, mode, )?; let pending = call.invoke(py, mode)?; diff --git a/litellm-rust/crates/python-interop/tests/callback_lifecycle.rs b/litellm-rust/crates/python-interop/tests/synthetic/lifecycle.rs similarity index 68% rename from litellm-rust/crates/python-interop/tests/callback_lifecycle.rs rename to litellm-rust/crates/python-interop/tests/synthetic/lifecycle.rs index 36cf22c538b..39c2ace6bca 100644 --- a/litellm-rust/crates/python-interop/tests/callback_lifecycle.rs +++ b/litellm-rust/crates/python-interop/tests/synthetic/lifecycle.rs @@ -1,36 +1,51 @@ -use std::process::Command; +use std::io::Read; +use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; use litellm_python_interop::{InvocationMode, InvocationOutcome, PreparedCall}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; -use rstest::{fixture, rstest}; +use rstest::rstest; use serial_test::{parallel, serial}; -#[path = "support/callback_owner.rs"] -mod callback_owner; - -#[path = "support/mod.rs"] -mod support; - -use support::python::{InitializedPython, initialized_python, item, run_fixture}; +use crate::support::Backend; +use crate::support::python::{InitializedPython, initialized_python, item, run_fixture}; +use crate::support::scenarios::{run_scenario_fixture, scenario_scope}; #[test] fn cold_awaited_adapter_initialization_allows_reentry() -> PyResult<()> { - let test = "cold_awaited_adapter_initialization_allows_reentry"; + let test = concat!( + module_path!(), + "::cold_awaited_adapter_initialization_allows_reentry" + ) + .split_once("::") + .unwrap() + .1; let child_env = "LITELLM_INTEROP_COLD_REENTRY_CHILD"; if std::env::var(child_env).as_deref() != Ok(test) { let mut child = Command::new(std::env::current_exe().unwrap()) .args(["--exact", test, "--nocapture"]) .env(child_env, test) + .stdout(Stdio::piped()) .spawn() .unwrap(); let deadline = Instant::now() + Duration::from_secs(15); loop { if let Some(status) = child.try_wait().unwrap() { + let mut output = String::new(); + child + .stdout + .take() + .unwrap() + .read_to_string(&mut output) + .unwrap(); assert!( status.success(), - "awaited adapter reentry child failed: {status}" + "awaited adapter reentry child failed: {status}\n{output}" + ); + assert!( + output.contains("test result: ok. 1 passed; 0 failed; 0 ignored;"), + "awaited adapter reentry child did not run exactly one test:\n{output}" ); return Ok(()); } @@ -54,39 +69,6 @@ fn cold_awaited_adapter_initialization_allows_reentry() -> PyResult<()> { }) } -#[fixture] -fn scenario_scope(initialized_python: &InitializedPython) -> Py { - let _ = initialized_python; - Python::attach(|py| { - let globals = PyDict::new(py); - globals - .set_item( - "factory", - Py::new(py, callback_owner::OwnerFactory::default()).unwrap(), - ) - .unwrap(); - globals - .set_item( - "AWAIT_ADAPTER_FILENAME", - litellm_python_interop::AWAIT_ADAPTER_FILENAME - .to_str() - .unwrap(), - ) - .unwrap(); - run_fixture( - py, - &globals, - include_str!("fixtures/callback_lifecycle.py"), - concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/callback_lifecycle.py" - ), - ) - .unwrap(); - globals.unbind() - }) -} - #[rstest] #[case::awaitable_kinds("awaitable_kinds")] #[case::identity_and_context("identity_and_context")] @@ -106,116 +88,9 @@ fn scenario_scope(initialized_python: &InitializedPython) -> Py { fn lifecycle_contract( scenario_scope: Py, #[case] scenario: &str, - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, ) -> PyResult<()> { - run_scenario_fixture(scenario_scope, scenario, retained, None) -} - -#[rstest] -#[case::identity_and_ignored_returns("pre_call_identity_and_ignored_returns")] -#[case::mutations_visible_to_later_callbacks("pre_call_mutations_visible_to_later_callbacks")] -#[case::mutation_survives_failure("pre_call_mutation_survives_failure")] -#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] -#[serial(python_interpreter)] -fn pre_call_contract( - scenario_scope: Py, - #[case] scenario: &str, - #[values(false, true)] retained: bool, -) -> PyResult<()> { - run_scenario_fixture( - scenario_scope, - scenario, - retained, - Some(( - include_str!("fixtures/callback_components.py"), - concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/callback_components.py" - ), - )), - ) -} - -#[rstest] -#[case::real_post_call_logging("real_post_call_logging")] -#[case::real_post_call_dict_response("real_post_call_dict_response")] -#[case::real_sync_logging("real_sync_logging")] -#[case::real_sync_logging_hook_failure("real_sync_logging_hook_failure")] -#[case::real_sync_failure_chain("real_sync_failure_chain")] -#[case::real_async_failure_chain("real_async_failure_chain")] -#[case::real_async_logging("real_async_logging")] -#[case::real_copy_boundaries("real_copy_boundaries")] -#[case::real_logging_worker("real_logging_worker")] -#[case::real_sync_stream_copies("real_sync_stream_copies")] -#[case::real_stream_completion("real_stream_completion")] -#[case::real_stream_close("real_stream_close")] -#[case::real_stream_cancellation("real_stream_cancellation")] -#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] -#[serial(python_interpreter)] -fn component_contract( - scenario_scope: Py, - #[case] scenario: &str, - #[values(false, true)] retained: bool, -) -> PyResult<()> { - run_scenario_fixture( - scenario_scope, - scenario, - retained, - Some(( - include_str!("fixtures/callback_components.py"), - concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/callback_components.py" - ), - )), - ) -} - -#[rstest] -#[case::real_logging_queue_copy_control("real_logging_queue_copy_control")] -#[case::real_crowdstrike_translator_identity("real_crowdstrike_translator_identity")] -#[case::real_rubrik_block_lifecycle("real_rubrik_block_lifecycle")] -#[case::real_parallel_guardrail_sharing_and_exception_order("real_parallel_guardrail_snapshots")] -#[case::real_purview_sync_background_and_active_loop("real_purview_sync_background")] -#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] -#[serial(python_interpreter)] -fn integration_contract( - scenario_scope: Py, - #[case] scenario: &str, - #[values(false, true)] retained: bool, -) -> PyResult<()> { - run_scenario_fixture( - scenario_scope, - scenario, - retained, - Some(( - include_str!("fixtures/callback_integrations.py"), - concat!( - env!("CARGO_MANIFEST_DIR"), - "/tests/fixtures/callback_integrations.py" - ), - )), - ) -} - -fn run_scenario_fixture( - scenario_scope: Py, - scenario: &str, - retained: bool, - fixture: Option<(&str, &str)>, -) -> PyResult<()> { - Python::attach(|py| { - let globals = scenario_scope.bind(py); - if let Some((source, filename)) = fixture { - run_fixture(py, globals, source, filename)?; - } - globals.get_item("run_scenario")?.unwrap().call1(( - scenario, - retained, - globals.get_item("factory")?.unwrap(), - ))?; - Ok(()) - }) + run_scenario_fixture(scenario_scope, scenario, backend) } #[rstest] @@ -229,14 +104,12 @@ fn control_contract( scenario_scope: Py, #[case] witness: &str, #[case] control: &str, - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, #[values(false, true)] awaited: bool, ) -> PyResult<()> { - run_control_fixture(scenario_scope, witness, control, retained, awaited) + run_control_fixture(scenario_scope, witness, control, backend, awaited) } -// The `weak` control wraps nothing: it holds only weak references and never -// calls the factory, so the `retained` axis has no effect on it. #[rstest] #[case::expired_borrow("deferred_lifetime")] #[case::externally_owned_borrow("borrowed_lifetime")] @@ -246,7 +119,7 @@ fn weak_control( #[case] witness: &str, #[values(false, true)] awaited: bool, ) -> PyResult<()> { - run_control_fixture(scenario_scope, witness, "weak", false, awaited) + run_control_fixture(scenario_scope, witness, "weak", Backend::Python, awaited) } #[rstest] @@ -256,16 +129,16 @@ fn weak_control( fn pending_handoff_control( scenario_scope: Py, #[case] control: &str, - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, ) -> PyResult<()> { - run_control_fixture(scenario_scope, "pending_handoff", control, retained, true) + run_control_fixture(scenario_scope, "pending_handoff", control, backend, true) } fn run_control_fixture( scenario_scope: Py, witness: &str, control: &str, - retained: bool, + backend: Backend, awaited: bool, ) -> PyResult<()> { Python::attach(|py| { @@ -273,7 +146,7 @@ fn run_control_fixture( run_fixture( py, globals, - include_str!("fixtures/callback_controls.py"), + include_str!("../fixtures/callback_controls.py"), concat!( env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/callback_controls.py" @@ -282,7 +155,7 @@ fn run_control_fixture( globals.get_item("run_control")?.unwrap().call1(( witness, control, - retained, + matches!(backend, Backend::PreparedCall), awaited, globals.get_item("factory")?.unwrap(), ))?; @@ -295,10 +168,10 @@ fn invoke_direct_callback( globals: &Bound<'_, PyDict>, callback: &str, argument: &str, - retained: bool, + backend: Backend, ) -> PyResult> { let args = PyTuple::new(py, [item(globals, argument)])?; - if !retained { + if matches!(backend, Backend::Python) { let factory = item(globals, "ReferenceFactory").call0()?; let owner = factory.call_method1("prepare", (item(globals, callback), args))?; let result = owner.call_method0("invoke"); @@ -322,7 +195,7 @@ fn invoke_direct_callback( #[serial(python_interpreter)] fn retained_field_survives_replacement_and_observes_original_mutations( scenario_scope: Py, - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, ) -> PyResult<()> { Python::attach(|py| { let globals = scenario_scope.bind(py); @@ -344,7 +217,7 @@ def replace(value): None, )?; for callback in ["retain", "replace"] { - assert!(invoke_direct_callback(py, globals, callback, "event", retained)?.is_none(py)); + assert!(invoke_direct_callback(py, globals, callback, "event", backend)?.is_none(py)); } let original = item(globals, "original"); let replacement = item(globals, "replacement"); @@ -390,7 +263,7 @@ def replace(value): #[serial(python_interpreter)] fn queued_graph_outlives_invocation_and_stays_live_until_serialized( scenario_scope: Py, - #[values(false, true)] retained: bool, + #[values(Backend::Python, Backend::PreparedCall)] backend: Backend, ) -> PyResult<()> { Python::attach(|py| { let globals = scenario_scope.bind(py); @@ -408,7 +281,7 @@ def enqueue(value): Some(globals), None, )?; - assert!(invoke_direct_callback(py, globals, "enqueue", "payload", retained)?.is_none(py)); + assert!(invoke_direct_callback(py, globals, "enqueue", "payload", backend)?.is_none(py)); py.run(c"del sentinel, payload\ngc.collect()", Some(globals), None)?; let reference = item(globals, "reference"); assert!(!reference.call0()?.is_none()); diff --git a/litellm-rust/crates/python-interop/tests/synthetic/mod.rs b/litellm-rust/crates/python-interop/tests/synthetic/mod.rs new file mode 100644 index 00000000000..2e4f9bc28ef --- /dev/null +++ b/litellm-rust/crates/python-interop/tests/synthetic/mod.rs @@ -0,0 +1,8 @@ +#[path = "../support/mod.rs"] +mod support; + +mod controls; +mod lifecycle; +mod patterns; +mod prepared_call; +mod primitives; diff --git a/litellm-rust/crates/python-interop/tests/callback_patterns.rs b/litellm-rust/crates/python-interop/tests/synthetic/patterns.rs similarity index 99% rename from litellm-rust/crates/python-interop/tests/callback_patterns.rs rename to litellm-rust/crates/python-interop/tests/synthetic/patterns.rs index 58e7d5535cc..f9489af6e94 100644 --- a/litellm-rust/crates/python-interop/tests/callback_patterns.rs +++ b/litellm-rust/crates/python-interop/tests/synthetic/patterns.rs @@ -6,16 +6,8 @@ use pyo3::types::{PyDict, PyTuple}; use rstest::rstest; use serial_test::serial; -#[path = "support/mod.rs"] -mod support; - -use support::python::{InitializedPython, initialized_python, item, run_fixture, scope}; - -#[derive(Clone, Copy, Debug)] -enum Backend { - Python, - PreparedCall, -} +use crate::support::Backend; +use crate::support::python::{InitializedPython, initialized_python, item, run_fixture, scope}; #[pyfunction] fn invoke_prepared<'py>( diff --git a/litellm-rust/crates/python-interop/tests/prepared_call.rs b/litellm-rust/crates/python-interop/tests/synthetic/prepared_call.rs similarity index 65% rename from litellm-rust/crates/python-interop/tests/prepared_call.rs rename to litellm-rust/crates/python-interop/tests/synthetic/prepared_call.rs index 6ddad3f2510..6ff35c344c0 100644 --- a/litellm-rust/crates/python-interop/tests/prepared_call.rs +++ b/litellm-rust/crates/python-interop/tests/synthetic/prepared_call.rs @@ -3,13 +3,12 @@ use pyo3::exceptions::{PyAssertionError, PyKeyboardInterrupt, PyValueError}; use pyo3::prelude::*; use pyo3::types::{PyDict, PyList, PyTuple}; use rstest::rstest; +use serial_test::parallel; -#[path = "support/mod.rs"] -mod support; - -use support::python::{InitializedPython, initialized_python, item, run_fixture, scope}; +use crate::support::python::{InitializedPython, initialized_python, item, run_fixture, scope}; #[rstest] +#[parallel(python_interpreter)] fn retains_aliases_mutations_and_original_result( initialized_python: &InitializedPython, ) -> PyResult<()> { @@ -61,6 +60,7 @@ def callback(data, *, alias): } #[rstest] +#[parallel(python_interpreter)] fn preserves_exception_identity_cause_traceback_and_prior_mutation( initialized_python: &InitializedPython, ) -> PyResult<()> { @@ -105,6 +105,7 @@ def callback(data): } #[rstest] +#[parallel(python_interpreter)] fn returns_coroutine_without_executing_it(initialized_python: &InitializedPython) -> PyResult<()> { let _ = initialized_python; Python::attach(|py| { @@ -152,6 +153,7 @@ fn reenter(py: Python<'_>, callback: Py, payload: Py) -> PyResult< } #[rstest] +#[parallel(python_interpreter)] fn preserves_current_context_thread_and_reentry( initialized_python: &InitializedPython, ) -> PyResult<()> { @@ -208,6 +210,7 @@ def outer(): } #[rstest] +#[parallel(python_interpreter)] fn owns_arguments_until_release_and_preserves_callback_retention( initialized_python: &InitializedPython, ) -> PyResult<()> { @@ -269,24 +272,8 @@ class Callback: }) } -fn prepare_pre_call( - py: Python<'_>, - logger: &Bound<'_, PyAny>, - view: &Bound<'_, PyDict>, -) -> PyResult { - let keywords = PyDict::new(py); - keywords.set_item("input", "OCR document processing")?; - keywords.set_item("api_key", py.None())?; - keywords.set_item("additional_args", view)?; - Ok(PreparedCall::new( - InvocationMode::Direct, - logger.getattr("pre_call")?.unbind(), - PyTuple::empty(py).unbind(), - Some(keywords.unbind()), - )) -} - #[rstest] +#[parallel(python_interpreter)] fn checked_runner_rejects_unhandled_background_failures( initialized_python: &InitializedPython, ) -> PyResult<()> { @@ -296,7 +283,7 @@ fn checked_runner_rejects_unhandled_background_failures( run_fixture( py, &globals, - include_str!("fixtures/callback_lifecycle.py"), + include_str!("../fixtures/callback_lifecycle.py"), concat!( env!("CARGO_MANIFEST_DIR"), "/tests/fixtures/callback_lifecycle.py" @@ -355,151 +342,6 @@ async def scenario(cyclic, handled, observed): }) } -#[rstest] -#[ignore = "requires the repository Python environment and LiteLLM on PYTHONPATH"] -fn real_ocr_logging_preserves_execution_roots_and_continues_after_error( - initialized_python: &InitializedPython, -) -> PyResult<()> { - let _ = initialized_python; - Python::attach(|py| { - let globals = scope( - py, - c" -from litellm.integrations.custom_logger import CustomLogger - -class Retain(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - order.append('retain') - self.view = kwargs['additional_args'] - self.headers = self.view['headers'] - self.body = self.view['complete_input_dict'] - self.snapshot = (self.headers['X-Trace'], self.body['document']['value']) - return {'ignored_replacement': True} - -class MutateThenFail(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - order.append('mutate_then_fail') - view = kwargs['additional_args'] - view['headers']['X-Trace'] = 'mutated' - view['complete_input_dict']['document']['value'] = 'mutated' - view['headers'] = {'X-Trace': 'replacement'} - view['complete_input_dict'] = {'replacement': True} - raise RuntimeError('expected callback failure') - -class Observe(CustomLogger): - def log_pre_api_call(self, model, messages, kwargs): - order.append('observe') - self.view = kwargs['additional_args'] - self.snapshot = ( - tuple(sorted(self.view['headers'].items())), - self.view['complete_input_dict'].get('replacement'), - 'document' in self.view['complete_input_dict'], - ) - -", - )?; - let order = PyList::empty(py); - globals.set_item("order", &order)?; - let first = item(&globals, "Retain").call0()?; - let last = item(&globals, "Observe").call0()?; - let document = PyDict::new(py); - document.set_item("value", "original")?; - let headers = PyDict::new(py); - headers.set_item("X-Trace", "original")?; - let body = PyDict::new(py); - body.set_item("document", &document)?; - body.set_item("alias", &document)?; - let view = PyDict::new(py); - view.set_item("headers", &headers)?; - view.set_item("complete_input_dict", &body)?; - view.set_item("api_base", "https://example.invalid/ocr")?; - let keywords = PyDict::new(py); - keywords.set_item("model", "test")?; - keywords.set_item("messages", PyList::empty(py))?; - keywords.set_item("stream", false)?; - keywords.set_item("call_type", "ocr")?; - keywords.set_item( - "start_time", - py.import("datetime")? - .getattr("datetime")? - .call_method0("now")?, - )?; - keywords.set_item("litellm_call_id", "retained-test")?; - keywords.set_item("function_id", "retained-test")?; - keywords.set_item( - "dynamic_input_callbacks", - PyList::new( - py, - [&first, &item(&globals, "MutateThenFail").call0()?, &last], - )?, - )?; - let logger = py - .import("litellm.litellm_core_utils.litellm_logging")? - .getattr("Logging")? - .call((), Some(&keywords))?; - let invocation = prepare_pre_call(py, &logger, &view)?; - assert!(invoke_direct(&invocation, py)?.is_none(py)); - drop(invocation); - assert!(headers.is(first.getattr("headers")?)); - assert!(body.is(first.getattr("body")?)); - assert!(view.is(last.getattr("view")?)); - assert_eq!(item(&headers, "X-Trace").extract::()?, "mutated"); - assert_eq!( - order.extract::>()?, - ["retain", "mutate_then_fail", "observe"] - ); - assert_eq!( - first.getattr("snapshot")?.extract::<(String, String)>()?, - ("original".to_owned(), "original".to_owned()) - ); - assert_eq!( - last.getattr("snapshot")? - .extract::<(Vec<(String, String)>, bool, bool)>()?, - ( - vec![("X-Trace".to_owned(), "replacement".to_owned())], - true, - false - ) - ); - assert!(first.getattr("view")?.is(last.getattr("view")?)); - assert!(first.getattr("body")?.get_item("document")?.is(&document)); - assert!(first.getattr("body")?.get_item("alias")?.is(&document)); - assert_eq!(item(&document, "value").extract::()?, "mutated"); - assert_eq!( - last.getattr("view")? - .get_item("headers")? - .get_item("X-Trace")? - .extract::()?, - "replacement" - ); - let replacement = PyDict::new(py); - replacement.set_item("replacement", true)?; - assert!( - last.getattr("view")? - .get_item("complete_input_dict")? - .eq(replacement)? - ); - document.set_item("value", "after invocation")?; - assert_eq!( - first - .getattr("body")? - .get_item("document")? - .get_item("value")? - .extract::()?, - "after invocation" - ); - drop((headers, body, view)); - assert_eq!( - first - .getattr("headers")? - .get_item("X-Trace")? - .extract::()?, - "mutated" - ); - Ok(()) - }) -} - fn invoke_direct(call: &PreparedCall, py: Python<'_>) -> PyResult> { match call.invoke(py)? { InvocationOutcome::Returned(value) => Ok(value), diff --git a/litellm-rust/crates/python-interop/tests/interop.rs b/litellm-rust/crates/python-interop/tests/synthetic/primitives.rs similarity index 84% rename from litellm-rust/crates/python-interop/tests/interop.rs rename to litellm-rust/crates/python-interop/tests/synthetic/primitives.rs index a40a73b2b6d..5b948b6246e 100644 --- a/litellm-rust/crates/python-interop/tests/interop.rs +++ b/litellm-rust/crates/python-interop/tests/synthetic/primitives.rs @@ -1,14 +1,13 @@ use rstest::rstest; use serde_json::{Value, json}; +use serial_test::parallel; use litellm_python_interop::{from_py, release_count, release_gil, to_py}; -#[path = "support/mod.rs"] -mod support; - -use support::python::{InitializedPython, initialized_python}; +use crate::support::python::{InitializedPython, initialized_python}; #[rstest] +#[parallel(python_interpreter)] fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &InitializedPython) { python.attach(|py| { let expected = json!({"model": "test", "items": [1, true, null]}); @@ -21,6 +20,7 @@ fn serde_values_round_trip_through_python(#[from(initialized_python)] python: &I } #[rstest] +#[parallel(python_interpreter)] fn release_gil_runs_work_and_records_it(#[from(initialized_python)] python: &InitializedPython) { let before = release_count(); let result = python.attach(|py| release_gil(py, || 42));