From 9f6f2446f90c9ddc8b56f74cd903cb13e7ecda92 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Thu, 17 Sep 2026 15:30:47 -0700 Subject: [PATCH] test(rust): add allocation ceilings bench for the framing crate --- litellm-rust/crates/framer/Cargo.toml | 5 + .../crates/framer/benches/allocations.rs | 180 ++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 litellm-rust/crates/framer/benches/allocations.rs diff --git a/litellm-rust/crates/framer/Cargo.toml b/litellm-rust/crates/framer/Cargo.toml index d22502f871a..c1a4dc96984 100644 --- a/litellm-rust/crates/framer/Cargo.toml +++ b/litellm-rust/crates/framer/Cargo.toml @@ -21,3 +21,8 @@ thiserror.workspace = true [dev-dependencies] rstest.workspace = true tokio.workspace = true + +[[bench]] +name = "allocations" +harness = false +required-features = ["aws", "sse"] diff --git a/litellm-rust/crates/framer/benches/allocations.rs b/litellm-rust/crates/framer/benches/allocations.rs new file mode 100644 index 00000000000..cb5291be47c --- /dev/null +++ b/litellm-rust/crates/framer/benches/allocations.rs @@ -0,0 +1,180 @@ +use std::alloc::{GlobalAlloc, Layout, System}; +use std::hint::black_box; +use std::io; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use aws_smithy_eventstream::frame::write_message_to; +use aws_smithy_types::event_stream::{Header, HeaderValue, Message}; +use bytes::Bytes; +use futures_util::{Stream, TryStreamExt}; +use litellm_framing::Framer; +use litellm_framing::aws_event_stream::AwsEventStreamFramer; +use litellm_framing::sse::SseFramer; + +struct CountingAllocator; + +static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0); +static BYTES: AtomicUsize = AtomicUsize::new(0); + +unsafe impl GlobalAlloc for CountingAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + BYTES.fetch_add(layout.size(), Ordering::Relaxed); + // SAFETY: This allocator delegates the unchanged layout to `System`. + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) { + // SAFETY: The pointer and layout came from the delegated `System` allocation. + unsafe { System.dealloc(pointer, layout) } + } + + unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, size: usize) -> *mut u8 { + ALLOCATIONS.fetch_add(1, Ordering::Relaxed); + BYTES.fetch_add(size, Ordering::Relaxed); + // SAFETY: The pointer and layout came from `System`; the new size is unchanged. + unsafe { System.realloc(pointer, layout, size) } + } +} + +#[global_allocator] +static ALLOCATOR: CountingAllocator = CountingAllocator; + +const FRAMES: usize = 1_000; +const SPLIT_CHUNK_BYTES: usize = 64; +const DELTA: &str = r#"{"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":" The quick brown fox jumps over the lazy dog and keeps running."}}"#; + +#[derive(Clone, Copy)] +struct AllocationCount { + allocations: usize, + bytes: usize, +} + +impl AllocationCount { + fn assert_max(self, label: &str, maximum: Self) { + eprintln!( + "{label}: {} allocations, {} bytes ({:.1} allocations/frame)", + self.allocations, + self.bytes, + self.allocations as f64 / FRAMES as f64 + ); + assert!( + self.allocations <= maximum.allocations, + "{label} allocation count exceeded {}", + maximum.allocations + ); + assert!( + self.bytes <= maximum.bytes, + "{label} allocated bytes exceeded {}", + maximum.bytes + ); + } +} + +fn measure(operation: impl FnOnce()) -> AllocationCount { + ALLOCATIONS.store(0, Ordering::Relaxed); + BYTES.store(0, Ordering::Relaxed); + operation(); + AllocationCount { + allocations: ALLOCATIONS.load(Ordering::Relaxed), + bytes: BYTES.load(Ordering::Relaxed), + } +} + +fn bedrock_wire() -> Vec { + let message = Message::new(Bytes::from(format!(r#"{{"bytes":"{DELTA}"}}"#))) + .add_header(Header::new(":event-type", HeaderValue::String("chunk".into()))) + .add_header(Header::new( + ":content-type", + HeaderValue::String("application/json".into()), + )) + .add_header(Header::new(":message-type", HeaderValue::String("event".into()))); + let mut frame = Vec::new(); + write_message_to(&message, &mut frame).expect("frame encodes"); + frame.repeat(FRAMES) +} + +fn sse_wire() -> Vec { + format!("event: content_block_delta\ndata: {DELTA}\n\n") + .into_bytes() + .repeat(FRAMES) +} + +fn chunked(wire: &[u8], chunk_bytes: usize) -> impl Stream> + Send { + let chunks: Vec = wire.chunks(chunk_bytes).map(Bytes::copy_from_slice).collect(); + futures_util::stream::iter(chunks.into_iter().map(Ok)) +} + +fn main() { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .expect("runtime builds"); + let bedrock = bedrock_wire(); + let bedrock_frame_bytes = bedrock.len() / FRAMES; + let sse = sse_wire(); + let sse_frame_bytes = sse.len() / FRAMES; + + let aligned = chunked(&bedrock, bedrock_frame_bytes); + measure(|| { + black_box( + runtime + .block_on(AwsEventStreamFramer.frame(aligned).try_collect::>()) + .expect("aligned bedrock frames decode"), + ); + }) + .assert_max( + "aws event stream, one frame per chunk", + AllocationCount { + allocations: 16_100, + bytes: 1_070_000, + }, + ); + + let split = chunked(&bedrock, SPLIT_CHUNK_BYTES); + measure(|| { + black_box( + runtime + .block_on(AwsEventStreamFramer.frame(split).try_collect::>()) + .expect("split bedrock frames decode"), + ); + }) + .assert_max( + "aws event stream, 64-byte chunks", + AllocationCount { + allocations: 16_100, + bytes: 1_070_000, + }, + ); + + let aligned = chunked(&sse, sse_frame_bytes); + measure(|| { + black_box( + runtime + .block_on(SseFramer.frame(aligned).try_collect::>()) + .expect("aligned sse frames decode"), + ); + }) + .assert_max( + "sse, one event per chunk", + AllocationCount { + allocations: 2_100, + bytes: 350_000, + }, + ); + + let split = chunked(&sse, SPLIT_CHUNK_BYTES); + measure(|| { + black_box( + runtime + .block_on(SseFramer.frame(split).try_collect::>()) + .expect("split sse frames decode"), + ); + }) + .assert_max( + "sse, 64-byte chunks", + AllocationCount { + allocations: 2_100, + bytes: 350_000, + }, + ); +}