litellm/litellm-rust/crates/token-counter/benches/allocations.rs

use std::alloc::{GlobalAlloc, Layout, System};
use std::hint::black_box;
use std::sync::atomic::{AtomicUsize, Ordering};

use litellm_token_counter::{CountableRequest, TokenCounter};

struct CountingAllocator;

static ALLOCATIONS: AtomicUsize = AtomicUsize::new(0);
static BYTES: AtomicUsize = AtomicUsize::new(0);

unsafe impl GlobalAlloc for CountingAllocator {
    unsafe fn alloc(&self, layout: Layout) -> *mut u8 {
        ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
        BYTES.fetch_add(layout.size(), Ordering::Relaxed);
        // SAFETY: This allocator delegates the unchanged layout to `System`.
        unsafe { System.alloc(layout) }
    }

    unsafe fn dealloc(&self, pointer: *mut u8, layout: Layout) {
        // SAFETY: The pointer and layout came from the delegated `System` allocation.
        unsafe { System.dealloc(pointer, layout) }
    }

    unsafe fn realloc(&self, pointer: *mut u8, layout: Layout, size: usize) -> *mut u8 {
        ALLOCATIONS.fetch_add(1, Ordering::Relaxed);
        BYTES.fetch_add(size, Ordering::Relaxed);
        // SAFETY: The pointer and layout came from `System`; the new size is unchanged.
        unsafe { System.realloc(pointer, layout, size) }
    }
}

#[global_allocator]
static ALLOCATOR: CountingAllocator = CountingAllocator;

#[derive(Clone, Copy)]
struct AllocationCount {
    allocations: usize,
    bytes: usize,
}

impl AllocationCount {
    fn assert_max(self, label: &str, maximum: Self) {
        eprintln!(
            "{label}: {} allocations, {} bytes",
            self.allocations, self.bytes
        );
        assert!(
            self.allocations <= maximum.allocations,
            "{label} allocation count exceeded {}",
            maximum.allocations
        );
        assert!(
            self.bytes <= maximum.bytes,
            "{label} allocated bytes exceeded {}",
            maximum.bytes
        );
    }
}

const TOKENIZER_JSON: &str = include_str!(concat!(
    env!("CARGO_MANIFEST_DIR"),
    "/../../../litellm/litellm_core_utils/tokenizers/anthropic_tokenizer.json"
));
const OBJECT_BODY: &[u8] = br#"{"model":"claude-sonnet-4-5","input":{"text":"caf\u00e9","n":3,"ok":true,"list":[1,"a",{"z":[]}]}}"#;
const INTEGER_BODY: &[u8] = br#"{"input":[-9223372036854775808,0,18446744073709551615]}"#;

fn measure(operation: impl FnOnce()) -> AllocationCount {
    ALLOCATIONS.store(0, Ordering::Relaxed);
    BYTES.store(0, Ordering::Relaxed);
    operation();
    AllocationCount {
        allocations: ALLOCATIONS.load(Ordering::Relaxed),
        bytes: BYTES.load(Ordering::Relaxed),
    }
}

fn main() {
    measure(|| {
        black_box(CountableRequest::parse(OBJECT_BODY).expect("object request parses"));
    })
    .assert_max(
        "parse object request",
        AllocationCount {
            allocations: 16,
            bytes: 1_900,
        },
    );

    let counter = TokenCounter::from_json_fast(TOKENIZER_JSON).expect("tokenizer loads");
    let object = CountableRequest::parse(OBJECT_BODY).expect("object request parses");
    counter
        .count_request(&object)
        .expect("object warmup succeeds");
    measure(|| {
        black_box(
            counter
                .count_request(black_box(&object))
                .expect("object counts"),
        );
    })
    .assert_max(
        "count object request",
        AllocationCount {
            allocations: 74,
            bytes: 2_200,
        },
    );

    let integers = CountableRequest::parse(INTEGER_BODY).expect("integer request parses");
    counter
        .count_request(&integers)
        .expect("integer warmup succeeds");
    measure(|| {
        black_box(
            counter
                .count_request(black_box(&integers))
                .expect("integers count"),
        );
    })
    .assert_max(
        "count integer list",
        AllocationCount {
            allocations: 26,
            bytes: 1_050,
        },
    );
}