From d63224db151bda93c8c818f19930e4b391f950bd Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Sun, 15 Nov 2026 11:19:03 -0800 Subject: [PATCH] refactor(s3-log): move python-compatible helpers into python-compat (#51895) --- .../rust/crates/callbacks-s3-log/src/key.rs | 49 +- .../rust/crates/callbacks-s3-log/src/lib.rs | 1 - .../crates/callbacks-s3-log/src/pyjson.rs | 214 -------- .../crates/callbacks-s3-log/src/settings.rs | 110 ++-- .../crates/callbacks-s3-log/src/upload.rs | 2 +- litellm/rust/crates/python-compat/AGENTS.md | 2 + .../python-compat/generated/.gitattributes | 1 + .../python-compat/generated/pydantic.json | 485 ++++++++++++++++++ .../python-compat/generated/stdlib.json | 345 +++++++++++++ .../scripts/generate_pydantic_fixtures.py | 142 +++++ .../scripts/generate_stdlib_fixtures.py | 66 +++ .../rust/crates/python-compat/src/datetime.rs | 53 ++ litellm/rust/crates/python-compat/src/lib.rs | 6 + .../rust/crates/python-compat/src/pydantic.rs | 70 +++ .../rust/crates/python-compat/src/urllib.rs | 18 + .../crates/python-compat/tests/pydantic.rs | 48 ++ .../rust/crates/python-compat/tests/stdlib.rs | 62 +++ 17 files changed, 1345 insertions(+), 329 deletions(-) delete mode 100644 litellm/rust/crates/callbacks-s3-log/src/pyjson.rs create mode 100644 litellm/rust/crates/python-compat/generated/.gitattributes create mode 100644 litellm/rust/crates/python-compat/generated/pydantic.json create mode 100644 litellm/rust/crates/python-compat/generated/stdlib.json create mode 100644 litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py create mode 100644 litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py create mode 100644 litellm/rust/crates/python-compat/src/datetime.rs create mode 100644 litellm/rust/crates/python-compat/src/pydantic.rs create mode 100644 litellm/rust/crates/python-compat/src/urllib.rs create mode 100644 litellm/rust/crates/python-compat/tests/pydantic.rs create mode 100644 litellm/rust/crates/python-compat/tests/stdlib.rs diff --git a/litellm/rust/crates/callbacks-s3-log/src/key.rs b/litellm/rust/crates/callbacks-s3-log/src/key.rs index a0f3fc12b10..6f408e37c43 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/key.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/key.rs @@ -1,6 +1,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use litellm_callbacks::Moment; +use litellm_python_compat::datetime::UtcDateTime; use sha2::{Digest, Sha256}; use crate::settings::{Granularity, Settings}; @@ -58,16 +59,18 @@ impl Stamp { pub fn utc(time: SystemTime) -> Self { let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default(); let seconds = elapsed.as_secs(); - let (year, month, day) = civil_from_days((seconds / 86_400) as i64); - let of_day = seconds % 86_400; + let moment = UtcDateTime::from_timestamp( + i64::try_from(seconds).unwrap_or(i64::MAX), + elapsed.subsec_micros(), + ); Self { - year, - month, - day, - hour: (of_day / 3600) as u32, - minute: (of_day / 60 % 60) as u32, - second: (of_day % 60) as u32, - micros: elapsed.subsec_micros(), + year: moment.year, + month: moment.month, + day: moment.day, + hour: moment.hour, + minute: moment.minute, + second: moment.second, + micros: moment.microsecond, } } @@ -84,24 +87,6 @@ impl Stamp { } } -fn civil_from_days(days: i64) -> (i64, u32, u32) { - let shifted = days + 719_468; - let era = shifted.div_euclid(146_097); - let day_of_era = shifted.rem_euclid(146_097); - let year_of_era = - (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; - let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); - let month_index = (5 * day_of_year + 2) / 153; - let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32; - let month = if month_index < 10 { - month_index + 3 - } else { - month_index - 9 - } as u32; - let year = year_of_era + era * 400 + i64::from(month <= 2); - (year, month, day) -} - pub fn logging_id(start: &Stamp, id: &str) -> String { format!("time-{}_{id}", start.clock_micros()) } @@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str { } pub fn quote(key: &str) -> String { - let mut encoded = String::with_capacity(key.len()); - for byte in key.bytes() { - if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') { - encoded.push(byte as char); - } else { - encoded.push_str(&format!("%{byte:02X}")); - } - } - encoded + litellm_python_compat::urllib::quote(key, "/") } pub fn dns_suffix(region: Option<&str>) -> &'static str { diff --git a/litellm/rust/crates/callbacks-s3-log/src/lib.rs b/litellm/rust/crates/callbacks-s3-log/src/lib.rs index c8328c6ace6..6a4a0de1f73 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/lib.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/lib.rs @@ -1,6 +1,5 @@ mod key; mod limiter; -mod pyjson; mod settings; mod sink; mod upload; diff --git a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs b/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs deleted file mode 100644 index c2f442c9af9..00000000000 --- a/litellm/rust/crates/callbacks-s3-log/src/pyjson.rs +++ /dev/null @@ -1,214 +0,0 @@ -use serde_json::Value; - -const HEX: &[u8; 16] = b"0123456789abcdef"; - -pub fn write_str(out: &mut Vec, text: &str) { - out.push(b'"'); - let mut units = [0u16; 2]; - for character in text.chars() { - match character { - '"' => out.extend_from_slice(b"\\\""), - '\\' => out.extend_from_slice(b"\\\\"), - '\n' => out.extend_from_slice(b"\\n"), - '\r' => out.extend_from_slice(b"\\r"), - '\t' => out.extend_from_slice(b"\\t"), - '\u{8}' => out.extend_from_slice(b"\\b"), - '\u{c}' => out.extend_from_slice(b"\\f"), - ' '..='~' => out.push(character as u8), - _ => { - for unit in character.encode_utf16(&mut units) { - write_unit(out, *unit); - } - } - } - } - out.push(b'"'); -} - -fn write_unit(out: &mut Vec, unit: u16) { - out.extend_from_slice(b"\\u"); - for shift in [12, 8, 4, 0] { - out.push(HEX[usize::from((unit >> shift) & 0xf)]); - } -} - -pub fn float_repr(value: f64) -> String { - if value.is_nan() { - return "NaN".to_owned(); - } - if value.is_infinite() { - return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned(); - } - let shortest = format!("{value:e}"); - let length = shortest - .split('e') - .next() - .unwrap_or_default() - .chars() - .filter(char::is_ascii_digit) - .count(); - let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1)); - let (negative, scientific) = match scientific.strip_prefix('-') { - Some(rest) => (true, rest), - None => (false, scientific.as_str()), - }; - let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0")); - let exponent: i32 = exponent.parse().unwrap_or(0); - let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect(); - let mut text = String::with_capacity(digits.len() + 8); - if negative { - text.push('-'); - } - let point = exponent + 1; - if (-3..=16).contains(&point) { - let count = digits.len() as i32; - if point <= 0 { - text.push_str("0."); - text.extend(std::iter::repeat_n('0', (-point) as usize)); - text.push_str(&digits); - } else if point >= count { - text.push_str(&digits); - text.extend(std::iter::repeat_n('0', (point - count) as usize)); - text.push_str(".0"); - } else { - let (whole, fraction) = digits.split_at(point as usize); - text.push_str(whole); - text.push('.'); - text.push_str(fraction); - } - return text; - } - let (first, rest) = digits.split_at(1); - text.push_str(first); - if !rest.is_empty() { - text.push('.'); - text.push_str(rest); - } - text.push('e'); - text.push(if exponent < 0 { '-' } else { '+' }); - text.push_str(&format!("{:02}", exponent.unsigned_abs())); - text -} - -pub fn write_value(out: &mut Vec, value: &Value) { - match value { - Value::Null => out.extend_from_slice(b"null"), - Value::Bool(true) => out.extend_from_slice(b"true"), - Value::Bool(false) => out.extend_from_slice(b"false"), - Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) { - (Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()), - (None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()), - (None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()), - (None, None, None) => out.extend_from_slice(number.to_string().as_bytes()), - }, - Value::String(text) => write_str(out, text), - Value::Array(items) => { - out.push(b'['); - for (index, item) in items.iter().enumerate() { - if index > 0 { - out.extend_from_slice(b", "); - } - write_value(out, item); - } - out.push(b']'); - } - Value::Object(entries) => { - out.push(b'{'); - for (index, (key, item)) in entries.iter().enumerate() { - if index > 0 { - out.extend_from_slice(b", "); - } - write_str(out, key); - out.extend_from_slice(b": "); - write_value(out, item); - } - out.push(b'}'); - } - } -} - -pub fn dumps(value: &Value) -> String { - let mut out = Vec::new(); - write_value(&mut out, value); - String::from_utf8(out).unwrap_or_default() -} - -pub fn py_str(value: &Value) -> String { - match value { - Value::Null => "None".to_owned(), - Value::Bool(true) => "True".to_owned(), - Value::Bool(false) => "False".to_owned(), - Value::String(text) => text.clone(), - Value::Number(number) => match number.as_f64() { - Some(float) if number.is_f64() => float_repr(float), - _ => number.to_string(), - }, - Value::Array(_) | Value::Object(_) => dumps(value), - } -} - -pub fn truthy(value: &Value) -> bool { - match value { - Value::Null => false, - Value::Bool(flag) => *flag, - Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0), - Value::String(text) => !text.is_empty(), - Value::Array(items) => !items.is_empty(), - Value::Object(entries) => !entries.is_empty(), - } -} - -#[cfg(test)] -mod tests { - use rstest::rstest; - use serde_json::json; - - use super::*; - - #[rstest] - #[case(0.0, "0.0")] - #[case(-0.0, "-0.0")] - #[case(1.0, "1.0")] - #[case(0.1, "0.1")] - #[case(1.5e-5, "1.5e-05")] - #[case(1e-4, "0.0001")] - #[case(1e-5, "1e-05")] - #[case(1e15, "1000000000000000.0")] - #[case(1e16, "1e+16")] - #[case(1.2345678901234568e17, "1.2345678901234568e+17")] - #[case(123.456, "123.456")] - #[case(1e100, "1e+100")] - #[case(-2.5e-300, "-2.5e-300")] - #[case(f64::NAN, "NaN")] - #[case(f64::INFINITY, "Infinity")] - #[case(f64::NEG_INFINITY, "-Infinity")] - #[case(0.30000000000000004, "0.30000000000000004")] - #[case(0.07872390747070312, "0.07872390747070312")] - #[case(1794700894.4414062, "1794700894.4414062")] - fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) { - assert_eq!(float_repr(value), expected); - } - - #[rstest] - #[case("plain", "\"plain\"")] - #[case("quote\" back\\", "\"quote\\\" back\\\\\"")] - #[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")] - #[case("café", "\"caf\\u00e9\"")] - #[case("😀", "\"\\ud83d\\ude00\"")] - fn strings_are_escaped_as_ensure_ascii_escapes_them( - #[case] text: &str, - #[case] expected: &str, - ) { - let mut out = Vec::new(); - write_str(&mut out, text); - assert_eq!(String::from_utf8(out).unwrap(), expected); - } - - #[rstest] - fn containers_use_the_default_separators_and_keep_key_order() { - assert_eq!( - dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})), - "{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}" - ); - } -} diff --git a/litellm/rust/crates/callbacks-s3-log/src/settings.rs b/litellm/rust/crates/callbacks-s3-log/src/settings.rs index c266f612c94..d3d32eae77f 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/settings.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/settings.rs @@ -5,28 +5,28 @@ use litellm_secret_value::SecretValue; use serde::Deserialize; use serde_json::{Map, Value}; -use crate::pyjson::{py_str, truthy}; +use litellm_python_compat::{BigInt, json::from_json, pydantic, repr}; + +fn python(value: &Value) -> litellm_python_compat::Value { + from_json(value.clone()) +} fn py_repr(value: &Value) -> String { - litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone())) + repr::repr(&python(value)) +} + +pub fn py_str(value: &Value) -> String { + repr::to_str(&python(value)) +} + +pub fn truthy(value: &Value) -> bool { + litellm_python_compat::truthy::truthy(&python(value)) } fn py_optional(value: Option) -> String { value.map_or_else(|| "None".to_owned(), |value| value.to_string()) } -fn py_type_name(value: &Value) -> &'static str { - match value { - Value::Null => "NoneType", - Value::Bool(_) => "bool", - Value::Number(number) if number.is_f64() => "float", - Value::Number(_) => "int", - Value::String(_) => "str", - Value::Array(_) => "list", - Value::Object(_) => "dict", - } -} - pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY"; pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY"; const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000; @@ -122,71 +122,17 @@ fn is_blank(value: &Value) -> bool { } pub fn lax_bool(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(*flag), - Value::Number(number) => number - .as_f64() - .and_then(|float| { - [(0.0, false), (1.0, true)] - .into_iter() - .find(|(bound, _)| *bound == float) - }) - .map(|(_, flag)| flag), - Value::String(text) => match text.trim().to_ascii_lowercase().as_str() { - "0" | "off" | "f" | "false" | "n" | "no" => Some(false), - "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), - _ => None, - }, - _ => None, - } + pydantic::lax_bool(&python(value)) } pub fn lax_int(value: &Value) -> Option { - match value { - Value::Bool(flag) => Some(i128::from(*flag)), - Value::Number(number) => { - if let Some(integer) = number.as_i64() { - return Some(i128::from(integer)); - } - if let Some(integer) = number.as_u64() { - return Some(i128::from(integer)); - } - let float = number.as_f64()?; - (float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64) - .then_some(float as i128) - } - Value::String(text) => int_text(text.trim()), - _ => None, - } -} - -fn int_text(text: &str) -> Option { - let (whole, fraction) = text.split_once('.').unwrap_or((text, "")); - if !fraction.chars().all(|character| character == '0') { - return None; - } - if text.contains('.') && fraction.is_empty() { - return None; - } - let (sign, digits) = match whole.strip_prefix('-') { - Some(rest) => (-1, rest), - None => (1, whole.strip_prefix('+').unwrap_or(whole)), - }; - let well_formed = !digits.is_empty() - && !digits.starts_with('_') - && !digits.ends_with('_') - && !digits.contains("__") - && digits - .chars() - .all(|character| character.is_ascii_digit() || character == '_'); - if !well_formed { - return None; - } - digits - .replace('_', "") - .parse::() - .ok() - .map(|number| sign * number) + pydantic::lax_int(&python(value)).map(|number| { + i128::try_from(&number).unwrap_or(if number < BigInt::from(0) { + i128::MIN + } else { + i128::MAX + }) + }) } fn strip_text(value: &Value) -> Value { @@ -291,7 +237,7 @@ fn sse_value(name: &str, value: Option<&Value>) -> Option { Some(other) => { tracing::warn!( "s3 logging: ignoring {name} because it has invalid type {}; expected a string", - py_type_name(other) + python(other).type_name() ); None } @@ -556,6 +502,16 @@ mod tests { assert_eq!(lax_bool(&strip_text(&value)), expected); } + #[rstest] + #[case(json!(["us-east-1"]), "['us-east-1']")] + #[case(json!({"a": true}), "{'a': True}")] + #[case(json!(null), "None")] + #[case(json!(1.5), "1.5")] + #[case(json!("us-east-1"), "us-east-1")] + fn values_print_as_python_str_prints_them(#[case] value: Value, #[case] expected: &str) { + assert_eq!(py_str(&value), expected); + } + #[rstest] fn defaults_match_the_python_constructor() { let settings = settings(json!({})); diff --git a/litellm/rust/crates/callbacks-s3-log/src/upload.rs b/litellm/rust/crates/callbacks-s3-log/src/upload.rs index b5e4f0aa39f..6ca69562e39 100644 --- a/litellm/rust/crates/callbacks-s3-log/src/upload.rs +++ b/litellm/rust/crates/callbacks-s3-log/src/upload.rs @@ -384,7 +384,7 @@ impl Uploader { .settings .region_name .as_ref() - .map(crate::pyjson::py_str) + .map(crate::settings::py_str) else { tracing::error!("Error downloading from S3: no s3_region_name to sign with"); return None; diff --git a/litellm/rust/crates/python-compat/AGENTS.md b/litellm/rust/crates/python-compat/AGENTS.md index 55049daf612..7f8185d04d0 100644 --- a/litellm/rust/crates/python-compat/AGENTS.md +++ b/litellm/rust/crates/python-compat/AGENTS.md @@ -6,6 +6,7 @@ - `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`) - Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`) - Byte-identical writes where Python compares raw values (`json::dumps`, `repr`) + - `callbacks-s3-log`: settings parsed as pydantic parses them (`pydantic`), object keys quoted as `urllib.parse.quote` quotes them (`urllib`), UTC key dates (`datetime`) - Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce - Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128) - Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()` @@ -15,6 +16,7 @@ - Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail. Writes protocol 3 - Every behavior is pinned by CPython output, not by reasoning. Everything under `generated/` is script output, never hand-edited - Regenerate `generated/values.json` with `scripts/generate_fixtures.py`. Add a corpus row before changing behavior + - Regenerate `generated/stdlib.json` (`urllib.parse.quote`, UTC `datetime` fields) with `scripts/generate_stdlib_fixtures.py`, and `generated/pydantic.json` (lax `bool`/`int`) with `scripts/generate_pydantic_fixtures.py` at the pydantic version in `uv.lock` - Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason. An entry that starts matching fails until deleted - Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes - `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python diff --git a/litellm/rust/crates/python-compat/generated/.gitattributes b/litellm/rust/crates/python-compat/generated/.gitattributes new file mode 100644 index 00000000000..807ab52b2c8 --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/.gitattributes @@ -0,0 +1 @@ +* linguist-generated=true -diff diff --git a/litellm/rust/crates/python-compat/generated/pydantic.json b/litellm/rust/crates/python-compat/generated/pydantic.json new file mode 100644 index 00000000000..23ebee71515 --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/pydantic.json @@ -0,0 +1,485 @@ +{ + "pydantic": "2.13.4", + "rows": [ + { + "input": "null", + "bool": null, + "int": null + }, + { + "input": "true", + "bool": true, + "int": "1" + }, + { + "input": "false", + "bool": false, + "int": "0" + }, + { + "input": "0", + "bool": false, + "int": "0" + }, + { + "input": "1", + "bool": true, + "int": "1" + }, + { + "input": "2", + "bool": null, + "int": "2" + }, + { + "input": "-1", + "bool": null, + "int": "-1" + }, + { + "input": "9223372036854775807", + "bool": null, + "int": "9223372036854775807" + }, + { + "input": "9223372036854775808", + "bool": null, + "int": "9223372036854775808" + }, + { + "input": "18446744073709551616", + "bool": null, + "int": "18446744073709551616" + }, + { + "input": "-9223372036854775808", + "bool": null, + "int": "-9223372036854775808" + }, + { + "input": "0.0", + "bool": false, + "int": "0" + }, + { + "input": "-0.0", + "bool": false, + "int": "0" + }, + { + "input": "1.0", + "bool": true, + "int": "1" + }, + { + "input": "0.5", + "bool": null, + "int": null + }, + { + "input": "1.5", + "bool": null, + "int": null + }, + { + "input": "2.0", + "bool": null, + "int": "2" + }, + { + "input": "-3.0", + "bool": null, + "int": "-3" + }, + { + "input": "1000000000000000.0", + "bool": null, + "int": "1000000000000000" + }, + { + "input": "1e+20", + "bool": null, + "int": null + }, + { + "input": "1e+300", + "bool": null, + "int": null + }, + { + "input": "4.611686018427388e+18", + "bool": null, + "int": "4611686018427387904" + }, + { + "input": "9.223372036854776e+18", + "bool": null, + "int": null + }, + { + "input": "-9.223372036854776e+18", + "bool": null, + "int": null + }, + { + "input": "9007199254740992.0", + "bool": null, + "int": "9007199254740992" + }, + { + "input": "Infinity", + "bool": null, + "int": null + }, + { + "input": "-Infinity", + "bool": null, + "int": null + }, + { + "input": "NaN", + "bool": null, + "int": null + }, + { + "input": "[]", + "bool": null, + "int": null + }, + { + "input": "[1]", + "bool": null, + "int": null + }, + { + "input": "{}", + "bool": null, + "int": null + }, + { + "input": "{\"a\": 1}", + "bool": null, + "int": null + }, + { + "input": "\"\"", + "bool": null, + "int": null + }, + { + "input": "\" \"", + "bool": null, + "int": null + }, + { + "input": "\"0\"", + "bool": false, + "int": "0" + }, + { + "input": "\"1\"", + "bool": true, + "int": "1" + }, + { + "input": "\"2\"", + "bool": null, + "int": "2" + }, + { + "input": "\"-1\"", + "bool": null, + "int": "-1" + }, + { + "input": "\"+1\"", + "bool": null, + "int": "1" + }, + { + "input": "\"-0\"", + "bool": null, + "int": "0" + }, + { + "input": "\"00\"", + "bool": null, + "int": "0" + }, + { + "input": "\"007\"", + "bool": null, + "int": "7" + }, + { + "input": "\"1_000\"", + "bool": null, + "int": "1000" + }, + { + "input": "\"1__000\"", + "bool": null, + "int": null + }, + { + "input": "\"_1\"", + "bool": null, + "int": null + }, + { + "input": "\"1_\"", + "bool": null, + "int": null + }, + { + "input": "\"1.\"", + "bool": null, + "int": null + }, + { + "input": "\".5\"", + "bool": null, + "int": null + }, + { + "input": "\"1.0\"", + "bool": null, + "int": "1" + }, + { + "input": "\"1.00\"", + "bool": null, + "int": "1" + }, + { + "input": "\"1.5\"", + "bool": null, + "int": null + }, + { + "input": "\"-1.0\"", + "bool": null, + "int": "-1" + }, + { + "input": "\"1e3\"", + "bool": null, + "int": null + }, + { + "input": "\"1E3\"", + "bool": null, + "int": null + }, + { + "input": "\"0x10\"", + "bool": null, + "int": null + }, + { + "input": "\"0b1\"", + "bool": null, + "int": null + }, + { + "input": "\"0o7\"", + "bool": null, + "int": null + }, + { + "input": "\" 5 \"", + "bool": null, + "int": "5" + }, + { + "input": "\"\\t5\\n\"", + "bool": null, + "int": "5" + }, + { + "input": "\"5 0\"", + "bool": null, + "int": null + }, + { + "input": "\"\\u0661\\u0662\"", + "bool": null, + "int": null + }, + { + "input": "\"inf\"", + "bool": null, + "int": null + }, + { + "input": "\"nan\"", + "bool": null, + "int": null + }, + { + "input": "\"99999999999999999999\"", + "bool": null, + "int": "99999999999999999999" + }, + { + "input": "\"-99999999999999999999\"", + "bool": null, + "int": "-99999999999999999999" + }, + { + "input": "\"true\"", + "bool": true, + "int": null + }, + { + "input": "\"True\"", + "bool": true, + "int": null + }, + { + "input": "\"TRUE\"", + "bool": true, + "int": null + }, + { + "input": "\" true \"", + "bool": null, + "int": null + }, + { + "input": "\"false\"", + "bool": false, + "int": null + }, + { + "input": "\"False\"", + "bool": false, + "int": null + }, + { + "input": "\"t\"", + "bool": true, + "int": null + }, + { + "input": "\"f\"", + "bool": false, + "int": null + }, + { + "input": "\"T\"", + "bool": true, + "int": null + }, + { + "input": "\"y\"", + "bool": true, + "int": null + }, + { + "input": "\"n\"", + "bool": false, + "int": null + }, + { + "input": "\"yes\"", + "bool": true, + "int": null + }, + { + "input": "\"Yes\"", + "bool": true, + "int": null + }, + { + "input": "\"no\"", + "bool": false, + "int": null + }, + { + "input": "\"on\"", + "bool": true, + "int": null + }, + { + "input": "\"On\"", + "bool": true, + "int": null + }, + { + "input": "\"off\"", + "bool": false, + "int": null + }, + { + "input": "\"OFF\"", + "bool": false, + "int": null + }, + { + "input": "\"maybe\"", + "bool": null, + "int": null + }, + { + "input": "\"truthy\"", + "bool": null, + "int": null + }, + { + "input": "\"nul\"", + "bool": null, + "int": null + }, + { + "input": "\"+\"", + "bool": null, + "int": null + }, + { + "input": "\"-\"", + "bool": null, + "int": null + }, + { + "input": "\"+1.0\"", + "bool": null, + "int": "1" + }, + { + "input": "\"-0.0\"", + "bool": null, + "int": "0" + }, + { + "input": "\"1_0.0\"", + "bool": null, + "int": "10" + }, + { + "input": "\"1.0_0\"", + "bool": null, + "int": null + }, + { + "input": "\"1.0.0\"", + "bool": null, + "int": null + }, + { + "input": "\"1 \"", + "bool": null, + "int": "1" + }, + { + "input": "\"\\u00a05\"", + "bool": null, + "int": "5" + }, + { + "input": "\"\\u20035\"", + "bool": null, + "int": "5" + } + ] +} \ No newline at end of file diff --git a/litellm/rust/crates/python-compat/generated/stdlib.json b/litellm/rust/crates/python-compat/generated/stdlib.json new file mode 100644 index 00000000000..f8165ada7fc --- /dev/null +++ b/litellm/rust/crates/python-compat/generated/stdlib.json @@ -0,0 +1,345 @@ +{ + "python": "3.14.6", + "quote": [ + { + "text": "", + "safe": "/", + "quoted": "" + }, + { + "text": "", + "safe": "", + "quoted": "" + }, + { + "text": "", + "safe": ":/", + "quoted": "" + }, + { + "text": "plain", + "safe": "/", + "quoted": "plain" + }, + { + "text": "plain", + "safe": "", + "quoted": "plain" + }, + { + "text": "plain", + "safe": ":/", + "quoted": "plain" + }, + { + "text": "a/b/c", + "safe": "/", + "quoted": "a/b/c" + }, + { + "text": "a/b/c", + "safe": "", + "quoted": "a%2Fb%2Fc" + }, + { + "text": "a/b/c", + "safe": ":/", + "quoted": "a/b/c" + }, + { + "text": "a b", + "safe": "/", + "quoted": "a%20b" + }, + { + "text": "a b", + "safe": "", + "quoted": "a%20b" + }, + { + "text": "a b", + "safe": ":/", + "quoted": "a%20b" + }, + { + "text": "a+b=c&d", + "safe": "/", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "a+b=c&d", + "safe": "", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "a+b=c&d", + "safe": ":/", + "quoted": "a%2Bb%3Dc%26d" + }, + { + "text": "~_.-", + "safe": "/", + "quoted": "~_.-" + }, + { + "text": "~_.-", + "safe": "", + "quoted": "~_.-" + }, + { + "text": "~_.-", + "safe": ":/", + "quoted": "~_.-" + }, + { + "text": "100%", + "safe": "/", + "quoted": "100%25" + }, + { + "text": "100%", + "safe": "", + "quoted": "100%25" + }, + { + "text": "100%", + "safe": ":/", + "quoted": "100%25" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": "/", + "quoted": "a/b%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": "", + "quoted": "a%2Fb%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "a/b:c d%#?+=&~\u00e9\"q", + "safe": ":/", + "quoted": "a/b:c%20d%25%23%3F%2B%3D%26~%C3%A9%22q" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": "/", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": "", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "id-\u00e9-\ud83d\ude00", + "safe": ":/", + "quoted": "id-%C3%A9-%F0%9F%98%80" + }, + { + "text": "\u65e5\u672c", + "safe": "/", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u65e5\u672c", + "safe": "", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u65e5\u672c", + "safe": ":/", + "quoted": "%E6%97%A5%E6%9C%AC" + }, + { + "text": "\u0000\u001f\u007f", + "safe": "/", + "quoted": "%00%1F%7F" + }, + { + "text": "\u0000\u001f\u007f", + "safe": "", + "quoted": "%00%1F%7F" + }, + { + "text": "\u0000\u001f\u007f", + "safe": ":/", + "quoted": "%00%1F%7F" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": "/", + "quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C/%3F%23%5B%5D" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": "", + "quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C%2F%3F%23%5B%5D" + }, + { + "text": "!*'();:@&=+$,/?#[]", + "safe": ":/", + "quoted": "%21%2A%27%28%29%3B:%40%26%3D%2B%24%2C/%3F%23%5B%5D" + }, + { + "text": "<>\\^`{|}", + "safe": "/", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + }, + { + "text": "<>\\^`{|}", + "safe": "", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + }, + { + "text": "<>\\^`{|}", + "safe": ":/", + "quoted": "%3C%3E%5C%5E%60%7B%7C%7D" + } + ], + "utc": [ + { + "seconds": 0, + "micros": 0, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 1, + "micros": 999999, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 1, + 999999 + ] + }, + { + "seconds": 59, + "micros": 0, + "fields": [ + 1970, + 1, + 1, + 0, + 0, + 59, + 0 + ] + }, + { + "seconds": 86399, + "micros": 123456, + "fields": [ + 1970, + 1, + 1, + 23, + 59, + 59, + 123456 + ] + }, + { + "seconds": 86400, + "micros": 0, + "fields": [ + 1970, + 1, + 2, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 951782400, + "micros": 0, + "fields": [ + 2000, + 2, + 29, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 951868800, + "micros": 0, + "fields": [ + 2000, + 3, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 1700000000, + "micros": 42, + "fields": [ + 2023, + 11, + 14, + 22, + 13, + 20, + 42 + ] + }, + { + "seconds": 1757840707, + "micros": 42, + "fields": [ + 2025, + 9, + 14, + 9, + 5, + 7, + 42 + ] + }, + { + "seconds": 4102444800, + "micros": 0, + "fields": [ + 2100, + 1, + 1, + 0, + 0, + 0, + 0 + ] + }, + { + "seconds": 253402300799, + "micros": 999999, + "fields": [ + 9999, + 12, + 31, + 23, + 59, + 59, + 999999 + ] + } + ] +} \ No newline at end of file diff --git a/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py b/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py new file mode 100644 index 00000000000..139f48d4832 --- /dev/null +++ b/litellm/rust/crates/python-compat/scripts/generate_pydantic_fixtures.py @@ -0,0 +1,142 @@ +"""Regenerate generated/pydantic.json: what pydantic's lax `bool` and `int` validation return. + + uv run --no-project --with pydantic==2.13.4 python scripts/generate_pydantic_fixtures.py > generated/pydantic.json + +Each row records `TypeAdapter(bool).validate_python(value)` and `TypeAdapter(int).validate_python(value)` +for one input written as JSON, or null where pydantic raises `ValidationError`. Inputs are what a +JSON-shaped setting can carry: None, booleans, integers, floats, strings, lists and dicts. +""" + +import json +import sys + +import pydantic +from pydantic import TypeAdapter, ValidationError + +TEXTS = [ + "", + " ", + "0", + "1", + "2", + "-1", + "+1", + "-0", + "00", + "007", + "1_000", + "1__000", + "_1", + "1_", + "1.", + ".5", + "1.0", + "1.00", + "1.5", + "-1.0", + "1e3", + "1E3", + "0x10", + "0b1", + "0o7", + " 5 ", + "\t5\n", + "5 0", + "١٢", + "inf", + "nan", + "99999999999999999999", + "-99999999999999999999", + "true", + "True", + "TRUE", + " true ", + "false", + "False", + "t", + "f", + "T", + "y", + "n", + "yes", + "Yes", + "no", + "on", + "On", + "off", + "OFF", + "maybe", + "truthy", + "nul", + "+", + "-", + "+1.0", + "-0.0", + "1_0.0", + "1.0_0", + "1.0.0", + "1 ", + "\u00a05", + "\u20035", +] + +VALUES = [ + None, + True, + False, + 0, + 1, + 2, + -1, + 2**63 - 1, + 2**63, + 2**64, + -(2**63), + 0.0, + -0.0, + 1.0, + 0.5, + 1.5, + 2.0, + -3.0, + 1e15, + 1e20, + 1e300, + 2.0**62, + 2.0**63, + -(2.0**63), + 9007199254740993.0, + float("inf"), + float("-inf"), + float("nan"), + [], + [1], + {}, + {"a": 1}, + *TEXTS, +] + + +def validated(adapter, value): + try: + return adapter.validate_python(value) + except ValidationError: + return None + + +def row(value): + as_int = validated(TypeAdapter(int), value) + return { + "input": json.dumps(value), + "bool": validated(TypeAdapter(bool), value), + "int": None if as_int is None else str(as_int), + } + + +if __name__ == "__main__": + json.dump( + {"pydantic": pydantic.VERSION, "rows": [row(value) for value in VALUES]}, + sys.stdout, + indent=2, + ensure_ascii=True, + ) diff --git a/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py b/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py new file mode 100644 index 00000000000..2daa1e5ca33 --- /dev/null +++ b/litellm/rust/crates/python-compat/scripts/generate_stdlib_fixtures.py @@ -0,0 +1,66 @@ +"""Regenerate generated/stdlib.json: `urllib.parse.quote` and UTC `datetime` fields in CPython. + + python scripts/generate_stdlib_fixtures.py > generated/stdlib.json + +`quote` rows record `urllib.parse.quote(text, safe=safe)`. `utc` rows record the fields of +`datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)`. +""" + +import json +import sys +from datetime import datetime, timezone +from urllib.parse import quote + +QUOTE_TEXTS = [ + "", + "plain", + "a/b/c", + "a b", + "a+b=c&d", + "~_.-", + "100%", + "a/b:c d%#?+=&~é\"q", + "id-é-\U0001f600", + "日本", + "\x00\x1f\x7f", + "!*'();:@&=+$,/?#[]", + "<>\\^`{|}", +] + +SAFES = ["/", "", ":/"] + +TIMESTAMPS = [ + (0, 0), + (1, 999999), + (59, 0), + (86399, 123456), + (86400, 0), + (951782400, 0), + (951868800, 0), + (1700000000, 42), + (1757840707, 42), + (4102444800, 0), + (253402300799, 999999), +] + + +def utc(seconds, micros): + moment = datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros) + return { + "seconds": seconds, + "micros": micros, + "fields": [moment.year, moment.month, moment.day, moment.hour, moment.minute, moment.second, moment.microsecond], + } + + +if __name__ == "__main__": + json.dump( + { + "python": sys.version.split()[0], + "quote": [{"text": text, "safe": safe, "quoted": quote(text, safe=safe)} for text in QUOTE_TEXTS for safe in SAFES], + "utc": [utc(seconds, micros) for seconds, micros in TIMESTAMPS], + }, + sys.stdout, + indent=2, + ensure_ascii=True, + ) diff --git a/litellm/rust/crates/python-compat/src/datetime.rs b/litellm/rust/crates/python-compat/src/datetime.rs new file mode 100644 index 00000000000..bc9b022dac7 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/datetime.rs @@ -0,0 +1,53 @@ +//! The fields of `datetime.fromtimestamp(seconds, timezone.utc)`. + +/// A UTC `datetime`'s fields, as Python's `datetime` exposes them. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct UtcDateTime { + pub year: i64, + pub month: u32, + pub day: u32, + pub hour: u32, + pub minute: u32, + pub second: u32, + pub microsecond: u32, +} + +impl UtcDateTime { + /// `datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=microsecond)`. + pub fn from_timestamp(seconds: i64, microsecond: u32) -> Self { + let (year, month, day) = civil_from_days(seconds.div_euclid(86_400)); + let of_day = seconds.rem_euclid(86_400); + Self { + year, + month, + day, + hour: (of_day / 3600) as u32, + minute: (of_day / 60 % 60) as u32, + second: (of_day % 60) as u32, + microsecond, + } + } +} + +/// Days since 1970-01-01 to a proleptic Gregorian `(year, month, day)`. +/// +/// Howard Hinnant's `civil_from_days`, : +/// shifts the epoch to 0000-03-01 so the leap day ends the year, then counts 400-year eras +/// of 146097 days. +fn civil_from_days(days: i64) -> (i64, u32, u32) { + let shifted = days + 719_468; + let era = shifted.div_euclid(146_097); + let day_of_era = shifted.rem_euclid(146_097); + let year_of_era = + (day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365; + let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100); + let month_index = (5 * day_of_year + 2) / 153; + let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32; + let month = if month_index < 10 { + month_index + 3 + } else { + month_index - 9 + } as u32; + let year = year_of_era + era * 400 + i64::from(month <= 2); + (year, month, day) +} diff --git a/litellm/rust/crates/python-compat/src/lib.rs b/litellm/rust/crates/python-compat/src/lib.rs index e23dbdb285e..16f6ca239a3 100644 --- a/litellm/rust/crates/python-compat/src/lib.rs +++ b/litellm/rust/crates/python-compat/src/lib.rs @@ -12,6 +12,9 @@ //! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data | //! | [`truthy`] | `bool(value)` | //! | [`number`] | `int(text)`, `float(text)`, and `int(seconds)` as a `Duration` | +//! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation | +//! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` | +//! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields | //! //! [`Value`] is the closed data model these formats share. Live Python objects //! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the @@ -24,14 +27,17 @@ //! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists. //! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit. +pub mod datetime; mod error; pub mod json; pub mod literal; pub mod number; pub mod pickle; +pub mod pydantic; pub mod repr; pub mod reprlib; pub mod truthy; +pub mod urllib; mod value; pub use error::Error; diff --git a/litellm/rust/crates/python-compat/src/pydantic.rs b/litellm/rust/crates/python-compat/src/pydantic.rs new file mode 100644 index 00000000000..35a12890dda --- /dev/null +++ b/litellm/rust/crates/python-compat/src/pydantic.rs @@ -0,0 +1,70 @@ +//! Pydantic's lax `bool` and `int` validation: `TypeAdapter(bool)` and `TypeAdapter(int)`. + +use num_bigint::BigInt; + +use crate::Value; + +const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0; + +/// `TypeAdapter(bool).validate_python(value)`, or `None` where pydantic raises `ValidationError`. +pub fn lax_bool(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(*flag), + Value::Int(number) => match i8::try_from(number) { + Ok(0) => Some(false), + Ok(1) => Some(true), + _ => None, + }, + Value::Float(number) if *number == 0.0 => Some(false), + Value::Float(number) if *number == 1.0 => Some(true), + Value::Str(text) => match text.to_ascii_lowercase().as_str() { + "0" | "off" | "f" | "false" | "n" | "no" => Some(false), + "1" | "on" | "t" | "true" | "y" | "yes" => Some(true), + _ => None, + }, + _ => None, + } +} + +/// `TypeAdapter(int).validate_python(value)`, or `None` where pydantic raises `ValidationError`. +pub fn lax_int(value: &Value) -> Option { + match value { + Value::Bool(flag) => Some(BigInt::from(u8::from(*flag))), + Value::Int(number) => Some(number.clone()), + Value::Float(number) + if number.fract() == 0.0 && (-TWO_POW_63 < *number && *number < TWO_POW_63) => + { + Some(BigInt::from(*number as i64)) + } + Value::Str(text) => int_text(text.trim()), + _ => None, + } +} + +fn int_text(text: &str) -> Option { + let whole = match text.split_once('.') { + Some((whole, fraction)) + if !fraction.is_empty() && fraction.bytes().all(|byte| byte == b'0') => + { + whole + } + Some(_) => return None, + None => text, + }; + let (negative, digits) = match whole.strip_prefix('-') { + Some(rest) => (true, rest), + None => (false, whole.strip_prefix('+').unwrap_or(whole)), + }; + let well_formed = !digits.is_empty() + && !digits.starts_with('_') + && !digits.ends_with('_') + && !digits.contains("__") + && digits + .bytes() + .all(|byte| byte.is_ascii_digit() || byte == b'_'); + if !well_formed { + return None; + } + let magnitude = BigInt::parse_bytes(digits.replace('_', "").as_bytes(), 10)?; + Some(if negative { -magnitude } else { magnitude }) +} diff --git a/litellm/rust/crates/python-compat/src/urllib.rs b/litellm/rust/crates/python-compat/src/urllib.rs new file mode 100644 index 00000000000..6d68249c693 --- /dev/null +++ b/litellm/rust/crates/python-compat/src/urllib.rs @@ -0,0 +1,18 @@ +//! `urllib.parse.quote(text, safe=safe)` over the UTF-8 bytes of `text`. + +/// `urllib.parse.quote(text, safe=safe)`: ASCII letters, digits and `_.-~` are never quoted, +/// the ASCII characters in `safe` are kept, and every other byte becomes `%XX`. +pub fn quote(text: &str, safe: &str) -> String { + let mut encoded = String::with_capacity(text.len()); + for byte in text.bytes() { + if byte.is_ascii_alphanumeric() + || matches!(byte, b'_' | b'.' | b'-' | b'~') + || (byte.is_ascii() && safe.as_bytes().contains(&byte)) + { + encoded.push(char::from(byte)); + } else { + encoded.push_str(&format!("%{byte:02X}")); + } + } + encoded +} diff --git a/litellm/rust/crates/python-compat/tests/pydantic.rs b/litellm/rust/crates/python-compat/tests/pydantic.rs new file mode 100644 index 00000000000..bf4d690f6f6 --- /dev/null +++ b/litellm/rust/crates/python-compat/tests/pydantic.rs @@ -0,0 +1,48 @@ +//! Replays `generated/pydantic.json`, which pydantic wrote with `scripts/generate_pydantic_fixtures.py`. + +use litellm_python_compat::{ + json, + pydantic::{lax_bool, lax_int}, +}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + rows: Vec, +} + +#[derive(Deserialize)] +struct Row { + input: String, + bool: Option, + int: Option, +} + +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/pydantic.json" + ))) + .expect("pydantic.json matches the fixture schema") +} + +#[test] +fn lax_bool_and_int_match_pydantic() { + let mismatches: Vec = fixtures() + .rows + .iter() + .flat_map(|row| { + let value = json::loads(&row.input).expect("inputs are json.dumps output"); + let as_bool = lax_bool(&value); + let as_int = lax_int(&value).map(|number| number.to_string()); + [ + (as_bool != row.bool) + .then(|| format!("bool {}: {as_bool:?} != {:?}", row.input, row.bool)), + (as_int != row.int) + .then(|| format!("int {}: {as_int:?} != {:?}", row.input, row.int)), + ] + }) + .flatten() + .collect(); + assert!(mismatches.is_empty(), "{mismatches:#?}"); +} diff --git a/litellm/rust/crates/python-compat/tests/stdlib.rs b/litellm/rust/crates/python-compat/tests/stdlib.rs new file mode 100644 index 00000000000..0b6db098557 --- /dev/null +++ b/litellm/rust/crates/python-compat/tests/stdlib.rs @@ -0,0 +1,62 @@ +//! Replays `generated/stdlib.json`, which CPython wrote with `scripts/generate_stdlib_fixtures.py`. + +use litellm_python_compat::{datetime::UtcDateTime, urllib::quote}; +use serde::Deserialize; + +#[derive(Deserialize)] +struct Fixtures { + quote: Vec, + utc: Vec, +} + +#[derive(Deserialize)] +struct Quote { + text: String, + safe: String, + quoted: String, +} + +#[derive(Deserialize)] +struct Utc { + seconds: i64, + micros: u32, + fields: [i64; 7], +} + +fn fixtures() -> Fixtures { + serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/generated/stdlib.json" + ))) + .expect("stdlib.json matches the fixture schema") +} + +#[test] +fn quote_matches_urllib() { + for row in fixtures().quote { + assert_eq!( + quote(&row.text, &row.safe), + row.quoted, + "{:?} safe={:?}", + row.text, + row.safe + ); + } +} + +#[test] +fn utc_fields_match_datetime() { + for row in fixtures().utc { + let moment = UtcDateTime::from_timestamp(row.seconds, row.micros); + let fields = [ + moment.year, + i64::from(moment.month), + i64::from(moment.day), + i64::from(moment.hour), + i64::from(moment.minute), + i64::from(moment.second), + i64::from(moment.microsecond), + ]; + assert_eq!(fields, row.fields, "{} {}", row.seconds, row.micros); + } +}