refactor(s3-log): move python-compatible helpers into python-compat (#51895)

This commit is contained in:
Nate Armstrong 2026-11-15 11:19:03 -08:00 • committed by GitHub
parent 09ce4f50ed
commit d63224db15
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
17 changed files with 1345 additions and 329 deletions

View file

@ -1,6 +1,7 @@
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_callbacks::Moment;
use litellm_python_compat::datetime::UtcDateTime;
use sha2::{Digest, Sha256};
use crate::settings::{Granularity, Settings};
@ -58,16 +59,18 @@ impl Stamp {
pub fn utc(time: SystemTime) -> Self {
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
let seconds = elapsed.as_secs();
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
let of_day = seconds % 86_400;
let moment = UtcDateTime::from_timestamp(
i64::try_from(seconds).unwrap_or(i64::MAX),
elapsed.subsec_micros(),
);
Self {
year,
month,
day,
hour: (of_day / 3600) as u32,
minute: (of_day / 60 % 60) as u32,
second: (of_day % 60) as u32,
micros: elapsed.subsec_micros(),
year: moment.year,
month: moment.month,
day: moment.day,
hour: moment.hour,
minute: moment.minute,
second: moment.second,
micros: moment.microsecond,
}
}
@ -84,24 +87,6 @@ impl Stamp {
}
}
fn civil_from_days(days: i64) -> (i64, u32, u32) {
let shifted = days + 719_468;
let era = shifted.div_euclid(146_097);
let day_of_era = shifted.rem_euclid(146_097);
let year_of_era =
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let month_index = (5 * day_of_year + 2) / 153;
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
let month = if month_index < 10 {
month_index + 3
} else {
month_index - 9
} as u32;
let year = year_of_era + era * 400 + i64::from(month <= 2);
(year, month, day)
}
pub fn logging_id(start: &Stamp, id: &str) -> String {
format!("time-{}_{id}", start.clock_micros())
}
@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str {
}
pub fn quote(key: &str) -> String {
let mut encoded = String::with_capacity(key.len());
for byte in key.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
encoded.push(byte as char);
} else {
encoded.push_str(&format!("%{byte:02X}"));
}
}
encoded
litellm_python_compat::urllib::quote(key, "/")
}
pub fn dns_suffix(region: Option<&str>) -> &'static str {

View file

@ -1,6 +1,5 @@
mod key;
mod limiter;
mod pyjson;
mod settings;
mod sink;
mod upload;

View file

@ -1,214 +0,0 @@
use serde_json::Value;
const HEX: &[u8; 16] = b"0123456789abcdef";
pub fn write_str(out: &mut Vec<u8>, text: &str) {
out.push(b'"');
let mut units = [0u16; 2];
for character in text.chars() {
match character {
'"' => out.extend_from_slice(b"\\\""),
'\\' => out.extend_from_slice(b"\\\\"),
'\n' => out.extend_from_slice(b"\\n"),
'\r' => out.extend_from_slice(b"\\r"),
'\t' => out.extend_from_slice(b"\\t"),
'\u{8}' => out.extend_from_slice(b"\\b"),
'\u{c}' => out.extend_from_slice(b"\\f"),
' '..='~' => out.push(character as u8),
_ => {
for unit in character.encode_utf16(&mut units) {
write_unit(out, *unit);
}
}
}
}
out.push(b'"');
}
fn write_unit(out: &mut Vec<u8>, unit: u16) {
out.extend_from_slice(b"\\u");
for shift in [12, 8, 4, 0] {
out.push(HEX[usize::from((unit >> shift) & 0xf)]);
}
}
pub fn float_repr(value: f64) -> String {
if value.is_nan() {
return "NaN".to_owned();
}
if value.is_infinite() {
return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned();
}
let shortest = format!("{value:e}");
let length = shortest
.split('e')
.next()
.unwrap_or_default()
.chars()
.filter(char::is_ascii_digit)
.count();
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
let (negative, scientific) = match scientific.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, scientific.as_str()),
};
let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0"));
let exponent: i32 = exponent.parse().unwrap_or(0);
let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
let mut text = String::with_capacity(digits.len() + 8);
if negative {
text.push('-');
}
let point = exponent + 1;
if (-3..=16).contains(&point) {
let count = digits.len() as i32;
if point <= 0 {
text.push_str("0.");
text.extend(std::iter::repeat_n('0', (-point) as usize));
text.push_str(&digits);
} else if point >= count {
text.push_str(&digits);
text.extend(std::iter::repeat_n('0', (point - count) as usize));
text.push_str(".0");
} else {
let (whole, fraction) = digits.split_at(point as usize);
text.push_str(whole);
text.push('.');
text.push_str(fraction);
}
return text;
}
let (first, rest) = digits.split_at(1);
text.push_str(first);
if !rest.is_empty() {
text.push('.');
text.push_str(rest);
}
text.push('e');
text.push(if exponent < 0 { '-' } else { '+' });
text.push_str(&format!("{:02}", exponent.unsigned_abs()));
text
}
pub fn write_value(out: &mut Vec<u8>, value: &Value) {
match value {
Value::Null => out.extend_from_slice(b"null"),
Value::Bool(true) => out.extend_from_slice(b"true"),
Value::Bool(false) => out.extend_from_slice(b"false"),
Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) {
(Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()),
(None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()),
(None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()),
(None, None, None) => out.extend_from_slice(number.to_string().as_bytes()),
},
Value::String(text) => write_str(out, text),
Value::Array(items) => {
out.push(b'[');
for (index, item) in items.iter().enumerate() {
if index > 0 {
out.extend_from_slice(b", ");
}
write_value(out, item);
}
out.push(b']');
}
Value::Object(entries) => {
out.push(b'{');
for (index, (key, item)) in entries.iter().enumerate() {
if index > 0 {
out.extend_from_slice(b", ");
}
write_str(out, key);
out.extend_from_slice(b": ");
write_value(out, item);
}
out.push(b'}');
}
}
}
pub fn dumps(value: &Value) -> String {
let mut out = Vec::new();
write_value(&mut out, value);
String::from_utf8(out).unwrap_or_default()
}
pub fn py_str(value: &Value) -> String {
match value {
Value::Null => "None".to_owned(),
Value::Bool(true) => "True".to_owned(),
Value::Bool(false) => "False".to_owned(),
Value::String(text) => text.clone(),
Value::Number(number) => match number.as_f64() {
Some(float) if number.is_f64() => float_repr(float),
_ => number.to_string(),
},
Value::Array(_) | Value::Object(_) => dumps(value),
}
}
pub fn truthy(value: &Value) -> bool {
match value {
Value::Null => false,
Value::Bool(flag) => *flag,
Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0),
Value::String(text) => !text.is_empty(),
Value::Array(items) => !items.is_empty(),
Value::Object(entries) => !entries.is_empty(),
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use serde_json::json;
use super::*;
#[rstest]
#[case(0.0, "0.0")]
#[case(-0.0, "-0.0")]
#[case(1.0, "1.0")]
#[case(0.1, "0.1")]
#[case(1.5e-5, "1.5e-05")]
#[case(1e-4, "0.0001")]
#[case(1e-5, "1e-05")]
#[case(1e15, "1000000000000000.0")]
#[case(1e16, "1e+16")]
#[case(1.2345678901234568e17, "1.2345678901234568e+17")]
#[case(123.456, "123.456")]
#[case(1e100, "1e+100")]
#[case(-2.5e-300, "-2.5e-300")]
#[case(f64::NAN, "NaN")]
#[case(f64::INFINITY, "Infinity")]
#[case(f64::NEG_INFINITY, "-Infinity")]
#[case(0.30000000000000004, "0.30000000000000004")]
#[case(0.07872390747070312, "0.07872390747070312")]
#[case(1794700894.4414062, "1794700894.4414062")]
fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) {
assert_eq!(float_repr(value), expected);
}
#[rstest]
#[case("plain", "\"plain\"")]
#[case("quote\" back\\", "\"quote\\\" back\\\\\"")]
#[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")]
#[case("café", "\"caf\\u00e9\"")]
#[case("😀", "\"\\ud83d\\ude00\"")]
fn strings_are_escaped_as_ensure_ascii_escapes_them(
#[case] text: &str,
#[case] expected: &str,
) {
let mut out = Vec::new();
write_str(&mut out, text);
assert_eq!(String::from_utf8(out).unwrap(), expected);
}
#[rstest]
fn containers_use_the_default_separators_and_keep_key_order() {
assert_eq!(
dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})),
"{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}"
);
}
}

View file

@ -5,28 +5,28 @@ use litellm_secret_value::SecretValue;
use serde::Deserialize;
use serde_json::{Map, Value};
use crate::pyjson::{py_str, truthy};
use litellm_python_compat::{BigInt, json::from_json, pydantic, repr};
fn python(value: &Value) -> litellm_python_compat::Value {
from_json(value.clone())
}
fn py_repr(value: &Value) -> String {
litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone()))
repr::repr(&python(value))
}
pub fn py_str(value: &Value) -> String {
repr::to_str(&python(value))
}
pub fn truthy(value: &Value) -> bool {
litellm_python_compat::truthy::truthy(&python(value))
}
fn py_optional(value: Option<u64>) -> String {
value.map_or_else(|| "None".to_owned(), |value| value.to_string())
}
fn py_type_name(value: &Value) -> &'static str {
match value {
Value::Null => "NoneType",
Value::Bool(_) => "bool",
Value::Number(number) if number.is_f64() => "float",
Value::Number(_) => "int",
Value::String(_) => "str",
Value::Array(_) => "list",
Value::Object(_) => "dict",
}
}
pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY";
pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY";
const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000;
@ -122,71 +122,17 @@ fn is_blank(value: &Value) -> bool {
}
pub fn lax_bool(value: &Value) -> Option<bool> {
match value {
Value::Bool(flag) => Some(*flag),
Value::Number(number) => number
.as_f64()
.and_then(|float| {
[(0.0, false), (1.0, true)]
.into_iter()
.find(|(bound, _)| *bound == float)
})
.map(|(_, flag)| flag),
Value::String(text) => match text.trim().to_ascii_lowercase().as_str() {
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
_ => None,
},
_ => None,
}
pydantic::lax_bool(&python(value))
}
pub fn lax_int(value: &Value) -> Option<i128> {
match value {
Value::Bool(flag) => Some(i128::from(*flag)),
Value::Number(number) => {
if let Some(integer) = number.as_i64() {
return Some(i128::from(integer));
}
if let Some(integer) = number.as_u64() {
return Some(i128::from(integer));
}
let float = number.as_f64()?;
(float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64)
.then_some(float as i128)
}
Value::String(text) => int_text(text.trim()),
_ => None,
}
}
fn int_text(text: &str) -> Option<i128> {
let (whole, fraction) = text.split_once('.').unwrap_or((text, ""));
if !fraction.chars().all(|character| character == '0') {
return None;
}
if text.contains('.') && fraction.is_empty() {
return None;
}
let (sign, digits) = match whole.strip_prefix('-') {
Some(rest) => (-1, rest),
None => (1, whole.strip_prefix('+').unwrap_or(whole)),
};
let well_formed = !digits.is_empty()
&& !digits.starts_with('_')
&& !digits.ends_with('_')
&& !digits.contains("__")
&& digits
.chars()
.all(|character| character.is_ascii_digit() || character == '_');
if !well_formed {
return None;
}
digits
.replace('_', "")
.parse::<i128>()
.ok()
.map(|number| sign * number)
pydantic::lax_int(&python(value)).map(|number| {
i128::try_from(&number).unwrap_or(if number < BigInt::from(0) {
i128::MIN
} else {
i128::MAX
})
})
}
fn strip_text(value: &Value) -> Value {
@ -291,7 +237,7 @@ fn sse_value(name: &str, value: Option<&Value>) -> Option<String> {
Some(other) => {
tracing::warn!(
"s3 logging: ignoring {name} because it has invalid type {}; expected a string",
py_type_name(other)
python(other).type_name()
);
None
}
@ -556,6 +502,16 @@ mod tests {
assert_eq!(lax_bool(&strip_text(&value)), expected);
}
#[rstest]
#[case(json!(["us-east-1"]), "['us-east-1']")]
#[case(json!({"a": true}), "{'a': True}")]
#[case(json!(null), "None")]
#[case(json!(1.5), "1.5")]
#[case(json!("us-east-1"), "us-east-1")]
fn values_print_as_python_str_prints_them(#[case] value: Value, #[case] expected: &str) {
assert_eq!(py_str(&value), expected);
}
#[rstest]
fn defaults_match_the_python_constructor() {
let settings = settings(json!({}));

View file

@ -384,7 +384,7 @@ impl Uploader {
.settings
.region_name
.as_ref()
.map(crate::pyjson::py_str)
.map(crate::settings::py_str)
else {
tracing::error!("Error downloading from S3: no s3_region_name to sign with");
return None;

View file

@ -6,6 +6,7 @@
- `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`)
- Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`)
- Byte-identical writes where Python compares raw values (`json::dumps`, `repr`)
- `callbacks-s3-log`: settings parsed as pydantic parses them (`pydantic`), object keys quoted as `urllib.parse.quote` quotes them (`urllib`), UTC key dates (`datetime`)
- Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce
- Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128)
- Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()`
@ -15,6 +16,7 @@
- Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail. Writes protocol 3
- Every behavior is pinned by CPython output, not by reasoning. Everything under `generated/` is script output, never hand-edited
- Regenerate `generated/values.json` with `scripts/generate_fixtures.py`. Add a corpus row before changing behavior
- Regenerate `generated/stdlib.json` (`urllib.parse.quote`, UTC `datetime` fields) with `scripts/generate_stdlib_fixtures.py`, and `generated/pydantic.json` (lax `bool`/`int`) with `scripts/generate_pydantic_fixtures.py` at the pydantic version in `uv.lock`
- Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason. An entry that starts matching fails until deleted
- Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes
- `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python

View file

@ -0,0 +1 @@
* linguist-generated=true -diff

View file

@ -0,0 +1,485 @@
{
"pydantic": "2.13.4",
"rows": [
{
"input": "null",
"bool": null,
"int": null
},
{
"input": "true",
"bool": true,
"int": "1"
},
{
"input": "false",
"bool": false,
"int": "0"
},
{
"input": "0",
"bool": false,
"int": "0"
},
{
"input": "1",
"bool": true,
"int": "1"
},
{
"input": "2",
"bool": null,
"int": "2"
},
{
"input": "-1",
"bool": null,
"int": "-1"
},
{
"input": "9223372036854775807",
"bool": null,
"int": "9223372036854775807"
},
{
"input": "9223372036854775808",
"bool": null,
"int": "9223372036854775808"
},
{
"input": "18446744073709551616",
"bool": null,
"int": "18446744073709551616"
},
{
"input": "-9223372036854775808",
"bool": null,
"int": "-9223372036854775808"
},
{
"input": "0.0",
"bool": false,
"int": "0"
},
{
"input": "-0.0",
"bool": false,
"int": "0"
},
{
"input": "1.0",
"bool": true,
"int": "1"
},
{
"input": "0.5",
"bool": null,
"int": null
},
{
"input": "1.5",
"bool": null,
"int": null
},
{
"input": "2.0",
"bool": null,
"int": "2"
},
{
"input": "-3.0",
"bool": null,
"int": "-3"
},
{
"input": "1000000000000000.0",
"bool": null,
"int": "1000000000000000"
},
{
"input": "1e+20",
"bool": null,
"int": null
},
{
"input": "1e+300",
"bool": null,
"int": null
},
{
"input": "4.611686018427388e+18",
"bool": null,
"int": "4611686018427387904"
},
{
"input": "9.223372036854776e+18",
"bool": null,
"int": null
},
{
"input": "-9.223372036854776e+18",
"bool": null,
"int": null
},
{
"input": "9007199254740992.0",
"bool": null,
"int": "9007199254740992"
},
{
"input": "Infinity",
"bool": null,
"int": null
},
{
"input": "-Infinity",
"bool": null,
"int": null
},
{
"input": "NaN",
"bool": null,
"int": null
},
{
"input": "[]",
"bool": null,
"int": null
},
{
"input": "[1]",
"bool": null,
"int": null
},
{
"input": "{}",
"bool": null,
"int": null
},
{
"input": "{\"a\": 1}",
"bool": null,
"int": null
},
{
"input": "\"\"",
"bool": null,
"int": null
},
{
"input": "\" \"",
"bool": null,
"int": null
},
{
"input": "\"0\"",
"bool": false,
"int": "0"
},
{
"input": "\"1\"",
"bool": true,
"int": "1"
},
{
"input": "\"2\"",
"bool": null,
"int": "2"
},
{
"input": "\"-1\"",
"bool": null,
"int": "-1"
},
{
"input": "\"+1\"",
"bool": null,
"int": "1"
},
{
"input": "\"-0\"",
"bool": null,
"int": "0"
},
{
"input": "\"00\"",
"bool": null,
"int": "0"
},
{
"input": "\"007\"",
"bool": null,
"int": "7"
},
{
"input": "\"1_000\"",
"bool": null,
"int": "1000"
},
{
"input": "\"1__000\"",
"bool": null,
"int": null
},
{
"input": "\"_1\"",
"bool": null,
"int": null
},
{
"input": "\"1_\"",
"bool": null,
"int": null
},
{
"input": "\"1.\"",
"bool": null,
"int": null
},
{
"input": "\".5\"",
"bool": null,
"int": null
},
{
"input": "\"1.0\"",
"bool": null,
"int": "1"
},
{
"input": "\"1.00\"",
"bool": null,
"int": "1"
},
{
"input": "\"1.5\"",
"bool": null,
"int": null
},
{
"input": "\"-1.0\"",
"bool": null,
"int": "-1"
},
{
"input": "\"1e3\"",
"bool": null,
"int": null
},
{
"input": "\"1E3\"",
"bool": null,
"int": null
},
{
"input": "\"0x10\"",
"bool": null,
"int": null
},
{
"input": "\"0b1\"",
"bool": null,
"int": null
},
{
"input": "\"0o7\"",
"bool": null,
"int": null
},
{
"input": "\" 5 \"",
"bool": null,
"int": "5"
},
{
"input": "\"\\t5\\n\"",
"bool": null,
"int": "5"
},
{
"input": "\"5 0\"",
"bool": null,
"int": null
},
{
"input": "\"\\u0661\\u0662\"",
"bool": null,
"int": null
},
{
"input": "\"inf\"",
"bool": null,
"int": null
},
{
"input": "\"nan\"",
"bool": null,
"int": null
},
{
"input": "\"99999999999999999999\"",
"bool": null,
"int": "99999999999999999999"
},
{
"input": "\"-99999999999999999999\"",
"bool": null,
"int": "-99999999999999999999"
},
{
"input": "\"true\"",
"bool": true,
"int": null
},
{
"input": "\"True\"",
"bool": true,
"int": null
},
{
"input": "\"TRUE\"",
"bool": true,
"int": null
},
{
"input": "\" true \"",
"bool": null,
"int": null
},
{
"input": "\"false\"",
"bool": false,
"int": null
},
{
"input": "\"False\"",
"bool": false,
"int": null
},
{
"input": "\"t\"",
"bool": true,
"int": null
},
{
"input": "\"f\"",
"bool": false,
"int": null
},
{
"input": "\"T\"",
"bool": true,
"int": null
},
{
"input": "\"y\"",
"bool": true,
"int": null
},
{
"input": "\"n\"",
"bool": false,
"int": null
},
{
"input": "\"yes\"",
"bool": true,
"int": null
},
{
"input": "\"Yes\"",
"bool": true,
"int": null
},
{
"input": "\"no\"",
"bool": false,
"int": null
},
{
"input": "\"on\"",
"bool": true,
"int": null
},
{
"input": "\"On\"",
"bool": true,
"int": null
},
{
"input": "\"off\"",
"bool": false,
"int": null
},
{
"input": "\"OFF\"",
"bool": false,
"int": null
},
{
"input": "\"maybe\"",
"bool": null,
"int": null
},
{
"input": "\"truthy\"",
"bool": null,
"int": null
},
{
"input": "\"nul\"",
"bool": null,
"int": null
},
{
"input": "\"+\"",
"bool": null,
"int": null
},
{
"input": "\"-\"",
"bool": null,
"int": null
},
{
"input": "\"+1.0\"",
"bool": null,
"int": "1"
},
{
"input": "\"-0.0\"",
"bool": null,
"int": "0"
},
{
"input": "\"1_0.0\"",
"bool": null,
"int": "10"
},
{
"input": "\"1.0_0\"",
"bool": null,
"int": null
},
{
"input": "\"1.0.0\"",
"bool": null,
"int": null
},
{
"input": "\"1 \"",
"bool": null,
"int": "1"
},
{
"input": "\"\\u00a05\"",
"bool": null,
"int": "5"
},
{
"input": "\"\\u20035\"",
"bool": null,
"int": "5"
}
]
}

View file

@ -0,0 +1,345 @@
{
"python": "3.14.6",
"quote": [
{
"text": "",
"safe": "/",
"quoted": ""
},
{
"text": "",
"safe": "",
"quoted": ""
},
{
"text": "",
"safe": ":/",
"quoted": ""
},
{
"text": "plain",
"safe": "/",
"quoted": "plain"
},
{
"text": "plain",
"safe": "",
"quoted": "plain"
},
{
"text": "plain",
"safe": ":/",
"quoted": "plain"
},
{
"text": "a/b/c",
"safe": "/",
"quoted": "a/b/c"
},
{
"text": "a/b/c",
"safe": "",
"quoted": "a%2Fb%2Fc"
},
{
"text": "a/b/c",
"safe": ":/",
"quoted": "a/b/c"
},
{
"text": "a b",
"safe": "/",
"quoted": "a%20b"
},
{
"text": "a b",
"safe": "",
"quoted": "a%20b"
},
{
"text": "a b",
"safe": ":/",
"quoted": "a%20b"
},
{
"text": "a+b=c&d",
"safe": "/",
"quoted": "a%2Bb%3Dc%26d"
},
{
"text": "a+b=c&d",
"safe": "",
"quoted": "a%2Bb%3Dc%26d"
},
{
"text": "a+b=c&d",
"safe": ":/",
"quoted": "a%2Bb%3Dc%26d"
},
{
"text": "~_.-",
"safe": "/",
"quoted": "~_.-"
},
{
"text": "~_.-",
"safe": "",
"quoted": "~_.-"
},
{
"text": "~_.-",
"safe": ":/",
"quoted": "~_.-"
},
{
"text": "100%",
"safe": "/",
"quoted": "100%25"
},
{
"text": "100%",
"safe": "",
"quoted": "100%25"
},
{
"text": "100%",
"safe": ":/",
"quoted": "100%25"
},
{
"text": "a/b:c d%#?+=&~\u00e9\"q",
"safe": "/",
"quoted": "a/b%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
},
{
"text": "a/b:c d%#?+=&~\u00e9\"q",
"safe": "",
"quoted": "a%2Fb%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
},
{
"text": "a/b:c d%#?+=&~\u00e9\"q",
"safe": ":/",
"quoted": "a/b:c%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
},
{
"text": "id-\u00e9-\ud83d\ude00",
"safe": "/",
"quoted": "id-%C3%A9-%F0%9F%98%80"
},
{
"text": "id-\u00e9-\ud83d\ude00",
"safe": "",
"quoted": "id-%C3%A9-%F0%9F%98%80"
},
{
"text": "id-\u00e9-\ud83d\ude00",
"safe": ":/",
"quoted": "id-%C3%A9-%F0%9F%98%80"
},
{
"text": "\u65e5\u672c",
"safe": "/",
"quoted": "%E6%97%A5%E6%9C%AC"
},
{
"text": "\u65e5\u672c",
"safe": "",
"quoted": "%E6%97%A5%E6%9C%AC"
},
{
"text": "\u65e5\u672c",
"safe": ":/",
"quoted": "%E6%97%A5%E6%9C%AC"
},
{
"text": "\u0000\u001f\u007f",
"safe": "/",
"quoted": "%00%1F%7F"
},
{
"text": "\u0000\u001f\u007f",
"safe": "",
"quoted": "%00%1F%7F"
},
{
"text": "\u0000\u001f\u007f",
"safe": ":/",
"quoted": "%00%1F%7F"
},
{
"text": "!*'();:@&=+$,/?#[]",
"safe": "/",
"quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C/%3F%23%5B%5D"
},
{
"text": "!*'();:@&=+$,/?#[]",
"safe": "",
"quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C%2F%3F%23%5B%5D"
},
{
"text": "!*'();:@&=+$,/?#[]",
"safe": ":/",
"quoted": "%21%2A%27%28%29%3B:%40%26%3D%2B%24%2C/%3F%23%5B%5D"
},
{
"text": "<>\\^`{|}",
"safe": "/",
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
},
{
"text": "<>\\^`{|}",
"safe": "",
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
},
{
"text": "<>\\^`{|}",
"safe": ":/",
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
}
],
"utc": [
{
"seconds": 0,
"micros": 0,
"fields": [
1970,
1,
1,
0,
0,
0,
0
]
},
{
"seconds": 1,
"micros": 999999,
"fields": [
1970,
1,
1,
0,
0,
1,
999999
]
},
{
"seconds": 59,
"micros": 0,
"fields": [
1970,
1,
1,
0,
0,
59,
0
]
},
{
"seconds": 86399,
"micros": 123456,
"fields": [
1970,
1,
1,
23,
59,
59,
123456
]
},
{
"seconds": 86400,
"micros": 0,
"fields": [
1970,
1,
2,
0,
0,
0,
0
]
},
{
"seconds": 951782400,
"micros": 0,
"fields": [
2000,
2,
29,
0,
0,
0,
0
]
},
{
"seconds": 951868800,
"micros": 0,
"fields": [
2000,
3,
1,
0,
0,
0,
0
]
},
{
"seconds": 1700000000,
"micros": 42,
"fields": [
2023,
11,
14,
22,
13,
20,
42
]
},
{
"seconds": 1757840707,
"micros": 42,
"fields": [
2025,
9,
14,
9,
5,
7,
42
]
},
{
"seconds": 4102444800,
"micros": 0,
"fields": [
2100,
1,
1,
0,
0,
0,
0
]
},
{
"seconds": 253402300799,
"micros": 999999,
"fields": [
9999,
12,
31,
23,
59,
59,
999999
]
}
]
}

View file

@ -0,0 +1,142 @@
"""Regenerate generated/pydantic.json: what pydantic's lax `bool` and `int` validation return.
uv run --no-project --with pydantic==2.13.4 python scripts/generate_pydantic_fixtures.py > generated/pydantic.json
Each row records `TypeAdapter(bool).validate_python(value)` and `TypeAdapter(int).validate_python(value)`
for one input written as JSON, or null where pydantic raises `ValidationError`. Inputs are what a
JSON-shaped setting can carry: None, booleans, integers, floats, strings, lists and dicts.
"""
import json
import sys
import pydantic
from pydantic import TypeAdapter, ValidationError
TEXTS = [
"",
" ",
"0",
"1",
"2",
"-1",
"+1",
"-0",
"00",
"007",
"1_000",
"1__000",
"_1",
"1_",
"1.",
".5",
"1.0",
"1.00",
"1.5",
"-1.0",
"1e3",
"1E3",
"0x10",
"0b1",
"0o7",
" 5 ",
"\t5\n",
"5 0",
"١٢",
"inf",
"nan",
"99999999999999999999",
"-99999999999999999999",
"true",
"True",
"TRUE",
" true ",
"false",
"False",
"t",
"f",
"T",
"y",
"n",
"yes",
"Yes",
"no",
"on",
"On",
"off",
"OFF",
"maybe",
"truthy",
"nul",
"+",
"-",
"+1.0",
"-0.0",
"1_0.0",
"1.0_0",
"1.0.0",
"1 ",
"\u00a05",
"\u20035",
]
VALUES = [
None,
True,
False,
0,
1,
2,
-1,
2**63 - 1,
2**63,
2**64,
-(2**63),
0.0,
-0.0,
1.0,
0.5,
1.5,
2.0,
-3.0,
1e15,
1e20,
1e300,
2.0**62,
2.0**63,
-(2.0**63),
9007199254740993.0,
float("inf"),
float("-inf"),
float("nan"),
[],
[1],
{},
{"a": 1},
*TEXTS,
]
def validated(adapter, value):
try:
return adapter.validate_python(value)
except ValidationError:
return None
def row(value):
as_int = validated(TypeAdapter(int), value)
return {
"input": json.dumps(value),
"bool": validated(TypeAdapter(bool), value),
"int": None if as_int is None else str(as_int),
}
if __name__ == "__main__":
json.dump(
{"pydantic": pydantic.VERSION, "rows": [row(value) for value in VALUES]},
sys.stdout,
indent=2,
ensure_ascii=True,
)

View file

@ -0,0 +1,66 @@
"""Regenerate generated/stdlib.json: `urllib.parse.quote` and UTC `datetime` fields in CPython.
python scripts/generate_stdlib_fixtures.py > generated/stdlib.json
`quote` rows record `urllib.parse.quote(text, safe=safe)`. `utc` rows record the fields of
`datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)`.
"""
import json
import sys
from datetime import datetime, timezone
from urllib.parse import quote
QUOTE_TEXTS = [
"",
"plain",
"a/b/c",
"a b",
"a+b=c&d",
"~_.-",
"100%",
"a/b:c d%#?+=&~é\"q",
"id-é-\U0001f600",
"日本",
"\x00\x1f\x7f",
"!*'();:@&=+$,/?#[]",
"<>\\^`{|}",
]
SAFES = ["/", "", ":/"]
TIMESTAMPS = [
(0, 0),
(1, 999999),
(59, 0),
(86399, 123456),
(86400, 0),
(951782400, 0),
(951868800, 0),
(1700000000, 42),
(1757840707, 42),
(4102444800, 0),
(253402300799, 999999),
]
def utc(seconds, micros):
moment = datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)
return {
"seconds": seconds,
"micros": micros,
"fields": [moment.year, moment.month, moment.day, moment.hour, moment.minute, moment.second, moment.microsecond],
}
if __name__ == "__main__":
json.dump(
{
"python": sys.version.split()[0],
"quote": [{"text": text, "safe": safe, "quoted": quote(text, safe=safe)} for text in QUOTE_TEXTS for safe in SAFES],
"utc": [utc(seconds, micros) for seconds, micros in TIMESTAMPS],
},
sys.stdout,
indent=2,
ensure_ascii=True,
)

View file

@ -0,0 +1,53 @@
//! The fields of `datetime.fromtimestamp(seconds, timezone.utc)`.
/// A UTC `datetime`'s fields, as Python's `datetime` exposes them.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct UtcDateTime {
pub year: i64,
pub month: u32,
pub day: u32,
pub hour: u32,
pub minute: u32,
pub second: u32,
pub microsecond: u32,
}
impl UtcDateTime {
/// `datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=microsecond)`.
pub fn from_timestamp(seconds: i64, microsecond: u32) -> Self {
let (year, month, day) = civil_from_days(seconds.div_euclid(86_400));
let of_day = seconds.rem_euclid(86_400);
Self {
year,
month,
day,
hour: (of_day / 3600) as u32,
minute: (of_day / 60 % 60) as u32,
second: (of_day % 60) as u32,
microsecond,
}
}
}
/// Days since 1970-01-01 to a proleptic Gregorian `(year, month, day)`.
///
/// Howard Hinnant's `civil_from_days`, <https://howardhinnant.github.io/date_algorithms.html#civil_from_days>:
/// shifts the epoch to 0000-03-01 so the leap day ends the year, then counts 400-year eras
/// of 146097 days.
fn civil_from_days(days: i64) -> (i64, u32, u32) {
let shifted = days + 719_468;
let era = shifted.div_euclid(146_097);
let day_of_era = shifted.rem_euclid(146_097);
let year_of_era =
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let month_index = (5 * day_of_year + 2) / 153;
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
let month = if month_index < 10 {
month_index + 3
} else {
month_index - 9
} as u32;
let year = year_of_era + era * 400 + i64::from(month <= 2);
(year, month, day)
}

View file

@ -12,6 +12,9 @@
//! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data |
//! | [`truthy`] | `bool(value)` |
//! | [`number`] | `int(text)`, `float(text)`, and `int(seconds)` as a `Duration` |
//! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation |
//! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` |
//! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields |
//!
//! [`Value`] is the closed data model these formats share. Live Python objects
//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the
@ -24,14 +27,17 @@
//! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists.
//! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit.
pub mod datetime;
mod error;
pub mod json;
pub mod literal;
pub mod number;
pub mod pickle;
pub mod pydantic;
pub mod repr;
pub mod reprlib;
pub mod truthy;
pub mod urllib;
mod value;
pub use error::Error;

View file

@ -0,0 +1,70 @@
//! Pydantic's lax `bool` and `int` validation: `TypeAdapter(bool)` and `TypeAdapter(int)`.
use num_bigint::BigInt;
use crate::Value;
const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0;
/// `TypeAdapter(bool).validate_python(value)`, or `None` where pydantic raises `ValidationError`.
pub fn lax_bool(value: &Value) -> Option<bool> {
match value {
Value::Bool(flag) => Some(*flag),
Value::Int(number) => match i8::try_from(number) {
Ok(0) => Some(false),
Ok(1) => Some(true),
_ => None,
},
Value::Float(number) if *number == 0.0 => Some(false),
Value::Float(number) if *number == 1.0 => Some(true),
Value::Str(text) => match text.to_ascii_lowercase().as_str() {
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
_ => None,
},
_ => None,
}
}
/// `TypeAdapter(int).validate_python(value)`, or `None` where pydantic raises `ValidationError`.
pub fn lax_int(value: &Value) -> Option<BigInt> {
match value {
Value::Bool(flag) => Some(BigInt::from(u8::from(*flag))),
Value::Int(number) => Some(number.clone()),
Value::Float(number)
if number.fract() == 0.0 && (-TWO_POW_63 < *number && *number < TWO_POW_63) =>
{
Some(BigInt::from(*number as i64))
}
Value::Str(text) => int_text(text.trim()),
_ => None,
}
}
fn int_text(text: &str) -> Option<BigInt> {
let whole = match text.split_once('.') {
Some((whole, fraction))
if !fraction.is_empty() && fraction.bytes().all(|byte| byte == b'0') =>
{
whole
}
Some(_) => return None,
None => text,
};
let (negative, digits) = match whole.strip_prefix('-') {
Some(rest) => (true, rest),
None => (false, whole.strip_prefix('+').unwrap_or(whole)),
};
let well_formed = !digits.is_empty()
&& !digits.starts_with('_')
&& !digits.ends_with('_')
&& !digits.contains("__")
&& digits
.bytes()
.all(|byte| byte.is_ascii_digit() || byte == b'_');
if !well_formed {
return None;
}
let magnitude = BigInt::parse_bytes(digits.replace('_', "").as_bytes(), 10)?;
Some(if negative { -magnitude } else { magnitude })
}

View file

@ -0,0 +1,18 @@
//! `urllib.parse.quote(text, safe=safe)` over the UTF-8 bytes of `text`.
/// `urllib.parse.quote(text, safe=safe)`: ASCII letters, digits and `_.-~` are never quoted,
/// the ASCII characters in `safe` are kept, and every other byte becomes `%XX`.
pub fn quote(text: &str, safe: &str) -> String {
let mut encoded = String::with_capacity(text.len());
for byte in text.bytes() {
if byte.is_ascii_alphanumeric()
|| matches!(byte, b'_' | b'.' | b'-' | b'~')
|| (byte.is_ascii() && safe.as_bytes().contains(&byte))
{
encoded.push(char::from(byte));
} else {
encoded.push_str(&format!("%{byte:02X}"));
}
}
encoded
}

View file

@ -0,0 +1,48 @@
//! Replays `generated/pydantic.json`, which pydantic wrote with `scripts/generate_pydantic_fixtures.py`.
use litellm_python_compat::{
json,
pydantic::{lax_bool, lax_int},
};
use serde::Deserialize;
#[derive(Deserialize)]
struct Fixtures {
rows: Vec<Row>,
}
#[derive(Deserialize)]
struct Row {
input: String,
bool: Option<bool>,
int: Option<String>,
}
fn fixtures() -> Fixtures {
serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/generated/pydantic.json"
)))
.expect("pydantic.json matches the fixture schema")
}
#[test]
fn lax_bool_and_int_match_pydantic() {
let mismatches: Vec<String> = fixtures()
.rows
.iter()
.flat_map(|row| {
let value = json::loads(&row.input).expect("inputs are json.dumps output");
let as_bool = lax_bool(&value);
let as_int = lax_int(&value).map(|number| number.to_string());
[
(as_bool != row.bool)
.then(|| format!("bool {}: {as_bool:?} != {:?}", row.input, row.bool)),
(as_int != row.int)
.then(|| format!("int {}: {as_int:?} != {:?}", row.input, row.int)),
]
})
.flatten()
.collect();
assert!(mismatches.is_empty(), "{mismatches:#?}");
}

View file

@ -0,0 +1,62 @@
//! Replays `generated/stdlib.json`, which CPython wrote with `scripts/generate_stdlib_fixtures.py`.
use litellm_python_compat::{datetime::UtcDateTime, urllib::quote};
use serde::Deserialize;
#[derive(Deserialize)]
struct Fixtures {
quote: Vec<Quote>,
utc: Vec<Utc>,
}
#[derive(Deserialize)]
struct Quote {
text: String,
safe: String,
quoted: String,
}
#[derive(Deserialize)]
struct Utc {
seconds: i64,
micros: u32,
fields: [i64; 7],
}
fn fixtures() -> Fixtures {
serde_json::from_str(include_str!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/generated/stdlib.json"
)))
.expect("stdlib.json matches the fixture schema")
}
#[test]
fn quote_matches_urllib() {
for row in fixtures().quote {
assert_eq!(
quote(&row.text, &row.safe),
row.quoted,
"{:?} safe={:?}",
row.text,
row.safe
);
}
}
#[test]
fn utc_fields_match_datetime() {
for row in fixtures().utc {
let moment = UtcDateTime::from_timestamp(row.seconds, row.micros);
let fields = [
moment.year,
i64::from(moment.month),
i64::from(moment.day),
i64::from(moment.hour),
i64::from(moment.minute),
i64::from(moment.second),
i64::from(moment.microsecond),
];
assert_eq!(fields, row.fields, "{} {}", row.seconds, row.micros);
}
}