mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
refactor(s3-log): move python-compatible helpers into python-compat (#51895)
This commit is contained in:
parent
09ce4f50ed
commit
d63224db15
17 changed files with 1345 additions and 329 deletions
|
|
@ -1,6 +1,7 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_callbacks::Moment;
|
||||
use litellm_python_compat::datetime::UtcDateTime;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::settings::{Granularity, Settings};
|
||||
|
|
@ -58,16 +59,18 @@ impl Stamp {
|
|||
pub fn utc(time: SystemTime) -> Self {
|
||||
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
|
||||
let seconds = elapsed.as_secs();
|
||||
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
|
||||
let of_day = seconds % 86_400;
|
||||
let moment = UtcDateTime::from_timestamp(
|
||||
i64::try_from(seconds).unwrap_or(i64::MAX),
|
||||
elapsed.subsec_micros(),
|
||||
);
|
||||
Self {
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
hour: (of_day / 3600) as u32,
|
||||
minute: (of_day / 60 % 60) as u32,
|
||||
second: (of_day % 60) as u32,
|
||||
micros: elapsed.subsec_micros(),
|
||||
year: moment.year,
|
||||
month: moment.month,
|
||||
day: moment.day,
|
||||
hour: moment.hour,
|
||||
minute: moment.minute,
|
||||
second: moment.second,
|
||||
micros: moment.microsecond,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -84,24 +87,6 @@ impl Stamp {
|
|||
}
|
||||
}
|
||||
|
||||
fn civil_from_days(days: i64) -> (i64, u32, u32) {
|
||||
let shifted = days + 719_468;
|
||||
let era = shifted.div_euclid(146_097);
|
||||
let day_of_era = shifted.rem_euclid(146_097);
|
||||
let year_of_era =
|
||||
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
|
||||
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
|
||||
let month_index = (5 * day_of_year + 2) / 153;
|
||||
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
|
||||
let month = if month_index < 10 {
|
||||
month_index + 3
|
||||
} else {
|
||||
month_index - 9
|
||||
} as u32;
|
||||
let year = year_of_era + era * 400 + i64::from(month <= 2);
|
||||
(year, month, day)
|
||||
}
|
||||
|
||||
pub fn logging_id(start: &Stamp, id: &str) -> String {
|
||||
format!("time-{}_{id}", start.clock_micros())
|
||||
}
|
||||
|
|
@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str {
|
|||
}
|
||||
|
||||
pub fn quote(key: &str) -> String {
|
||||
let mut encoded = String::with_capacity(key.len());
|
||||
for byte in key.bytes() {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
|
||||
encoded.push(byte as char);
|
||||
} else {
|
||||
encoded.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
litellm_python_compat::urllib::quote(key, "/")
|
||||
}
|
||||
|
||||
pub fn dns_suffix(region: Option<&str>) -> &'static str {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
mod key;
|
||||
mod limiter;
|
||||
mod pyjson;
|
||||
mod settings;
|
||||
mod sink;
|
||||
mod upload;
|
||||
|
|
|
|||
|
|
@ -1,214 +0,0 @@
|
|||
use serde_json::Value;
|
||||
|
||||
const HEX: &[u8; 16] = b"0123456789abcdef";
|
||||
|
||||
pub fn write_str(out: &mut Vec<u8>, text: &str) {
|
||||
out.push(b'"');
|
||||
let mut units = [0u16; 2];
|
||||
for character in text.chars() {
|
||||
match character {
|
||||
'"' => out.extend_from_slice(b"\\\""),
|
||||
'\\' => out.extend_from_slice(b"\\\\"),
|
||||
'\n' => out.extend_from_slice(b"\\n"),
|
||||
'\r' => out.extend_from_slice(b"\\r"),
|
||||
'\t' => out.extend_from_slice(b"\\t"),
|
||||
'\u{8}' => out.extend_from_slice(b"\\b"),
|
||||
'\u{c}' => out.extend_from_slice(b"\\f"),
|
||||
' '..='~' => out.push(character as u8),
|
||||
_ => {
|
||||
for unit in character.encode_utf16(&mut units) {
|
||||
write_unit(out, *unit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
out.push(b'"');
|
||||
}
|
||||
|
||||
fn write_unit(out: &mut Vec<u8>, unit: u16) {
|
||||
out.extend_from_slice(b"\\u");
|
||||
for shift in [12, 8, 4, 0] {
|
||||
out.push(HEX[usize::from((unit >> shift) & 0xf)]);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn float_repr(value: f64) -> String {
|
||||
if value.is_nan() {
|
||||
return "NaN".to_owned();
|
||||
}
|
||||
if value.is_infinite() {
|
||||
return if value > 0.0 { "Infinity" } else { "-Infinity" }.to_owned();
|
||||
}
|
||||
let shortest = format!("{value:e}");
|
||||
let length = shortest
|
||||
.split('e')
|
||||
.next()
|
||||
.unwrap_or_default()
|
||||
.chars()
|
||||
.filter(char::is_ascii_digit)
|
||||
.count();
|
||||
let scientific = format!("{value:.precision$e}", precision = length.saturating_sub(1));
|
||||
let (negative, scientific) = match scientific.strip_prefix('-') {
|
||||
Some(rest) => (true, rest),
|
||||
None => (false, scientific.as_str()),
|
||||
};
|
||||
let (mantissa, exponent) = scientific.split_once('e').unwrap_or((scientific, "0"));
|
||||
let exponent: i32 = exponent.parse().unwrap_or(0);
|
||||
let digits: String = mantissa.chars().filter(char::is_ascii_digit).collect();
|
||||
let mut text = String::with_capacity(digits.len() + 8);
|
||||
if negative {
|
||||
text.push('-');
|
||||
}
|
||||
let point = exponent + 1;
|
||||
if (-3..=16).contains(&point) {
|
||||
let count = digits.len() as i32;
|
||||
if point <= 0 {
|
||||
text.push_str("0.");
|
||||
text.extend(std::iter::repeat_n('0', (-point) as usize));
|
||||
text.push_str(&digits);
|
||||
} else if point >= count {
|
||||
text.push_str(&digits);
|
||||
text.extend(std::iter::repeat_n('0', (point - count) as usize));
|
||||
text.push_str(".0");
|
||||
} else {
|
||||
let (whole, fraction) = digits.split_at(point as usize);
|
||||
text.push_str(whole);
|
||||
text.push('.');
|
||||
text.push_str(fraction);
|
||||
}
|
||||
return text;
|
||||
}
|
||||
let (first, rest) = digits.split_at(1);
|
||||
text.push_str(first);
|
||||
if !rest.is_empty() {
|
||||
text.push('.');
|
||||
text.push_str(rest);
|
||||
}
|
||||
text.push('e');
|
||||
text.push(if exponent < 0 { '-' } else { '+' });
|
||||
text.push_str(&format!("{:02}", exponent.unsigned_abs()));
|
||||
text
|
||||
}
|
||||
|
||||
pub fn write_value(out: &mut Vec<u8>, value: &Value) {
|
||||
match value {
|
||||
Value::Null => out.extend_from_slice(b"null"),
|
||||
Value::Bool(true) => out.extend_from_slice(b"true"),
|
||||
Value::Bool(false) => out.extend_from_slice(b"false"),
|
||||
Value::Number(number) => match (number.as_i64(), number.as_u64(), number.as_f64()) {
|
||||
(Some(integer), _, _) => out.extend_from_slice(integer.to_string().as_bytes()),
|
||||
(None, Some(integer), _) => out.extend_from_slice(integer.to_string().as_bytes()),
|
||||
(None, None, Some(float)) => out.extend_from_slice(float_repr(float).as_bytes()),
|
||||
(None, None, None) => out.extend_from_slice(number.to_string().as_bytes()),
|
||||
},
|
||||
Value::String(text) => write_str(out, text),
|
||||
Value::Array(items) => {
|
||||
out.push(b'[');
|
||||
for (index, item) in items.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.extend_from_slice(b", ");
|
||||
}
|
||||
write_value(out, item);
|
||||
}
|
||||
out.push(b']');
|
||||
}
|
||||
Value::Object(entries) => {
|
||||
out.push(b'{');
|
||||
for (index, (key, item)) in entries.iter().enumerate() {
|
||||
if index > 0 {
|
||||
out.extend_from_slice(b", ");
|
||||
}
|
||||
write_str(out, key);
|
||||
out.extend_from_slice(b": ");
|
||||
write_value(out, item);
|
||||
}
|
||||
out.push(b'}');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn dumps(value: &Value) -> String {
|
||||
let mut out = Vec::new();
|
||||
write_value(&mut out, value);
|
||||
String::from_utf8(out).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub fn py_str(value: &Value) -> String {
|
||||
match value {
|
||||
Value::Null => "None".to_owned(),
|
||||
Value::Bool(true) => "True".to_owned(),
|
||||
Value::Bool(false) => "False".to_owned(),
|
||||
Value::String(text) => text.clone(),
|
||||
Value::Number(number) => match number.as_f64() {
|
||||
Some(float) if number.is_f64() => float_repr(float),
|
||||
_ => number.to_string(),
|
||||
},
|
||||
Value::Array(_) | Value::Object(_) => dumps(value),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn truthy(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::Bool(flag) => *flag,
|
||||
Value::Number(number) => number.as_f64().is_some_and(|float| float != 0.0),
|
||||
Value::String(text) => !text.is_empty(),
|
||||
Value::Array(items) => !items.is_empty(),
|
||||
Value::Object(entries) => !entries.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case(0.0, "0.0")]
|
||||
#[case(-0.0, "-0.0")]
|
||||
#[case(1.0, "1.0")]
|
||||
#[case(0.1, "0.1")]
|
||||
#[case(1.5e-5, "1.5e-05")]
|
||||
#[case(1e-4, "0.0001")]
|
||||
#[case(1e-5, "1e-05")]
|
||||
#[case(1e15, "1000000000000000.0")]
|
||||
#[case(1e16, "1e+16")]
|
||||
#[case(1.2345678901234568e17, "1.2345678901234568e+17")]
|
||||
#[case(123.456, "123.456")]
|
||||
#[case(1e100, "1e+100")]
|
||||
#[case(-2.5e-300, "-2.5e-300")]
|
||||
#[case(f64::NAN, "NaN")]
|
||||
#[case(f64::INFINITY, "Infinity")]
|
||||
#[case(f64::NEG_INFINITY, "-Infinity")]
|
||||
#[case(0.30000000000000004, "0.30000000000000004")]
|
||||
#[case(0.07872390747070312, "0.07872390747070312")]
|
||||
#[case(1794700894.4414062, "1794700894.4414062")]
|
||||
fn floats_print_as_python_repr_prints_them(#[case] value: f64, #[case] expected: &str) {
|
||||
assert_eq!(float_repr(value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case("plain", "\"plain\"")]
|
||||
#[case("quote\" back\\", "\"quote\\\" back\\\\\"")]
|
||||
#[case("\n\r\t\u{8}\u{c}\u{1}\u{7f}", "\"\\n\\r\\t\\b\\f\\u0001\\u007f\"")]
|
||||
#[case("café", "\"caf\\u00e9\"")]
|
||||
#[case("😀", "\"\\ud83d\\ude00\"")]
|
||||
fn strings_are_escaped_as_ensure_ascii_escapes_them(
|
||||
#[case] text: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let mut out = Vec::new();
|
||||
write_str(&mut out, text);
|
||||
assert_eq!(String::from_utf8(out).unwrap(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn containers_use_the_default_separators_and_keep_key_order() {
|
||||
assert_eq!(
|
||||
dumps(&json!({"b": [1, 2.5, null, true], "a": {"x": "y"}})),
|
||||
"{\"b\": [1, 2.5, null, true], \"a\": {\"x\": \"y\"}}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -5,28 +5,28 @@ use litellm_secret_value::SecretValue;
|
|||
use serde::Deserialize;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::pyjson::{py_str, truthy};
|
||||
use litellm_python_compat::{BigInt, json::from_json, pydantic, repr};
|
||||
|
||||
fn python(value: &Value) -> litellm_python_compat::Value {
|
||||
from_json(value.clone())
|
||||
}
|
||||
|
||||
fn py_repr(value: &Value) -> String {
|
||||
litellm_python_compat::repr::repr(&litellm_python_compat::json::from_json(value.clone()))
|
||||
repr::repr(&python(value))
|
||||
}
|
||||
|
||||
pub fn py_str(value: &Value) -> String {
|
||||
repr::to_str(&python(value))
|
||||
}
|
||||
|
||||
pub fn truthy(value: &Value) -> bool {
|
||||
litellm_python_compat::truthy::truthy(&python(value))
|
||||
}
|
||||
|
||||
fn py_optional(value: Option<u64>) -> String {
|
||||
value.map_or_else(|| "None".to_owned(), |value| value.to_string())
|
||||
}
|
||||
|
||||
fn py_type_name(value: &Value) -> &'static str {
|
||||
match value {
|
||||
Value::Null => "NoneType",
|
||||
Value::Bool(_) => "bool",
|
||||
Value::Number(number) if number.is_f64() => "float",
|
||||
Value::Number(_) => "int",
|
||||
Value::String(_) => "str",
|
||||
Value::Array(_) => "list",
|
||||
Value::Object(_) => "dict",
|
||||
}
|
||||
}
|
||||
|
||||
pub const PARTITION_GRANULARITY_ENV_VAR: &str = "S3_PARTITION_GRANULARITY";
|
||||
pub const LOG_PROMPTS_ONLY_ENV_VAR: &str = "S3_LOG_PROMPTS_ONLY";
|
||||
const DEFAULT_MAX_QUEUE_SIZE: usize = 50_000;
|
||||
|
|
@ -122,71 +122,17 @@ fn is_blank(value: &Value) -> bool {
|
|||
}
|
||||
|
||||
pub fn lax_bool(value: &Value) -> Option<bool> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(*flag),
|
||||
Value::Number(number) => number
|
||||
.as_f64()
|
||||
.and_then(|float| {
|
||||
[(0.0, false), (1.0, true)]
|
||||
.into_iter()
|
||||
.find(|(bound, _)| *bound == float)
|
||||
})
|
||||
.map(|(_, flag)| flag),
|
||||
Value::String(text) => match text.trim().to_ascii_lowercase().as_str() {
|
||||
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
|
||||
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
pydantic::lax_bool(&python(value))
|
||||
}
|
||||
|
||||
pub fn lax_int(value: &Value) -> Option<i128> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(i128::from(*flag)),
|
||||
Value::Number(number) => {
|
||||
if let Some(integer) = number.as_i64() {
|
||||
return Some(i128::from(integer));
|
||||
}
|
||||
if let Some(integer) = number.as_u64() {
|
||||
return Some(i128::from(integer));
|
||||
}
|
||||
let float = number.as_f64()?;
|
||||
(float.is_finite() && float.fract() == 0.0 && float.abs() <= i64::MAX as f64)
|
||||
.then_some(float as i128)
|
||||
}
|
||||
Value::String(text) => int_text(text.trim()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn int_text(text: &str) -> Option<i128> {
|
||||
let (whole, fraction) = text.split_once('.').unwrap_or((text, ""));
|
||||
if !fraction.chars().all(|character| character == '0') {
|
||||
return None;
|
||||
}
|
||||
if text.contains('.') && fraction.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let (sign, digits) = match whole.strip_prefix('-') {
|
||||
Some(rest) => (-1, rest),
|
||||
None => (1, whole.strip_prefix('+').unwrap_or(whole)),
|
||||
};
|
||||
let well_formed = !digits.is_empty()
|
||||
&& !digits.starts_with('_')
|
||||
&& !digits.ends_with('_')
|
||||
&& !digits.contains("__")
|
||||
&& digits
|
||||
.chars()
|
||||
.all(|character| character.is_ascii_digit() || character == '_');
|
||||
if !well_formed {
|
||||
return None;
|
||||
}
|
||||
digits
|
||||
.replace('_', "")
|
||||
.parse::<i128>()
|
||||
.ok()
|
||||
.map(|number| sign * number)
|
||||
pydantic::lax_int(&python(value)).map(|number| {
|
||||
i128::try_from(&number).unwrap_or(if number < BigInt::from(0) {
|
||||
i128::MIN
|
||||
} else {
|
||||
i128::MAX
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn strip_text(value: &Value) -> Value {
|
||||
|
|
@ -291,7 +237,7 @@ fn sse_value(name: &str, value: Option<&Value>) -> Option<String> {
|
|||
Some(other) => {
|
||||
tracing::warn!(
|
||||
"s3 logging: ignoring {name} because it has invalid type {}; expected a string",
|
||||
py_type_name(other)
|
||||
python(other).type_name()
|
||||
);
|
||||
None
|
||||
}
|
||||
|
|
@ -556,6 +502,16 @@ mod tests {
|
|||
assert_eq!(lax_bool(&strip_text(&value)), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case(json!(["us-east-1"]), "['us-east-1']")]
|
||||
#[case(json!({"a": true}), "{'a': True}")]
|
||||
#[case(json!(null), "None")]
|
||||
#[case(json!(1.5), "1.5")]
|
||||
#[case(json!("us-east-1"), "us-east-1")]
|
||||
fn values_print_as_python_str_prints_them(#[case] value: Value, #[case] expected: &str) {
|
||||
assert_eq!(py_str(&value), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
fn defaults_match_the_python_constructor() {
|
||||
let settings = settings(json!({}));
|
||||
|
|
|
|||
|
|
@ -384,7 +384,7 @@ impl Uploader {
|
|||
.settings
|
||||
.region_name
|
||||
.as_ref()
|
||||
.map(crate::pyjson::py_str)
|
||||
.map(crate::settings::py_str)
|
||||
else {
|
||||
tracing::error!("Error downloading from S3: no s3_region_name to sign with");
|
||||
return None;
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@
|
|||
- `cache-disk`: diskcache's pickled values (`pickle`), falsy-is-miss (`truthy`)
|
||||
- Cache-key derivation: sha256 over `str(value)` must match Python byte for byte (`repr::to_str`)
|
||||
- Byte-identical writes where Python compares raw values (`json::dumps`, `repr`)
|
||||
- `callbacks-s3-log`: settings parsed as pydantic parses them (`pydantic`), object keys quoted as `urllib.parse.quote` quotes them (`urllib`), UTC key dates (`datetime`)
|
||||
- Relation to [`py_literal`](https://docs.rs/py_literal/latest/py_literal/): replaced, do not reintroduce
|
||||
- Its pest grammar backtracks: parse time doubles per nested `[`/`{` (105 ms at depth 16). Ours is linear (19 µs at depth 128)
|
||||
- Its formatter is not `repr` (`2e-1`, always single quotes, escapes non-ASCII). It also lost `-0.0`, `(1+2j)`, `set()`
|
||||
|
|
@ -15,6 +16,7 @@
|
|||
- Accepted limits: ints beyond i64, `tuple`/`set`/`frozenset` decode as lists, class references (`GLOBAL`/`REDUCE`) fail. Writes protocol 3
|
||||
- Every behavior is pinned by CPython output, not by reasoning. Everything under `generated/` is script output, never hand-edited
|
||||
- Regenerate `generated/values.json` with `scripts/generate_fixtures.py`. Add a corpus row before changing behavior
|
||||
- Regenerate `generated/stdlib.json` (`urllib.parse.quote`, UTC `datetime` fields) with `scripts/generate_stdlib_fixtures.py`, and `generated/pydantic.json` (lax `bool`/`int`) with `scripts/generate_pydantic_fixtures.py` at the pydantic version in `uv.lock`
|
||||
- Divergences go in `KNOWN` in `tests/fixtures.rs` with a reason. An entry that starts matching fails until deleted
|
||||
- Regenerate `generated/nonprintable.rs` with `scripts/generate_nonprintable.py` when the target Python's Unicode version changes
|
||||
- `scripts/verify_rust_pickles.py` checks CPython reads Rust pickles, with class resolution disabled. CI does not run Python
|
||||
|
|
|
|||
1
litellm/rust/crates/python-compat/generated/.gitattributes
generated
vendored
Normal file
1
litellm/rust/crates/python-compat/generated/.gitattributes
generated
vendored
Normal file
|
|
@ -0,0 +1 @@
|
|||
* linguist-generated=true -diff
|
||||
485
litellm/rust/crates/python-compat/generated/pydantic.json
generated
Normal file
485
litellm/rust/crates/python-compat/generated/pydantic.json
generated
Normal file
|
|
@ -0,0 +1,485 @@
|
|||
{
|
||||
"pydantic": "2.13.4",
|
||||
"rows": [
|
||||
{
|
||||
"input": "null",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "true",
|
||||
"bool": true,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "false",
|
||||
"bool": false,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "0",
|
||||
"bool": false,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "1",
|
||||
"bool": true,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "2",
|
||||
"bool": null,
|
||||
"int": "2"
|
||||
},
|
||||
{
|
||||
"input": "-1",
|
||||
"bool": null,
|
||||
"int": "-1"
|
||||
},
|
||||
{
|
||||
"input": "9223372036854775807",
|
||||
"bool": null,
|
||||
"int": "9223372036854775807"
|
||||
},
|
||||
{
|
||||
"input": "9223372036854775808",
|
||||
"bool": null,
|
||||
"int": "9223372036854775808"
|
||||
},
|
||||
{
|
||||
"input": "18446744073709551616",
|
||||
"bool": null,
|
||||
"int": "18446744073709551616"
|
||||
},
|
||||
{
|
||||
"input": "-9223372036854775808",
|
||||
"bool": null,
|
||||
"int": "-9223372036854775808"
|
||||
},
|
||||
{
|
||||
"input": "0.0",
|
||||
"bool": false,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "-0.0",
|
||||
"bool": false,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "1.0",
|
||||
"bool": true,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "0.5",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "1.5",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "2.0",
|
||||
"bool": null,
|
||||
"int": "2"
|
||||
},
|
||||
{
|
||||
"input": "-3.0",
|
||||
"bool": null,
|
||||
"int": "-3"
|
||||
},
|
||||
{
|
||||
"input": "1000000000000000.0",
|
||||
"bool": null,
|
||||
"int": "1000000000000000"
|
||||
},
|
||||
{
|
||||
"input": "1e+20",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "1e+300",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "4.611686018427388e+18",
|
||||
"bool": null,
|
||||
"int": "4611686018427387904"
|
||||
},
|
||||
{
|
||||
"input": "9.223372036854776e+18",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "-9.223372036854776e+18",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "9007199254740992.0",
|
||||
"bool": null,
|
||||
"int": "9007199254740992"
|
||||
},
|
||||
{
|
||||
"input": "Infinity",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "-Infinity",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "NaN",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "[]",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "[1]",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "{}",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "{\"a\": 1}",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\" \"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"0\"",
|
||||
"bool": false,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "\"1\"",
|
||||
"bool": true,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"2\"",
|
||||
"bool": null,
|
||||
"int": "2"
|
||||
},
|
||||
{
|
||||
"input": "\"-1\"",
|
||||
"bool": null,
|
||||
"int": "-1"
|
||||
},
|
||||
{
|
||||
"input": "\"+1\"",
|
||||
"bool": null,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"-0\"",
|
||||
"bool": null,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "\"00\"",
|
||||
"bool": null,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "\"007\"",
|
||||
"bool": null,
|
||||
"int": "7"
|
||||
},
|
||||
{
|
||||
"input": "\"1_000\"",
|
||||
"bool": null,
|
||||
"int": "1000"
|
||||
},
|
||||
{
|
||||
"input": "\"1__000\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"_1\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1_\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1.\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\".5\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1.0\"",
|
||||
"bool": null,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"1.00\"",
|
||||
"bool": null,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"1.5\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"-1.0\"",
|
||||
"bool": null,
|
||||
"int": "-1"
|
||||
},
|
||||
{
|
||||
"input": "\"1e3\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1E3\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"0x10\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"0b1\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"0o7\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\" 5 \"",
|
||||
"bool": null,
|
||||
"int": "5"
|
||||
},
|
||||
{
|
||||
"input": "\"\\t5\\n\"",
|
||||
"bool": null,
|
||||
"int": "5"
|
||||
},
|
||||
{
|
||||
"input": "\"5 0\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"\\u0661\\u0662\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"inf\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"nan\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"99999999999999999999\"",
|
||||
"bool": null,
|
||||
"int": "99999999999999999999"
|
||||
},
|
||||
{
|
||||
"input": "\"-99999999999999999999\"",
|
||||
"bool": null,
|
||||
"int": "-99999999999999999999"
|
||||
},
|
||||
{
|
||||
"input": "\"true\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"True\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"TRUE\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\" true \"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"false\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"False\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"t\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"f\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"T\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"y\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"n\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"yes\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"Yes\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"no\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"on\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"On\"",
|
||||
"bool": true,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"off\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"OFF\"",
|
||||
"bool": false,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"maybe\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"truthy\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"nul\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"+\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"-\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"+1.0\"",
|
||||
"bool": null,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"-0.0\"",
|
||||
"bool": null,
|
||||
"int": "0"
|
||||
},
|
||||
{
|
||||
"input": "\"1_0.0\"",
|
||||
"bool": null,
|
||||
"int": "10"
|
||||
},
|
||||
{
|
||||
"input": "\"1.0_0\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1.0.0\"",
|
||||
"bool": null,
|
||||
"int": null
|
||||
},
|
||||
{
|
||||
"input": "\"1 \"",
|
||||
"bool": null,
|
||||
"int": "1"
|
||||
},
|
||||
{
|
||||
"input": "\"\\u00a05\"",
|
||||
"bool": null,
|
||||
"int": "5"
|
||||
},
|
||||
{
|
||||
"input": "\"\\u20035\"",
|
||||
"bool": null,
|
||||
"int": "5"
|
||||
}
|
||||
]
|
||||
}
|
||||
345
litellm/rust/crates/python-compat/generated/stdlib.json
generated
Normal file
345
litellm/rust/crates/python-compat/generated/stdlib.json
generated
Normal file
|
|
@ -0,0 +1,345 @@
|
|||
{
|
||||
"python": "3.14.6",
|
||||
"quote": [
|
||||
{
|
||||
"text": "",
|
||||
"safe": "/",
|
||||
"quoted": ""
|
||||
},
|
||||
{
|
||||
"text": "",
|
||||
"safe": "",
|
||||
"quoted": ""
|
||||
},
|
||||
{
|
||||
"text": "",
|
||||
"safe": ":/",
|
||||
"quoted": ""
|
||||
},
|
||||
{
|
||||
"text": "plain",
|
||||
"safe": "/",
|
||||
"quoted": "plain"
|
||||
},
|
||||
{
|
||||
"text": "plain",
|
||||
"safe": "",
|
||||
"quoted": "plain"
|
||||
},
|
||||
{
|
||||
"text": "plain",
|
||||
"safe": ":/",
|
||||
"quoted": "plain"
|
||||
},
|
||||
{
|
||||
"text": "a/b/c",
|
||||
"safe": "/",
|
||||
"quoted": "a/b/c"
|
||||
},
|
||||
{
|
||||
"text": "a/b/c",
|
||||
"safe": "",
|
||||
"quoted": "a%2Fb%2Fc"
|
||||
},
|
||||
{
|
||||
"text": "a/b/c",
|
||||
"safe": ":/",
|
||||
"quoted": "a/b/c"
|
||||
},
|
||||
{
|
||||
"text": "a b",
|
||||
"safe": "/",
|
||||
"quoted": "a%20b"
|
||||
},
|
||||
{
|
||||
"text": "a b",
|
||||
"safe": "",
|
||||
"quoted": "a%20b"
|
||||
},
|
||||
{
|
||||
"text": "a b",
|
||||
"safe": ":/",
|
||||
"quoted": "a%20b"
|
||||
},
|
||||
{
|
||||
"text": "a+b=c&d",
|
||||
"safe": "/",
|
||||
"quoted": "a%2Bb%3Dc%26d"
|
||||
},
|
||||
{
|
||||
"text": "a+b=c&d",
|
||||
"safe": "",
|
||||
"quoted": "a%2Bb%3Dc%26d"
|
||||
},
|
||||
{
|
||||
"text": "a+b=c&d",
|
||||
"safe": ":/",
|
||||
"quoted": "a%2Bb%3Dc%26d"
|
||||
},
|
||||
{
|
||||
"text": "~_.-",
|
||||
"safe": "/",
|
||||
"quoted": "~_.-"
|
||||
},
|
||||
{
|
||||
"text": "~_.-",
|
||||
"safe": "",
|
||||
"quoted": "~_.-"
|
||||
},
|
||||
{
|
||||
"text": "~_.-",
|
||||
"safe": ":/",
|
||||
"quoted": "~_.-"
|
||||
},
|
||||
{
|
||||
"text": "100%",
|
||||
"safe": "/",
|
||||
"quoted": "100%25"
|
||||
},
|
||||
{
|
||||
"text": "100%",
|
||||
"safe": "",
|
||||
"quoted": "100%25"
|
||||
},
|
||||
{
|
||||
"text": "100%",
|
||||
"safe": ":/",
|
||||
"quoted": "100%25"
|
||||
},
|
||||
{
|
||||
"text": "a/b:c d%#?+=&~\u00e9\"q",
|
||||
"safe": "/",
|
||||
"quoted": "a/b%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
|
||||
},
|
||||
{
|
||||
"text": "a/b:c d%#?+=&~\u00e9\"q",
|
||||
"safe": "",
|
||||
"quoted": "a%2Fb%3Ac%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
|
||||
},
|
||||
{
|
||||
"text": "a/b:c d%#?+=&~\u00e9\"q",
|
||||
"safe": ":/",
|
||||
"quoted": "a/b:c%20d%25%23%3F%2B%3D%26~%C3%A9%22q"
|
||||
},
|
||||
{
|
||||
"text": "id-\u00e9-\ud83d\ude00",
|
||||
"safe": "/",
|
||||
"quoted": "id-%C3%A9-%F0%9F%98%80"
|
||||
},
|
||||
{
|
||||
"text": "id-\u00e9-\ud83d\ude00",
|
||||
"safe": "",
|
||||
"quoted": "id-%C3%A9-%F0%9F%98%80"
|
||||
},
|
||||
{
|
||||
"text": "id-\u00e9-\ud83d\ude00",
|
||||
"safe": ":/",
|
||||
"quoted": "id-%C3%A9-%F0%9F%98%80"
|
||||
},
|
||||
{
|
||||
"text": "\u65e5\u672c",
|
||||
"safe": "/",
|
||||
"quoted": "%E6%97%A5%E6%9C%AC"
|
||||
},
|
||||
{
|
||||
"text": "\u65e5\u672c",
|
||||
"safe": "",
|
||||
"quoted": "%E6%97%A5%E6%9C%AC"
|
||||
},
|
||||
{
|
||||
"text": "\u65e5\u672c",
|
||||
"safe": ":/",
|
||||
"quoted": "%E6%97%A5%E6%9C%AC"
|
||||
},
|
||||
{
|
||||
"text": "\u0000\u001f\u007f",
|
||||
"safe": "/",
|
||||
"quoted": "%00%1F%7F"
|
||||
},
|
||||
{
|
||||
"text": "\u0000\u001f\u007f",
|
||||
"safe": "",
|
||||
"quoted": "%00%1F%7F"
|
||||
},
|
||||
{
|
||||
"text": "\u0000\u001f\u007f",
|
||||
"safe": ":/",
|
||||
"quoted": "%00%1F%7F"
|
||||
},
|
||||
{
|
||||
"text": "!*'();:@&=+$,/?#[]",
|
||||
"safe": "/",
|
||||
"quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C/%3F%23%5B%5D"
|
||||
},
|
||||
{
|
||||
"text": "!*'();:@&=+$,/?#[]",
|
||||
"safe": "",
|
||||
"quoted": "%21%2A%27%28%29%3B%3A%40%26%3D%2B%24%2C%2F%3F%23%5B%5D"
|
||||
},
|
||||
{
|
||||
"text": "!*'();:@&=+$,/?#[]",
|
||||
"safe": ":/",
|
||||
"quoted": "%21%2A%27%28%29%3B:%40%26%3D%2B%24%2C/%3F%23%5B%5D"
|
||||
},
|
||||
{
|
||||
"text": "<>\\^`{|}",
|
||||
"safe": "/",
|
||||
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
|
||||
},
|
||||
{
|
||||
"text": "<>\\^`{|}",
|
||||
"safe": "",
|
||||
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
|
||||
},
|
||||
{
|
||||
"text": "<>\\^`{|}",
|
||||
"safe": ":/",
|
||||
"quoted": "%3C%3E%5C%5E%60%7B%7C%7D"
|
||||
}
|
||||
],
|
||||
"utc": [
|
||||
{
|
||||
"seconds": 0,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
1970,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 1,
|
||||
"micros": 999999,
|
||||
"fields": [
|
||||
1970,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
999999
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 59,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
1970,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
59,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 86399,
|
||||
"micros": 123456,
|
||||
"fields": [
|
||||
1970,
|
||||
1,
|
||||
1,
|
||||
23,
|
||||
59,
|
||||
59,
|
||||
123456
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 86400,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
1970,
|
||||
1,
|
||||
2,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 951782400,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
2000,
|
||||
2,
|
||||
29,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 951868800,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
2000,
|
||||
3,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 1700000000,
|
||||
"micros": 42,
|
||||
"fields": [
|
||||
2023,
|
||||
11,
|
||||
14,
|
||||
22,
|
||||
13,
|
||||
20,
|
||||
42
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 1757840707,
|
||||
"micros": 42,
|
||||
"fields": [
|
||||
2025,
|
||||
9,
|
||||
14,
|
||||
9,
|
||||
5,
|
||||
7,
|
||||
42
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 4102444800,
|
||||
"micros": 0,
|
||||
"fields": [
|
||||
2100,
|
||||
1,
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0
|
||||
]
|
||||
},
|
||||
{
|
||||
"seconds": 253402300799,
|
||||
"micros": 999999,
|
||||
"fields": [
|
||||
9999,
|
||||
12,
|
||||
31,
|
||||
23,
|
||||
59,
|
||||
59,
|
||||
999999
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,142 @@
|
|||
"""Regenerate generated/pydantic.json: what pydantic's lax `bool` and `int` validation return.
|
||||
|
||||
uv run --no-project --with pydantic==2.13.4 python scripts/generate_pydantic_fixtures.py > generated/pydantic.json
|
||||
|
||||
Each row records `TypeAdapter(bool).validate_python(value)` and `TypeAdapter(int).validate_python(value)`
|
||||
for one input written as JSON, or null where pydantic raises `ValidationError`. Inputs are what a
|
||||
JSON-shaped setting can carry: None, booleans, integers, floats, strings, lists and dicts.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
|
||||
import pydantic
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
TEXTS = [
|
||||
"",
|
||||
" ",
|
||||
"0",
|
||||
"1",
|
||||
"2",
|
||||
"-1",
|
||||
"+1",
|
||||
"-0",
|
||||
"00",
|
||||
"007",
|
||||
"1_000",
|
||||
"1__000",
|
||||
"_1",
|
||||
"1_",
|
||||
"1.",
|
||||
".5",
|
||||
"1.0",
|
||||
"1.00",
|
||||
"1.5",
|
||||
"-1.0",
|
||||
"1e3",
|
||||
"1E3",
|
||||
"0x10",
|
||||
"0b1",
|
||||
"0o7",
|
||||
" 5 ",
|
||||
"\t5\n",
|
||||
"5 0",
|
||||
"١٢",
|
||||
"inf",
|
||||
"nan",
|
||||
"99999999999999999999",
|
||||
"-99999999999999999999",
|
||||
"true",
|
||||
"True",
|
||||
"TRUE",
|
||||
" true ",
|
||||
"false",
|
||||
"False",
|
||||
"t",
|
||||
"f",
|
||||
"T",
|
||||
"y",
|
||||
"n",
|
||||
"yes",
|
||||
"Yes",
|
||||
"no",
|
||||
"on",
|
||||
"On",
|
||||
"off",
|
||||
"OFF",
|
||||
"maybe",
|
||||
"truthy",
|
||||
"nul",
|
||||
"+",
|
||||
"-",
|
||||
"+1.0",
|
||||
"-0.0",
|
||||
"1_0.0",
|
||||
"1.0_0",
|
||||
"1.0.0",
|
||||
"1 ",
|
||||
"\u00a05",
|
||||
"\u20035",
|
||||
]
|
||||
|
||||
VALUES = [
|
||||
None,
|
||||
True,
|
||||
False,
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
-1,
|
||||
2**63 - 1,
|
||||
2**63,
|
||||
2**64,
|
||||
-(2**63),
|
||||
0.0,
|
||||
-0.0,
|
||||
1.0,
|
||||
0.5,
|
||||
1.5,
|
||||
2.0,
|
||||
-3.0,
|
||||
1e15,
|
||||
1e20,
|
||||
1e300,
|
||||
2.0**62,
|
||||
2.0**63,
|
||||
-(2.0**63),
|
||||
9007199254740993.0,
|
||||
float("inf"),
|
||||
float("-inf"),
|
||||
float("nan"),
|
||||
[],
|
||||
[1],
|
||||
{},
|
||||
{"a": 1},
|
||||
*TEXTS,
|
||||
]
|
||||
|
||||
|
||||
def validated(adapter, value):
|
||||
try:
|
||||
return adapter.validate_python(value)
|
||||
except ValidationError:
|
||||
return None
|
||||
|
||||
|
||||
def row(value):
|
||||
as_int = validated(TypeAdapter(int), value)
|
||||
return {
|
||||
"input": json.dumps(value),
|
||||
"bool": validated(TypeAdapter(bool), value),
|
||||
"int": None if as_int is None else str(as_int),
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
json.dump(
|
||||
{"pydantic": pydantic.VERSION, "rows": [row(value) for value in VALUES]},
|
||||
sys.stdout,
|
||||
indent=2,
|
||||
ensure_ascii=True,
|
||||
)
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
"""Regenerate generated/stdlib.json: `urllib.parse.quote` and UTC `datetime` fields in CPython.
|
||||
|
||||
python scripts/generate_stdlib_fixtures.py > generated/stdlib.json
|
||||
|
||||
`quote` rows record `urllib.parse.quote(text, safe=safe)`. `utc` rows record the fields of
|
||||
`datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)`.
|
||||
"""
|
||||
|
||||
import json
|
||||
import sys
|
||||
from datetime import datetime, timezone
|
||||
from urllib.parse import quote
|
||||
|
||||
QUOTE_TEXTS = [
|
||||
"",
|
||||
"plain",
|
||||
"a/b/c",
|
||||
"a b",
|
||||
"a+b=c&d",
|
||||
"~_.-",
|
||||
"100%",
|
||||
"a/b:c d%#?+=&~é\"q",
|
||||
"id-é-\U0001f600",
|
||||
"日本",
|
||||
"\x00\x1f\x7f",
|
||||
"!*'();:@&=+$,/?#[]",
|
||||
"<>\\^`{|}",
|
||||
]
|
||||
|
||||
SAFES = ["/", "", ":/"]
|
||||
|
||||
TIMESTAMPS = [
|
||||
(0, 0),
|
||||
(1, 999999),
|
||||
(59, 0),
|
||||
(86399, 123456),
|
||||
(86400, 0),
|
||||
(951782400, 0),
|
||||
(951868800, 0),
|
||||
(1700000000, 42),
|
||||
(1757840707, 42),
|
||||
(4102444800, 0),
|
||||
(253402300799, 999999),
|
||||
]
|
||||
|
||||
|
||||
def utc(seconds, micros):
|
||||
moment = datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=micros)
|
||||
return {
|
||||
"seconds": seconds,
|
||||
"micros": micros,
|
||||
"fields": [moment.year, moment.month, moment.day, moment.hour, moment.minute, moment.second, moment.microsecond],
|
||||
}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
json.dump(
|
||||
{
|
||||
"python": sys.version.split()[0],
|
||||
"quote": [{"text": text, "safe": safe, "quoted": quote(text, safe=safe)} for text in QUOTE_TEXTS for safe in SAFES],
|
||||
"utc": [utc(seconds, micros) for seconds, micros in TIMESTAMPS],
|
||||
},
|
||||
sys.stdout,
|
||||
indent=2,
|
||||
ensure_ascii=True,
|
||||
)
|
||||
53
litellm/rust/crates/python-compat/src/datetime.rs
Normal file
53
litellm/rust/crates/python-compat/src/datetime.rs
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
//! The fields of `datetime.fromtimestamp(seconds, timezone.utc)`.
|
||||
|
||||
/// A UTC `datetime`'s fields, as Python's `datetime` exposes them.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct UtcDateTime {
|
||||
pub year: i64,
|
||||
pub month: u32,
|
||||
pub day: u32,
|
||||
pub hour: u32,
|
||||
pub minute: u32,
|
||||
pub second: u32,
|
||||
pub microsecond: u32,
|
||||
}
|
||||
|
||||
impl UtcDateTime {
|
||||
/// `datetime.fromtimestamp(seconds, timezone.utc).replace(microsecond=microsecond)`.
|
||||
pub fn from_timestamp(seconds: i64, microsecond: u32) -> Self {
|
||||
let (year, month, day) = civil_from_days(seconds.div_euclid(86_400));
|
||||
let of_day = seconds.rem_euclid(86_400);
|
||||
Self {
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
hour: (of_day / 3600) as u32,
|
||||
minute: (of_day / 60 % 60) as u32,
|
||||
second: (of_day % 60) as u32,
|
||||
microsecond,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Days since 1970-01-01 to a proleptic Gregorian `(year, month, day)`.
|
||||
///
|
||||
/// Howard Hinnant's `civil_from_days`, <https://howardhinnant.github.io/date_algorithms.html#civil_from_days>:
|
||||
/// shifts the epoch to 0000-03-01 so the leap day ends the year, then counts 400-year eras
|
||||
/// of 146097 days.
|
||||
fn civil_from_days(days: i64) -> (i64, u32, u32) {
|
||||
let shifted = days + 719_468;
|
||||
let era = shifted.div_euclid(146_097);
|
||||
let day_of_era = shifted.rem_euclid(146_097);
|
||||
let year_of_era =
|
||||
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
|
||||
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
|
||||
let month_index = (5 * day_of_year + 2) / 153;
|
||||
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
|
||||
let month = if month_index < 10 {
|
||||
month_index + 3
|
||||
} else {
|
||||
month_index - 9
|
||||
} as u32;
|
||||
let year = year_of_era + era * 400 + i64::from(month <= 2);
|
||||
(year, month, day)
|
||||
}
|
||||
|
|
@ -12,6 +12,9 @@
|
|||
//! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data |
|
||||
//! | [`truthy`] | `bool(value)` |
|
||||
//! | [`number`] | `int(text)`, `float(text)`, and `int(seconds)` as a `Duration` |
|
||||
//! | [`pydantic`] | `TypeAdapter(bool)` and `TypeAdapter(int)` lax validation |
|
||||
//! | [`urllib`] | `urllib.parse.quote(text, safe=safe)` |
|
||||
//! | [`datetime`] | `datetime.fromtimestamp(seconds, timezone.utc)` fields |
|
||||
//!
|
||||
//! [`Value`] is the closed data model these formats share. Live Python objects
|
||||
//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the
|
||||
|
|
@ -24,14 +27,17 @@
|
|||
//! - [`pickle::loads`] decodes `tuple`, `set`, and `frozenset` as lists.
|
||||
//! - [`number`] reads ASCII digits only, while Python's `int` and `float` accept any Unicode decimal digit.
|
||||
|
||||
pub mod datetime;
|
||||
mod error;
|
||||
pub mod json;
|
||||
pub mod literal;
|
||||
pub mod number;
|
||||
pub mod pickle;
|
||||
pub mod pydantic;
|
||||
pub mod repr;
|
||||
pub mod reprlib;
|
||||
pub mod truthy;
|
||||
pub mod urllib;
|
||||
mod value;
|
||||
|
||||
pub use error::Error;
|
||||
|
|
|
|||
70
litellm/rust/crates/python-compat/src/pydantic.rs
Normal file
70
litellm/rust/crates/python-compat/src/pydantic.rs
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
//! Pydantic's lax `bool` and `int` validation: `TypeAdapter(bool)` and `TypeAdapter(int)`.
|
||||
|
||||
use num_bigint::BigInt;
|
||||
|
||||
use crate::Value;
|
||||
|
||||
const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0;
|
||||
|
||||
/// `TypeAdapter(bool).validate_python(value)`, or `None` where pydantic raises `ValidationError`.
|
||||
pub fn lax_bool(value: &Value) -> Option<bool> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(*flag),
|
||||
Value::Int(number) => match i8::try_from(number) {
|
||||
Ok(0) => Some(false),
|
||||
Ok(1) => Some(true),
|
||||
_ => None,
|
||||
},
|
||||
Value::Float(number) if *number == 0.0 => Some(false),
|
||||
Value::Float(number) if *number == 1.0 => Some(true),
|
||||
Value::Str(text) => match text.to_ascii_lowercase().as_str() {
|
||||
"0" | "off" | "f" | "false" | "n" | "no" => Some(false),
|
||||
"1" | "on" | "t" | "true" | "y" | "yes" => Some(true),
|
||||
_ => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// `TypeAdapter(int).validate_python(value)`, or `None` where pydantic raises `ValidationError`.
|
||||
pub fn lax_int(value: &Value) -> Option<BigInt> {
|
||||
match value {
|
||||
Value::Bool(flag) => Some(BigInt::from(u8::from(*flag))),
|
||||
Value::Int(number) => Some(number.clone()),
|
||||
Value::Float(number)
|
||||
if number.fract() == 0.0 && (-TWO_POW_63 < *number && *number < TWO_POW_63) =>
|
||||
{
|
||||
Some(BigInt::from(*number as i64))
|
||||
}
|
||||
Value::Str(text) => int_text(text.trim()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn int_text(text: &str) -> Option<BigInt> {
|
||||
let whole = match text.split_once('.') {
|
||||
Some((whole, fraction))
|
||||
if !fraction.is_empty() && fraction.bytes().all(|byte| byte == b'0') =>
|
||||
{
|
||||
whole
|
||||
}
|
||||
Some(_) => return None,
|
||||
None => text,
|
||||
};
|
||||
let (negative, digits) = match whole.strip_prefix('-') {
|
||||
Some(rest) => (true, rest),
|
||||
None => (false, whole.strip_prefix('+').unwrap_or(whole)),
|
||||
};
|
||||
let well_formed = !digits.is_empty()
|
||||
&& !digits.starts_with('_')
|
||||
&& !digits.ends_with('_')
|
||||
&& !digits.contains("__")
|
||||
&& digits
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || byte == b'_');
|
||||
if !well_formed {
|
||||
return None;
|
||||
}
|
||||
let magnitude = BigInt::parse_bytes(digits.replace('_', "").as_bytes(), 10)?;
|
||||
Some(if negative { -magnitude } else { magnitude })
|
||||
}
|
||||
18
litellm/rust/crates/python-compat/src/urllib.rs
Normal file
18
litellm/rust/crates/python-compat/src/urllib.rs
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
//! `urllib.parse.quote(text, safe=safe)` over the UTF-8 bytes of `text`.
|
||||
|
||||
/// `urllib.parse.quote(text, safe=safe)`: ASCII letters, digits and `_.-~` are never quoted,
|
||||
/// the ASCII characters in `safe` are kept, and every other byte becomes `%XX`.
|
||||
pub fn quote(text: &str, safe: &str) -> String {
|
||||
let mut encoded = String::with_capacity(text.len());
|
||||
for byte in text.bytes() {
|
||||
if byte.is_ascii_alphanumeric()
|
||||
|| matches!(byte, b'_' | b'.' | b'-' | b'~')
|
||||
|| (byte.is_ascii() && safe.as_bytes().contains(&byte))
|
||||
{
|
||||
encoded.push(char::from(byte));
|
||||
} else {
|
||||
encoded.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
}
|
||||
48
litellm/rust/crates/python-compat/tests/pydantic.rs
Normal file
48
litellm/rust/crates/python-compat/tests/pydantic.rs
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
//! Replays `generated/pydantic.json`, which pydantic wrote with `scripts/generate_pydantic_fixtures.py`.
|
||||
|
||||
use litellm_python_compat::{
|
||||
json,
|
||||
pydantic::{lax_bool, lax_int},
|
||||
};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Fixtures {
|
||||
rows: Vec<Row>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Row {
|
||||
input: String,
|
||||
bool: Option<bool>,
|
||||
int: Option<String>,
|
||||
}
|
||||
|
||||
fn fixtures() -> Fixtures {
|
||||
serde_json::from_str(include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/generated/pydantic.json"
|
||||
)))
|
||||
.expect("pydantic.json matches the fixture schema")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lax_bool_and_int_match_pydantic() {
|
||||
let mismatches: Vec<String> = fixtures()
|
||||
.rows
|
||||
.iter()
|
||||
.flat_map(|row| {
|
||||
let value = json::loads(&row.input).expect("inputs are json.dumps output");
|
||||
let as_bool = lax_bool(&value);
|
||||
let as_int = lax_int(&value).map(|number| number.to_string());
|
||||
[
|
||||
(as_bool != row.bool)
|
||||
.then(|| format!("bool {}: {as_bool:?} != {:?}", row.input, row.bool)),
|
||||
(as_int != row.int)
|
||||
.then(|| format!("int {}: {as_int:?} != {:?}", row.input, row.int)),
|
||||
]
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(mismatches.is_empty(), "{mismatches:#?}");
|
||||
}
|
||||
62
litellm/rust/crates/python-compat/tests/stdlib.rs
Normal file
62
litellm/rust/crates/python-compat/tests/stdlib.rs
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
//! Replays `generated/stdlib.json`, which CPython wrote with `scripts/generate_stdlib_fixtures.py`.
|
||||
|
||||
use litellm_python_compat::{datetime::UtcDateTime, urllib::quote};
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Fixtures {
|
||||
quote: Vec<Quote>,
|
||||
utc: Vec<Utc>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Quote {
|
||||
text: String,
|
||||
safe: String,
|
||||
quoted: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct Utc {
|
||||
seconds: i64,
|
||||
micros: u32,
|
||||
fields: [i64; 7],
|
||||
}
|
||||
|
||||
fn fixtures() -> Fixtures {
|
||||
serde_json::from_str(include_str!(concat!(
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/generated/stdlib.json"
|
||||
)))
|
||||
.expect("stdlib.json matches the fixture schema")
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quote_matches_urllib() {
|
||||
for row in fixtures().quote {
|
||||
assert_eq!(
|
||||
quote(&row.text, &row.safe),
|
||||
row.quoted,
|
||||
"{:?} safe={:?}",
|
||||
row.text,
|
||||
row.safe
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn utc_fields_match_datetime() {
|
||||
for row in fixtures().utc {
|
||||
let moment = UtcDateTime::from_timestamp(row.seconds, row.micros);
|
||||
let fields = [
|
||||
moment.year,
|
||||
i64::from(moment.month),
|
||||
i64::from(moment.day),
|
||||
i64::from(moment.hour),
|
||||
i64::from(moment.minute),
|
||||
i64::from(moment.second),
|
||||
i64::from(moment.microsecond),
|
||||
];
|
||||
assert_eq!(fields, row.fields, "{} {}", row.seconds, row.micros);
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue