feat(python-compat): reproduce int(), float(), and pydantic's lax int

number::int and number::float follow CPython's conversions of plain data, and
pydantic::lax_int follows TypeAdapter(int) in lax mode, which LiteLLM's usage
models apply to provider numbers. The fixture generator records each
conversion's result or exception class for every corpus row, now including
numeric text

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Yujong Lee 2026-09-23 19:44:05 -07:00
parent 3e8b7d6775
commit fcff8775c7
9 changed files with 1738 additions and 7 deletions

View file

@ -3159,6 +3159,7 @@ dependencies = [
"serde",
"serde-pickle",
"serde_json",
"strum",
"thiserror 2.0.19",
]

View file

@ -12,6 +12,7 @@ num-traits = "0.2"
serde.workspace = true
serde-pickle = "1.2"
serde_json = { workspace = true, features = ["preserve_order"] }
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]

File diff suppressed because it is too large Load diff

View file

@ -2,8 +2,9 @@
python scripts/generate_fixtures.py > generated/values.json
Each row records `repr`, `str`, `json.dumps` (or its error), `bool`, and `pickle.dumps` at
every protocol. `literal` says whether `ast.literal_eval(repr(value))` gives the value back,
Each row records `repr`, `str`, `json.dumps` (or its error), `bool`, `int`, `float`, pydantic's
lax `int` validation, and `pickle.dumps` at every protocol. Conversions record the result's
`repr` or the exception class raised. Run it with the repository environment, which has pydantic. `literal` says whether `ast.literal_eval(repr(value))` gives the value back,
which is how Python reads `str(dict)` text back from a cache; the Rust tests reach the other
rows only through pickle. `view` is `repr` of the value as `pickle::loads` decodes it, with
tuples, sets, and frozensets rendered as lists. `sources` records `ast.literal_eval` on raw
@ -16,6 +17,10 @@ import pickle
import sys
import warnings
from pydantic import TypeAdapter, ValidationError
LAX_INT = TypeAdapter(int)
# Entries are source texts, or `(name, source)` when the source is too long to read in a
# test report. `name` is what the Rust `KNOWN` table keys on.
CORPUS = [
@ -79,6 +84,35 @@ CORPUS = [
'b"a\'b"',
"b'a\"b\\'c'",
"b'\\x00\\t\\n\\r\\x7f\\x80\\xff'",
# Numeric text, as providers and config files send it
"' 12 '",
"'+5'",
"'-4'",
"'007'",
"'1_000'",
"'1__0'",
"'_1'",
"'1_'",
"'3.0'",
"'3.00'",
"'3.5'",
"' 0.5 '",
"'.5'",
"'5.'",
"'1e3'",
"'1_0.5'",
"'1e1_0'",
"'inf'",
"'-Infinity'",
"'nan'",
"'NaN'",
"'0x10'",
"'abc'",
"b'12'",
"b' 1.5 '",
"3.9",
"-3.9",
"2.5",
# Containers
"[]",
"[1, 'a', None, True]",
@ -314,6 +348,13 @@ def is_literal(value):
return repr(parsed) == repr(value)
def converted(convert, value):
try:
return {"value": repr(convert(value))}
except (TypeError, ValueError, OverflowError, ValidationError) as error:
return {"error": "ValueError" if isinstance(error, ValidationError) else type(error).__name__}
def row(entry):
name, source = named(entry)
value = eval(source)
@ -325,6 +366,9 @@ def row(entry):
"repr": repr(value),
"str": str(value),
"truthy": bool(value),
"int": converted(int, value),
"float": converted(float, value),
"lax_int": converted(LAX_INT.validate_python, value),
}
try:
entry["json"] = json.dumps(value)

View file

@ -1,5 +1,13 @@
use crate::MAX_DEPTH;
/// The CPython exception class a failed conversion raises.
#[derive(Clone, Copy, Debug, Eq, PartialEq, strum::Display, strum::IntoStaticStr)]
pub enum PythonException {
TypeError,
ValueError,
OverflowError,
}
/// A failure to read or write a Python format. Messages quote CPython's own wording where
/// the Python side raises (`TypeError`, `ValueError`), so callers can log them as is.
#[derive(Debug, thiserror::Error)]
@ -22,4 +30,18 @@ pub enum Error {
IntegerOutOfRange,
#[error("Object of type {0} cannot be pickled as plain data")]
NotPicklable(&'static str),
#[error("{exception}: {message}")]
Conversion {
exception: PythonException,
message: String,
},
}
impl Error {
pub(crate) fn conversion(exception: PythonException, message: impl Into<String>) -> Self {
Self::Conversion {
exception,
message: message.into(),
}
}
}

View file

@ -10,6 +10,8 @@
//! | [`json`] | `json.dumps(value)`, and `json.loads(json.dumps(value))` as a JSON value |
//! | [`pickle`] | `pickle.loads(data)` and `pickle.dumps(value)` for plain data |
//! | [`truthy`] | `bool(value)` |
//! | [`number`] | `int(value)` and `float(value)` |
//! | [`pydantic`] | pydantic's lax-mode `int` validation |
//!
//! [`Value`] is the closed data model these formats share. Live Python objects
//! (descriptors, `__bool__`, `__str__`, callbacks) are out of scope: those belong to the
@ -24,12 +26,14 @@
mod error;
pub mod json;
pub mod literal;
pub mod number;
pub mod pickle;
pub mod pydantic;
pub mod repr;
pub mod truthy;
mod value;
pub use error::Error;
pub use error::{Error, PythonException};
pub use num_bigint::BigInt;
pub use value::Value;

View file

@ -0,0 +1,118 @@
//! `int(value)` and `float(value)` for plain data.
use num_bigint::BigInt;
use num_traits::{FromPrimitive, ToPrimitive};
use crate::repr::repr;
use crate::{Error, PythonException, Value};
/// Digits with single underscores between them, as Python's numeric literals allow.
pub(crate) fn without_digit_separators(text: &str) -> Option<String> {
let bytes = text.as_bytes();
let separators_ok = bytes.iter().enumerate().all(|(index, byte)| {
*byte != b'_'
|| (index > 0
&& bytes[index - 1].is_ascii_digit()
&& bytes.get(index + 1).is_some_and(u8::is_ascii_digit))
});
separators_ok.then(|| text.replace('_', ""))
}
pub(crate) fn parse_int_literal(text: &str) -> Option<BigInt> {
let trimmed = text.trim();
let digits = trimmed.strip_prefix(['+', '-']).unwrap_or(trimmed);
if digits.is_empty()
|| !digits
.bytes()
.all(|byte| byte.is_ascii_digit() || byte == b'_')
{
return None;
}
without_digit_separators(trimmed)?.parse().ok()
}
fn int_from_text(text: &str, shown: &Value) -> Result<BigInt, Error> {
parse_int_literal(text).ok_or_else(|| {
Error::conversion(
PythonException::ValueError,
format!("invalid literal for int() with base 10: {}", repr(shown)),
)
})
}
pub(crate) fn int_from_float(value: f64) -> Result<BigInt, Error> {
if value.is_nan() {
return Err(Error::conversion(
PythonException::ValueError,
"cannot convert float NaN to integer",
));
}
BigInt::from_f64(value.trunc()).ok_or_else(|| {
Error::conversion(
PythonException::OverflowError,
"cannot convert float infinity to integer",
)
})
}
/// `int(value)`.
pub fn int(value: &Value) -> Result<BigInt, Error> {
match value {
Value::Bool(flag) => Ok(BigInt::from(u8::from(*flag))),
Value::Int(number) => Ok(number.clone()),
Value::Float(number) => int_from_float(*number),
Value::Str(text) => int_from_text(text, value),
Value::Bytes(bytes) => int_from_text(&String::from_utf8_lossy(bytes), value),
other => Err(Error::conversion(
PythonException::TypeError,
format!(
"int() argument must be a string, a bytes-like object or a real number, not '{}'",
other.type_name()
),
)),
}
}
fn float_from_text(text: &str, shown: &Value) -> Result<f64, Error> {
let trimmed = text.trim();
let well_formed = !trimmed.is_empty()
&& trimmed
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'.' | b'_' | b'+' | b'-'));
well_formed
.then(|| without_digit_separators(trimmed))
.flatten()
.and_then(|number| number.parse().ok())
.ok_or_else(|| {
Error::conversion(
PythonException::ValueError,
format!("could not convert string to float: {}", repr(shown)),
)
})
}
/// `float(value)`.
pub fn float(value: &Value) -> Result<f64, Error> {
match value {
Value::Bool(flag) => Ok(f64::from(u8::from(*flag))),
Value::Int(number) => number
.to_f64()
.filter(|float| float.is_finite())
.ok_or_else(|| {
Error::conversion(
PythonException::OverflowError,
"int too large to convert to float",
)
}),
Value::Float(number) => Ok(*number),
Value::Str(text) => float_from_text(text, value),
Value::Bytes(bytes) => float_from_text(&String::from_utf8_lossy(bytes), value),
other => Err(Error::conversion(
PythonException::TypeError,
format!(
"float() argument must be a string or a real number, not '{}'",
other.type_name()
),
)),
}
}

View file

@ -0,0 +1,54 @@
//! pydantic's lax-mode validation, which LiteLLM's usage and response models apply to the
//! numbers providers send.
use num_bigint::BigInt;
use num_traits::Zero;
use crate::number::{int_from_float, parse_int_literal};
use crate::{Error, PythonException, Value};
/// pydantic-core converts floats through `i64`, so larger magnitudes fail validation.
const I64_LIMIT: f64 = 9_223_372_036_854_775_808.0;
/// pydantic raises `ValidationError`, a `ValueError` subclass, for every rejected input.
fn invalid(message: &str) -> Error {
Error::conversion(PythonException::ValueError, message)
}
fn int_from_text(text: &str) -> Result<BigInt, Error> {
let trimmed = text.trim();
let without_zero_fraction = trimmed
.split_once('.')
.filter(|(_, fraction)| !fraction.is_empty() && fraction.bytes().all(|byte| byte == b'0'))
.map_or(trimmed, |(whole, _)| whole);
parse_int_literal(without_zero_fraction).ok_or_else(|| {
invalid("Input should be a valid integer, unable to parse string as an integer")
})
}
/// `TypeAdapter(int).validate_python(value)` in lax mode.
pub fn lax_int(value: &Value) -> Result<BigInt, Error> {
match value {
Value::Bool(flag) => Ok(BigInt::from(u8::from(*flag))),
Value::Int(number) => Ok(number.clone()),
Value::Float(number) if !number.is_finite() => {
Err(invalid("Input should be a finite number"))
}
Value::Float(number) if number.abs() >= I64_LIMIT => Err(invalid(
"Input should be a valid integer, unable to parse number as an integer",
)),
Value::Float(number) => {
let whole = int_from_float(*number)?;
if (number - number.trunc()).is_zero() {
Ok(whole)
} else {
Err(invalid(
"Input should be a valid integer, got a number with a fractional part",
))
}
}
Value::Str(text) => int_from_text(text),
Value::Bytes(bytes) => int_from_text(&String::from_utf8_lossy(bytes)),
_ => Err(invalid("Input should be a valid integer")),
}
}

View file

@ -3,10 +3,10 @@
use std::{collections::BTreeMap, fs::File, io::Write};
use litellm_python_compat::{
Value, json,
Error, Value, json,
literal::literal_eval,
pickle,
repr::{repr, to_str},
number, pickle, pydantic,
repr::{float_repr, repr, to_str},
truthy::truthy,
};
use rstest::{fixture, rstest};
@ -27,6 +27,31 @@ struct Source {
error: Option<String>,
}
/// A conversion's result `repr`, or the exception class it raised.
#[derive(Deserialize)]
struct Conversion {
value: Option<String>,
error: Option<String>,
}
impl Conversion {
fn expected(&self) -> String {
match (&self.value, &self.error) {
(Some(value), None) => value.clone(),
(None, Some(error)) => format!("raises {error}"),
_ => panic!("a conversion records a value or an error"),
}
}
}
fn outcome<T>(result: Result<T, Error>, show: impl Fn(T) -> String) -> String {
match result {
Ok(value) => show(value),
Err(Error::Conversion { exception, .. }) => format!("raises {exception}"),
Err(error) => format!("unexpected error: {error}"),
}
}
#[derive(Deserialize)]
struct Row {
name: String,
@ -36,6 +61,9 @@ struct Row {
repr: String,
str: String,
truthy: bool,
int: Conversion,
float: Conversion,
lax_int: Conversion,
json: Option<String>,
json_error: Option<String>,
pickle: Option<BTreeMap<String, String>>,
@ -190,6 +218,24 @@ fn check_value(mismatches: &mut Mismatches, row: &Row, value: &Value) {
&row.truthy.to_string(),
&truthy(value).to_string(),
);
mismatches.check(
row,
"int",
&row.int.expected(),
&outcome(number::int(value), |int| int.to_string()),
);
mismatches.check(
row,
"float",
&row.float.expected(),
&outcome(number::float(value), float_repr),
);
mismatches.check(
row,
"pydantic lax int",
&row.lax_int.expected(),
&outcome(pydantic::lax_int(value), |int| int.to_string()),
);
let expected = row.json.clone().or_else(|| {
row.json_error
.clone()
@ -208,7 +254,7 @@ fn check_value(mismatches: &mut Mismatches, row: &Row, value: &Value) {
}
#[rstest]
fn literal_rows_match_python_repr_str_bool_and_json(fixtures: &Fixtures) {
fn literal_rows_match_python_repr_str_bool_numbers_and_json(fixtures: &Fixtures) {
let mut mismatches = Mismatches::default();
for row in fixtures.rows.iter().filter(|row| row.literal) {
match literal_eval(&row.repr) {