Merge remote-tracking branch 'origin/main' into litellm_bedrock_v1_messages_stream_passthrough

This commit is contained in:
mateo 2026-09-23 01:52:08 +00:00
commit 550511a983
232 changed files with 15555 additions and 2198 deletions

View file

@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
r"|^tests/e2e/secret_manager/"
)
HARNESS: Final = re.compile(
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"

View file

@ -4,7 +4,13 @@ on:
workflow_call:
inputs:
test-path:
description: "Pytest path(s) to run"
description: >-
Space-separated pytest paths to run. A path that no longer exists is
dropped with a warning instead of being passed to pytest, because one
missing path makes pytest-xdist collect nothing and report exit 5, which
the step treats as a drained shard. Options are passed through as
written, so use the `--flag=value` form: a bare `--ignore path` would
have its path existence-checked like any other token.
required: true
type: string
workers:
@ -165,14 +171,22 @@ jobs:
DIST: ${{ inputs.dist }}
COVERAGE_CORE: sysmon
run: |
found_path=false
for path in ${TEST_PATH}; do
if [ -e "${path%%::*}" ]; then
found_path=true
break
fi
pytest_args=()
existing_paths=0
for token in ${TEST_PATH:?}; do
case "${token}" in
-*) pytest_args+=("${token}") ;;
*)
if [ -e "${token%%::*}" ]; then
pytest_args+=("${token}")
existing_paths=$((existing_paths + 1))
else
echo "::warning::${token} does not exist; drop it from this shard's test-path"
fi
;;
esac
done
if [ "$found_path" = false ]; then
if [ "${existing_paths}" -eq 0 ]; then
echo "No path in TEST_PATH exists (${TEST_PATH}); nothing to run"
exit 0
fi
@ -181,7 +195,7 @@ jobs:
xdist_args=(-n "${WORKERS}" --dist="${DIST}")
fi
set +e
uv run --no-sync pytest ${TEST_PATH:?} \
uv run --no-sync pytest "${pytest_args[@]}" \
--tb=short -vv \
--maxfail="${MAX_FAILURES}" \
"${xdist_args[@]}" \

View file

@ -6,8 +6,12 @@ on:
workflow_dispatch:
inputs:
issue_number:
description: "Closed issue number to comment on manually."
required: true
description: "Closed issue number to comment on and close the superseded pull requests of. Ignored by a sweep."
required: false
sweep:
description: "Close every open pull request whose linked issues were all fixed on the default branch. Reads every open pull request, so run it at most once an hour."
type: boolean
default: false
pull_request:
paths:
- .github/workflows/issue_fixed_comment.yml
@ -39,16 +43,17 @@ jobs:
with:
bun-version: "1.4.0"
- name: Test the closer lookup, the release placement and the comment
- name: Test the closer lookup, the release placement, the comment and the superseded pull request close
run: bun test scripts/comment-fixed-issue.test.ts
comment-fixed-issue:
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
timeout-minutes: 15
permissions:
contents: read
issues: write
pull-requests: write
steps:
- name: Checkout scripts
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@ -59,13 +64,16 @@ jobs:
- name: Setup Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
# Exact version, never latest: the next step holds an issues: write token
# Exact version, never latest: the next step holds issues: write and pull-requests: write tokens
bun-version: "1.4.0"
- name: Name the release that carries the fix
- name: Name the release that carries the fix and close the pull requests it supersedes
shell: bash
run: bun run scripts/comment-fixed-issue.ts | tee -a "${GITHUB_STEP_SUMMARY}"
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
SWEEP: ${{ github.event.inputs.sweep }}
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
DRY_RUN: ${{ vars.ISSUE_FIXED_COMMENT_ENABLED != 'true' }}
CLOSE_PRS_DRY_RUN: ${{ vars.ISSUE_FIXED_CLOSE_PRS_ENABLED != 'true' }}

View file

@ -105,6 +105,16 @@ jobs:
with:
python-version: "3.12"
- uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install Python dependencies for the bridge tests
working-directory: .
run: |
uv sync --frozen --no-install-project
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
- run: rustup toolchain install --no-self-update
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8

View file

@ -107,26 +107,18 @@ jobs:
tests/test_litellm/batches
tests/test_litellm/secret_managers
tests/test_litellm/a2a_protocol
tests/test_litellm/anthropic_interface
tests/test_litellm/chat_completions
tests/test_litellm/completion_extras
tests/test_litellm/compression
tests/test_litellm/containers
tests/test_litellm/endpoints
tests/test_litellm/models
tests/test_litellm/repositories
tests/test_litellm/images
tests/test_litellm/interactions
tests/test_litellm/messages
tests/test_litellm/ocr
tests/test_litellm/passthrough
tests/test_litellm/rag
tests/test_litellm/realtime_api
tests/test_litellm/rerank_api
tests/test_litellm/rust_bridge
tests/test_litellm/sandbox
tests/test_litellm/skills
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
tests/test_litellm/test_*.py

View file

@ -2946,6 +2946,7 @@ name = "litellm-core-utils"
version = "0.1.0"
dependencies = [
"fancy-regex 0.19.2",
"litellm-tracing",
"litellm-types",
"rstest",
"serde",
@ -2956,6 +2957,14 @@ dependencies = [
"url",
]
[[package]]
name = "litellm-cost"
version = "0.1.0"
dependencies = [
"criterion",
"proptest",
]
[[package]]
name = "litellm-framing"
version = "0.1.0"
@ -3046,6 +3055,20 @@ dependencies = [
"url",
]
[[package]]
name = "litellm-model-catalog"
version = "0.1.0"
dependencies = [
"criterion",
"indexmap 2.14.0",
"litellm-model-catalog",
"rstest",
"schemars 1.2.2",
"serde",
"serde_json",
"thiserror 2.0.19",
]
[[package]]
name = "litellm-python-bridge"
version = "0.1.0"
@ -3053,6 +3076,7 @@ dependencies = [
"aws-sdk-secretsmanager",
"bytes",
"criterion",
"fancy-regex 0.19.2",
"futures-util",
"litellm-auth",
"litellm-auth-aws",
@ -3071,6 +3095,7 @@ dependencies = [
"litellm-callbacks-legacy-python",
"litellm-core",
"litellm-core-utils",
"litellm-host",
"litellm-host-python",
"litellm-http",
"litellm-llms",
@ -3078,6 +3103,7 @@ dependencies = [
"litellm-secrets-aws",
"litellm-secrets-types",
"litellm-token-counter",
"litellm-tracing",
"litellm-types",
"pyo3",
"pyo3-async-runtimes",
@ -3150,11 +3176,11 @@ dependencies = [
"litellm-auth-aws",
"litellm-core-utils",
"litellm-secrets-types",
"litellm-tracing",
"rstest",
"serde_json",
"thiserror 2.0.19",
"tokio",
"tracing",
"veil",
"wiremock",
]
@ -3186,6 +3212,7 @@ dependencies = [
"base64 0.22.1",
"litellm-core-utils",
"litellm-secrets-types",
"litellm-tracing",
"moka",
"percent-encoding",
"reqwest 0.12.28",
@ -3194,7 +3221,6 @@ dependencies = [
"serde_json",
"thiserror 2.0.19",
"tokio",
"tracing",
"veil",
"wiremock",
]
@ -3309,6 +3335,19 @@ dependencies = [
"tiktoken-rs",
]
[[package]]
name = "litellm-tracing"
version = "0.1.0"
dependencies = [
"fancy-regex 0.19.2",
"percent-encoding",
"rstest",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "litellm-types"
version = "0.1.0"
@ -4748,10 +4787,23 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a"
dependencies = [
"dyn-clone",
"ref-cast",
"schemars_derive",
"serde",
"serde_json",
]
[[package]]
name = "schemars_derive"
version = "1.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba"
dependencies = [
"proc-macro2",
"quote",
"serde_derive_internals",
"syn 3.0.0",
]
[[package]]
name = "scopeguard"
version = "1.2.0"
@ -4840,6 +4892,17 @@ dependencies = [
"syn 3.0.0",
]
[[package]]
name = "serde_derive_internals"
version = "0.30.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd"
dependencies = [
"proc-macro2",
"quote",
"syn 3.0.0",
]
[[package]]
name = "serde_json"
version = "1.0.150"

View file

@ -9,6 +9,8 @@ license = "MIT"
repository = "https://github.com/BerriAI/litellm"
[workspace.dependencies]
litellm-tracing = { path = "crates/tracing" }
tracing = "0.1"
litellm-core = { path = "crates/core" }
litellm-host = { path = "crates/host" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }

View file

@ -7,6 +7,7 @@ repository.workspace = true
[dependencies]
fancy-regex.workspace = true
litellm-tracing.workspace = true
litellm-types.workspace = true
serde.workspace = true
serde_json.workspace = true

View file

@ -1,109 +1 @@
use fancy_regex::Regex;
pub const REDACTED: &str = "REDACTED";
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
fn minimum_custom_key_length() -> usize {
std::env::var("MINIMUM_CUSTOM_KEY_LENGTH")
.ok()
.and_then(|value| value.trim().parse().ok())
.unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH)
}
fn secret_patterns(minimum_custom_key_length: usize) -> String {
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
[
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
r"\bya29\.[A-Za-z0-9_.~+/-]+",
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r"x-ak-[A-Za-z0-9\-_]{20,}",
r"AIza[0-9A-Za-z\-_]{35}",
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
r"dapi[0-9a-f]{32}",
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
concat!(
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
r"aws_secret_access_key|aws_session_token|aws_access_key_id|",
r"signing_key|encryption_key|",
r"auth_token|access_token|refresh_token|",
r"slack_webhook_url|webhook_url|",
r"database_connection_string|",
r"huggingface_token|jwt_secret)",
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
),
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
]
.join("|")
}
/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration.
#[derive(Clone, Debug)]
pub struct SecretRedactor {
pattern: Regex,
}
impl SecretRedactor {
pub fn new(minimum_custom_key_length: usize) -> Self {
let pattern = Regex::new(&format!(
"(?i){}",
secret_patterns(minimum_custom_key_length)
))
.expect("secret redaction patterns compile");
Self { pattern }
}
/// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off.
pub fn from_env() -> Option<Self> {
let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS")
.is_ok_and(|value| value.eq_ignore_ascii_case("true"));
(!disabled).then(|| Self::new(minimum_custom_key_length()))
}
pub fn redact(&self, value: &str) -> String {
self.pattern.replace_all(value, REDACTED).into_owned()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
expected
);
}
#[test]
fn sk_threshold_follows_the_minimum_custom_key_length() {
let redactor = SecretRedactor::new(8);
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
}
}
pub use litellm_tracing::{REDACTED, SecretRedactor};

View file

@ -0,0 +1,14 @@
[package]
name = "litellm-cost"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dev-dependencies]
criterion.workspace = true
proptest.workspace = true
[[bench]]
name = "calculate"
harness = false

View file

@ -0,0 +1,13 @@
# litellm-cost
This crate calculates text token charges from rates and usage supplied by its caller. It is standalone and has no Python bridge or proxy integration
Call `compile(&pricing)` once for an immutable plan, then `plan.calculate(&request)` for each supported request. `calculate(&pricing, &request)` compiles on each call. A successful result exposes pre-multiplier component costs, selected rates, the multiplier, and derived `input()`, `output()`, and `total()` values
The caller states whether `prompt_tokens` includes cache tokens. Threshold selection uses total input tokens for either convention and selects one rate for the whole request. Thresholds are sorted when compiled, and duplicate thresholds or tier overrides fail deterministically. `Fast` selects priority rates; unknown tiers use standard rates
`Rate::Missing`, `Rate::Null`, and `Rate::Value(0.0)` remain distinct. Missing cache rates fall back to the selected input rate, and an absent one-hour write rate falls back to the selected write rate. Missing input or output rates return typed errors, including for zero usage. Python's sparse-entry behavior remains outside this native contract
The supported off-peak shape is one non-wrapping UTC daily window. The caller supplies the applicable regional multiplier after provider-specific selection. Negative or non-finite rates, ambiguous rules, inconsistent cache counts, incomplete write splits, invalid windows and overflow return errors. Callers must decline unsupported inputs before native execution if their public contract accepts those shapes
This crate does not select models, read catalogs, fetch provider prices, normalize multimodal usage, process provider-reported costs, or calculate non-token charges. It does not change proxy behavior. The reference fixture was generated by `tests/generate_python_reference.py` against the Python implementation at the commit recorded in `tests/python_reference.tsv`, using synthetic rates and fixed usage

View file

@ -0,0 +1,66 @@
use criterion::{Criterion, criterion_group, criterion_main};
use litellm_cost::{
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, ThresholdRates,
Usage, calculate, compile,
};
use std::hint::black_box;
fn bench(c: &mut Criterion) {
let pricing = Pricing {
standard: Rates {
input: Rate::Value(0.000002),
output: Rate::Value(0.000008),
cache_read: Rate::Value(0.0000005),
cache_write: Rate::Missing,
cache_write_1h: Rate::Missing,
},
tiers: &[],
thresholds: &[],
off_peak: None,
};
let request = Request {
usage: Usage {
prompt_tokens: 1000,
completion_tokens: 200,
cache_read_tokens: 250,
cache_write_tokens: 0,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
};
let plan = compile(&pricing).unwrap();
c.bench_function("native_compiled_calculation", |b| {
b.iter(|| black_box(plan.calculate(black_box(&request)).unwrap()))
});
c.bench_function("native_full_wrapper", |b| {
b.iter(|| black_box(calculate(black_box(&pricing), black_box(&request)).unwrap()))
});
c.bench_function("native_rate_compilation", |b| {
b.iter(|| black_box(compile(black_box(&pricing)).unwrap()))
});
let threshold = ThresholdRates {
above_prompt_tokens: 1000,
standard: Rates {
input: Rate::Value(0.000004),
output: Rate::Value(0.000016),
..Rates::EMPTY
},
tiers: &[],
};
let threshold_pricing = Pricing {
thresholds: &[threshold],
..pricing
};
let threshold_plan = compile(&threshold_pricing).unwrap();
c.bench_function("native_threshold_boundary", |b| {
b.iter(|| black_box(threshold_plan.calculate(black_box(&request)).unwrap()))
});
}
criterion_group!(benches, bench);
criterion_main!(benches);

View file

@ -0,0 +1,40 @@
use litellm_cost::{
Pricing, PromptConvention, Rate, Rates, Request, ServiceTier, ThresholdPolicy, Usage, compile,
};
fn main() {
let pricing = Pricing {
standard: Rates {
input: Rate::Value(2.0),
output: Rate::Value(4.0),
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
cache_write_1h: Rate::Missing,
},
tiers: &[],
thresholds: &[],
off_peak: None,
};
let request = Request {
usage: Usage {
prompt_tokens: 100,
completion_tokens: 20,
cache_read_tokens: 25,
cache_write_tokens: 10,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
};
let cost = compile(&pricing).unwrap().calculate(&request).unwrap();
println!(
"input={} output={} total={}",
cost.input(),
cost.output(),
cost.total()
);
}

View file

@ -0,0 +1,405 @@
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum Rate {
Missing,
Null,
Value(f64),
}
impl Rate {
fn value(self) -> Option<f64> {
match self {
Self::Value(value) => Some(value),
Self::Missing | Self::Null => None,
}
}
fn or(self, fallback: Self) -> Self {
if self.value().is_some() {
self
} else {
fallback
}
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Rates {
pub input: Rate,
pub output: Rate,
pub cache_read: Rate,
pub cache_write: Rate,
pub cache_write_1h: Rate,
}
impl Rates {
pub const EMPTY: Self = Self {
input: Rate::Missing,
output: Rate::Missing,
cache_read: Rate::Missing,
cache_write: Rate::Missing,
cache_write_1h: Rate::Missing,
};
fn overlay(self, base: Self) -> Self {
Self {
input: self.input.or(base.input),
output: self.output.or(base.output),
cache_read: self.cache_read.or(base.cache_read),
cache_write: self.cache_write.or(base.cache_write),
cache_write_1h: self.cache_write_1h.or(base.cache_write_1h),
}
}
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ServiceTier {
Standard,
Flex,
Priority,
Fast,
Ultrafast,
Unknown,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum ThresholdPolicy {
Exclusive,
Inclusive,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PromptConvention {
IncludesCache,
ExcludesCache,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Usage {
pub prompt_tokens: u64,
pub completion_tokens: u64,
pub cache_read_tokens: u64,
pub cache_write_tokens: u64,
pub cache_write_5m_tokens: Option<u64>,
pub cache_write_1h_tokens: Option<u64>,
pub prompt_convention: PromptConvention,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct TierRates {
pub tier: ServiceTier,
pub rates: Rates,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct ThresholdRates<'a> {
pub above_prompt_tokens: u64,
pub standard: Rates,
pub tiers: &'a [TierRates],
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct OffPeakRates {
pub start_utc_minute: u16,
pub end_utc_minute: u16,
pub rates: Rates,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Pricing<'a> {
pub standard: Rates,
pub tiers: &'a [TierRates],
pub thresholds: &'a [ThresholdRates<'a>],
pub off_peak: Option<OffPeakRates>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Request {
pub usage: Usage,
pub service_tier: ServiceTier,
pub threshold_policy: ThresholdPolicy,
pub region_multiplier: Option<f64>,
pub billed_at_utc_minute: Option<u16>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct Cost {
pub uncached_input: f64,
pub cache_read: f64,
pub cache_write_5m: f64,
pub cache_write_1h: f64,
pub output: f64,
pub multiplier: f64,
pub rates: EffectiveRates,
}
impl Cost {
pub fn input(self) -> f64 {
(self.uncached_input + self.cache_read + self.cache_write_5m + self.cache_write_1h)
* self.multiplier
}
pub fn output(self) -> f64 {
self.output * self.multiplier
}
pub fn total(self) -> f64 {
self.input() + self.output()
}
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct EffectiveRates {
pub input: f64,
pub output: f64,
pub cache_read: f64,
pub cache_write_5m: f64,
pub cache_write_1h: f64,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub enum PricingError {
MissingInputRate,
MissingOutputRate,
InvalidRate,
InvalidRegionMultiplier,
InvalidBillingTime,
InvalidOffPeakWindow,
CacheExceedsPrompt,
InvalidCacheWriteDetails,
TokenCountOverflow,
DuplicateTier,
DuplicateThreshold,
DuplicateThresholdTier,
}
fn selected_tier(tier: ServiceTier) -> ServiceTier {
if tier == ServiceTier::Fast {
ServiceTier::Priority
} else {
tier
}
}
#[derive(Clone, Debug)]
struct CompiledThreshold {
above_prompt_tokens: u64,
standard: Rates,
tiers: Vec<TierRates>,
}
#[derive(Clone, Debug)]
pub struct PricingPlan {
standard: Rates,
tiers: Vec<TierRates>,
thresholds: Vec<CompiledThreshold>,
off_peak: Option<OffPeakRates>,
}
fn valid_rates(rates: Rates) -> bool {
[
rates.input,
rates.output,
rates.cache_read,
rates.cache_write,
rates.cache_write_1h,
]
.into_iter()
.all(|rate| {
rate.value()
.is_none_or(|value| value.is_finite() && value >= 0.0)
})
}
fn validate_tiers(tiers: &[TierRates], duplicate: PricingError) -> Result<(), PricingError> {
if tiers.iter().any(|entry| !valid_rates(entry.rates)) {
return Err(PricingError::InvalidRate);
}
if tiers.iter().enumerate().any(|(index, entry)| {
matches!(
entry.tier,
ServiceTier::Standard | ServiceTier::Unknown | ServiceTier::Fast
) || tiers[..index]
.iter()
.any(|previous| previous.tier == entry.tier)
}) {
return Err(duplicate);
}
Ok(())
}
pub fn compile(pricing: &Pricing<'_>) -> Result<PricingPlan, PricingError> {
if !valid_rates(pricing.standard) {
return Err(PricingError::InvalidRate);
}
validate_tiers(pricing.tiers, PricingError::DuplicateTier)?;
if let Some(window) = pricing.off_peak {
if window.start_utc_minute >= 1440
|| window.end_utc_minute > 1440
|| window.start_utc_minute >= window.end_utc_minute
{
return Err(PricingError::InvalidOffPeakWindow);
}
if !valid_rates(window.rates) {
return Err(PricingError::InvalidRate);
}
}
let mut thresholds: Vec<_> = pricing
.thresholds
.iter()
.map(|entry| {
if !valid_rates(entry.standard) {
return Err(PricingError::InvalidRate);
}
validate_tiers(entry.tiers, PricingError::DuplicateThresholdTier)?;
Ok(CompiledThreshold {
above_prompt_tokens: entry.above_prompt_tokens,
standard: entry.standard,
tiers: entry.tiers.to_vec(),
})
})
.collect::<Result<_, _>>()?;
thresholds.sort_unstable_by_key(|entry| entry.above_prompt_tokens);
if thresholds
.windows(2)
.any(|pair| pair[0].above_prompt_tokens == pair[1].above_prompt_tokens)
{
return Err(PricingError::DuplicateThreshold);
}
Ok(PricingPlan {
standard: pricing.standard,
tiers: pricing.tiers.to_vec(),
thresholds,
off_peak: pricing.off_peak,
})
}
impl PricingPlan {
fn resolve_rates(
&self,
request: &Request,
threshold_tokens: u64,
) -> Result<Rates, PricingError> {
let tier = selected_tier(request.service_tier);
let base = self
.tiers
.iter()
.find(|entry| tier != ServiceTier::Standard && entry.tier == tier)
.map_or(self.standard, |entry| entry.rates.overlay(self.standard));
let threshold = self.thresholds.iter().rev().find(|entry| {
threshold_tokens > entry.above_prompt_tokens
|| (request.threshold_policy == ThresholdPolicy::Inclusive
&& threshold_tokens == entry.above_prompt_tokens)
});
let selected = threshold.map_or(base, |entry| {
let standard = entry.standard.overlay(base);
entry
.tiers
.iter()
.find(|specific| tier != ServiceTier::Standard && specific.tier == tier)
.map_or(standard, |specific| specific.rates.overlay(standard))
});
match self.off_peak {
None => Ok(selected),
Some(window) => {
if window.start_utc_minute >= 1440
|| window.end_utc_minute > 1440
|| window.start_utc_minute >= window.end_utc_minute
{
return Err(PricingError::InvalidOffPeakWindow);
}
let minute = request
.billed_at_utc_minute
.ok_or(PricingError::InvalidBillingTime)?;
if minute >= 1440 {
return Err(PricingError::InvalidBillingTime);
}
if (window.start_utc_minute..window.end_utc_minute).contains(&minute) {
Ok(window.rates.overlay(selected))
} else {
Ok(selected)
}
}
}
}
fn checked_rate(rate: Rate, missing: PricingError) -> Result<f64, PricingError> {
let value = rate.value().ok_or(missing)?;
if !value.is_finite() || value < 0.0 {
return Err(PricingError::InvalidRate);
}
Ok(value)
}
pub fn calculate(&self, request: &Request) -> Result<Cost, PricingError> {
let usage = request.usage;
let cached = usage
.cache_read_tokens
.checked_add(usage.cache_write_tokens)
.ok_or(PricingError::TokenCountOverflow)?;
let (regular, threshold_tokens) = match usage.prompt_convention {
PromptConvention::IncludesCache => (
usage
.prompt_tokens
.checked_sub(cached)
.ok_or(PricingError::CacheExceedsPrompt)?,
usage.prompt_tokens,
),
PromptConvention::ExcludesCache => (
usage.prompt_tokens,
usage
.prompt_tokens
.checked_add(cached)
.ok_or(PricingError::TokenCountOverflow)?,
),
};
let writes = match (usage.cache_write_5m_tokens, usage.cache_write_1h_tokens) {
(None, None) => (usage.cache_write_tokens, 0),
(Some(five), Some(one)) if five.checked_add(one) == Some(usage.cache_write_tokens) => {
(five, one)
}
_ => return Err(PricingError::InvalidCacheWriteDetails),
};
let rates = self.resolve_rates(request, threshold_tokens)?;
let input = Self::checked_rate(rates.input, PricingError::MissingInputRate)?;
let output = Self::checked_rate(rates.output, PricingError::MissingOutputRate)?;
let read = Self::checked_rate(
rates.cache_read.or(rates.input),
PricingError::MissingInputRate,
)?;
let write = Self::checked_rate(
rates.cache_write.or(rates.input),
PricingError::MissingInputRate,
)?;
let write_1h = Self::checked_rate(
rates.cache_write_1h.or(rates.cache_write).or(rates.input),
PricingError::MissingInputRate,
)?;
let multiplier = request.region_multiplier.unwrap_or(1.0);
if !multiplier.is_finite() || multiplier <= 0.0 {
return Err(PricingError::InvalidRegionMultiplier);
}
let cost = Cost {
uncached_input: regular as f64 * input,
cache_read: usage.cache_read_tokens as f64 * read,
cache_write_5m: writes.0 as f64 * write,
cache_write_1h: writes.1 as f64 * write_1h,
output: usage.completion_tokens as f64 * output,
multiplier,
rates: EffectiveRates {
input,
output,
cache_read: read,
cache_write_5m: write,
cache_write_1h: write_1h,
},
};
if !cost.total().is_finite() {
return Err(PricingError::TokenCountOverflow);
}
Ok(cost)
}
}
pub fn calculate(pricing: &Pricing<'_>, request: &Request) -> Result<Cost, PricingError> {
compile(pricing)?.calculate(request)
}

View file

@ -0,0 +1,458 @@
use litellm_cost::{
OffPeakRates, Pricing, PricingError, PromptConvention, Rate, Rates, Request, ServiceTier,
ThresholdPolicy, ThresholdRates, TierRates, Usage, calculate, compile,
};
fn rates(input: Rate, output: Rate) -> Rates {
Rates {
input,
output,
..Rates::EMPTY
}
}
fn request() -> Request {
Request {
usage: Usage {
prompt_tokens: 100,
completion_tokens: 20,
cache_read_tokens: 25,
cache_write_tokens: 10,
cache_write_5m_tokens: None,
cache_write_1h_tokens: None,
prompt_convention: PromptConvention::IncludesCache,
},
service_tier: ServiceTier::Standard,
threshold_policy: ThresholdPolicy::Exclusive,
region_multiplier: None,
billed_at_utc_minute: None,
}
}
fn pricing(standard: Rates) -> Pricing<'static> {
Pricing {
standard,
tiers: &[],
thresholds: &[],
off_peak: None,
}
}
#[test]
fn breakdown_and_total_agree() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let result = calculate(&pricing(standard), &request()).unwrap();
assert_eq!(result.uncached_input, 65.0 * 2.0);
assert_eq!(result.cache_read, 25.0 * 0.5);
assert_eq!(result.cache_write_5m, 10.0 * 3.0);
assert_eq!(result.output(), 20.0 * 4.0);
assert_eq!(result.total(), result.input() + result.output());
assert_eq!(result.rates.cache_read, 0.5);
}
#[test]
fn absent_null_and_zero_cache_rates_are_distinct() {
let base = rates(Rate::Value(2.0), Rate::Value(4.0));
for read in [Rate::Missing, Rate::Null] {
let standard = Rates {
cache_read: read,
..base
};
assert_eq!(
calculate(&pricing(standard), &request()).unwrap().input(),
200.0
);
}
let standard = Rates {
cache_read: Rate::Value(0.0),
cache_write: Rate::Value(0.0),
..base
};
assert_eq!(
calculate(&pricing(standard), &request()).unwrap().input(),
130.0
);
}
#[test]
fn equivalent_prompt_conventions_select_the_same_threshold() {
let threshold = ThresholdRates {
above_prompt_tokens: 90,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[threshold],
off_peak: None,
};
let included = request();
let excluded = Request {
usage: Usage {
prompt_tokens: 65,
prompt_convention: PromptConvention::ExcludesCache,
..included.usage
},
..included
};
let plan = compile(&specification).unwrap();
assert_eq!(plan.calculate(&included), plan.calculate(&excluded));
assert_eq!(plan.calculate(&included).unwrap().rates.input, 5.0);
}
#[test]
fn split_writes_and_invalid_accounting() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
cache_write_1h: Rate::Value(5.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let base = request();
let split = Request {
usage: Usage {
cache_write_5m_tokens: Some(4),
cache_write_1h_tokens: Some(6),
..base.usage
},
..base
};
let result = calculate(&pricing(standard), &split).unwrap();
assert_eq!(result.cache_write_5m, 12.0);
assert_eq!(result.cache_write_1h, 30.0);
let overlapping = Request {
usage: Usage {
prompt_tokens: 30,
..split.usage
},
..split
};
assert_eq!(
calculate(&pricing(standard), &overlapping),
Err(PricingError::CacheExceedsPrompt)
);
let incomplete = Request {
usage: Usage {
cache_write_1h_tokens: None,
..split.usage
},
..split
};
assert_eq!(
calculate(&pricing(standard), &incomplete),
Err(PricingError::InvalidCacheWriteDetails)
);
}
#[test]
fn threshold_tiers_and_boundaries() {
let priority = TierRates {
tier: ServiceTier::Priority,
rates: rates(Rate::Value(3.0), Rate::Missing),
};
let threshold = ThresholdRates {
above_prompt_tokens: 100,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[
TierRates {
tier: ServiceTier::Priority,
rates: rates(Rate::Value(7.0), Rate::Missing),
},
TierRates {
tier: ServiceTier::Flex,
rates: rates(Rate::Value(6.0), Rate::Missing),
},
],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[priority],
thresholds: &[threshold],
off_peak: None,
};
let base = request();
let no_cache = Request {
usage: Usage {
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
let fast = Request {
service_tier: ServiceTier::Fast,
..no_cache
};
let inclusive = Request {
threshold_policy: ThresholdPolicy::Inclusive,
..fast
};
let flex = Request {
service_tier: ServiceTier::Flex,
..inclusive
};
assert_eq!(calculate(&specification, &no_cache).unwrap().input(), 200.0);
assert_eq!(calculate(&specification, &fast).unwrap().input(), 300.0);
assert_eq!(
calculate(&specification, &inclusive).unwrap().input(),
700.0
);
assert_eq!(calculate(&specification, &flex).unwrap().input(), 600.0);
}
#[test]
fn compile_rejects_ambiguous_rates() {
let duplicate = ThresholdRates {
above_prompt_tokens: 100,
standard: Rates::EMPTY,
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(1.0), Rate::Value(1.0)),
tiers: &[],
thresholds: &[duplicate, duplicate],
off_peak: None,
};
assert_eq!(
compile(&specification).err(),
Some(PricingError::DuplicateThreshold)
);
let invalid = pricing(rates(Rate::Value(f64::NAN), Rate::Value(1.0)));
assert_eq!(compile(&invalid).err(), Some(PricingError::InvalidRate));
}
#[test]
fn off_peak_is_one_non_wrapping_utc_window() {
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[],
off_peak: Some(OffPeakRates {
start_utc_minute: 60,
end_utc_minute: 120,
rates: rates(Rate::Value(1.0), Rate::Value(2.0)),
}),
};
let base = request();
let start = Request {
billed_at_utc_minute: Some(60),
..base
};
let end = Request {
billed_at_utc_minute: Some(120),
..base
};
assert_eq!(
calculate(&specification, &base),
Err(PricingError::InvalidBillingTime)
);
assert_eq!(calculate(&specification, &start).unwrap().input(), 100.0);
assert_eq!(calculate(&specification, &end).unwrap().input(), 200.0);
}
#[test]
fn missing_rates_and_free_rates_remain_distinct() {
let base = request();
let empty = Request {
usage: Usage {
prompt_tokens: 0,
completion_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
assert_eq!(
calculate(&pricing(Rates::EMPTY), &empty),
Err(PricingError::MissingInputRate)
);
assert_eq!(
calculate(&pricing(rates(Rate::Value(0.0), Rate::Missing)), &empty),
Err(PricingError::MissingOutputRate)
);
assert_eq!(
calculate(&pricing(rates(Rate::Value(0.0), Rate::Value(0.0))), &empty)
.unwrap()
.total(),
0.0
);
}
#[test]
fn matches_executed_python_reference_cases() {
for row in include_str!("python_reference.tsv")
.lines()
.filter(|line| !line.starts_with('#'))
{
let fields: Vec<_> = row.split('\t').collect();
let count = |index: usize| fields[index].parse::<u64>().unwrap();
let number = |index: usize| fields[index].parse::<f64>().unwrap();
let optional_rate = |index: usize| {
if fields[index].is_empty() {
Rate::Missing
} else {
Rate::Value(number(index))
}
};
let threshold = ThresholdRates {
above_prompt_tokens: if fields[9].is_empty() { 0 } else { count(9) },
standard: rates(optional_rate(10), optional_rate(11)),
tiers: &[],
};
let thresholds = if fields[9].is_empty() {
&[][..]
} else {
std::slice::from_ref(&threshold)
};
let specification = Pricing {
standard: Rates {
cache_read: optional_rate(7),
cache_write: optional_rate(8),
..rates(Rate::Value(number(5)), Rate::Value(number(6)))
},
tiers: &[],
thresholds,
off_peak: None,
};
let base = request();
let input = Request {
usage: Usage {
prompt_tokens: count(1),
completion_tokens: count(2),
cache_read_tokens: count(3),
cache_write_tokens: count(4),
..base.usage
},
..base
};
let actual = calculate(&specification, &input).unwrap();
assert_eq!(actual.input(), number(12), "{}", fields[0]);
assert_eq!(actual.output(), number(13), "{}", fields[0]);
}
}
proptest::proptest! {
#[test]
fn equivalent_usage_conventions_and_breakdown_agree(
regular in 0_u64..1000,
read in 0_u64..1000,
write in 0_u64..1000,
output in 0_u64..1000,
) {
let threshold = ThresholdRates {
above_prompt_tokens: 1000,
standard: rates(Rate::Value(5.0), Rate::Value(8.0)),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[threshold],
off_peak: None,
};
let base = request();
let included = Request {
usage: Usage {
prompt_tokens: regular + read + write,
completion_tokens: output,
cache_read_tokens: read,
cache_write_tokens: write,
..base.usage
},
..base
};
let excluded = Request {
usage: Usage {
prompt_tokens: regular,
prompt_convention: PromptConvention::ExcludesCache,
..included.usage
},
..included
};
let plan = compile(&specification).unwrap();
let left = plan.calculate(&included).unwrap();
let right = plan.calculate(&excluded).unwrap();
proptest::prop_assert_eq!(left, right);
proptest::prop_assert_eq!(left.total(), left.input() + left.output());
}
}
#[test]
fn regional_multiplier_applies_after_input_components_are_summed() {
let standard = Rates {
cache_read: Rate::Value(0.5),
cache_write: Rate::Value(3.0),
..rates(Rate::Value(2.0), Rate::Value(4.0))
};
let base = request();
let regional = Request {
region_multiplier: Some(1.1),
..base
};
let result = calculate(&pricing(standard), &regional).unwrap();
assert_eq!(result.input(), (65.0 * 2.0 + 25.0 * 0.5 + 10.0 * 3.0) * 1.1);
assert_eq!(result.output(), 20.0 * 4.0 * 1.1);
let invalid = Request {
region_multiplier: Some(f64::NAN),
..base
};
assert_eq!(
calculate(&pricing(standard), &invalid),
Err(PricingError::InvalidRegionMultiplier)
);
}
#[test]
fn compilation_sorts_thresholds_and_rejects_duplicate_tiers() {
let high = ThresholdRates {
above_prompt_tokens: 200,
standard: rates(Rate::Value(7.0), Rate::Missing),
tiers: &[],
};
let low = ThresholdRates {
above_prompt_tokens: 100,
standard: rates(Rate::Value(5.0), Rate::Missing),
tiers: &[],
};
let specification = Pricing {
standard: rates(Rate::Value(2.0), Rate::Value(4.0)),
tiers: &[],
thresholds: &[high, low],
off_peak: None,
};
let base = request();
let above_both = Request {
usage: Usage {
prompt_tokens: 201,
cache_read_tokens: 0,
cache_write_tokens: 0,
..base.usage
},
..base
};
assert_eq!(
compile(&specification)
.unwrap()
.calculate(&above_both)
.unwrap()
.rates
.input,
7.0
);
let duplicate = TierRates {
tier: ServiceTier::Flex,
rates: Rates::EMPTY,
};
let invalid = Pricing {
tiers: &[duplicate, duplicate],
thresholds: &[],
..specification
};
assert_eq!(compile(&invalid).err(), Some(PricingError::DuplicateTier));
}

View file

@ -0,0 +1,96 @@
import subprocess
from dataclasses import dataclass
from pathlib import Path
from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token
from litellm.types.utils import Usage
@dataclass(frozen=True, slots=True)
class Case:
name: str
prompt: int
completion: int
cache_read: int
cache_write: int
input_rate: float
output_rate: float
cache_read_rate: float | None = None
cache_write_rate: float | None = None
threshold: int | None = None
threshold_input_rate: float | None = None
threshold_output_rate: float | None = None
CASES = (
Case("ordinary", 100, 20, 0, 0, 2.0, 4.0),
Case("cache_fallback", 100, 20, 25, 10, 2.0, 4.0),
Case("cache_specific", 100, 20, 25, 10, 2.0, 4.0, 0.5, 3.0),
Case("free_cache", 100, 20, 25, 10, 2.0, 4.0, 0.0, 0.0),
Case("threshold_below", 99, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
Case("threshold_at", 100, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0),
Case(
"threshold_above", 101, 20, 0, 0, 2.0, 4.0, threshold=100, threshold_input_rate=5.0, threshold_output_rate=8.0
),
Case(
"cache_threshold_above",
101,
20,
25,
10,
2.0,
4.0,
threshold=100,
threshold_input_rate=5.0,
threshold_output_rate=8.0,
),
)
def reference(case: Case) -> tuple[float, float]:
info = {"input_cost_per_token": case.input_rate, "output_cost_per_token": case.output_rate}
if case.cache_read_rate is not None:
info["cache_read_input_token_cost"] = case.cache_read_rate
if case.cache_write_rate is not None:
info["cache_creation_input_token_cost"] = case.cache_write_rate
if case.threshold is not None:
info[f"input_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_input_rate
info[f"output_cost_per_token_above_{case.threshold}_tokens"] = case.threshold_output_rate
details = {"cached_tokens": case.cache_read, "cache_write_tokens": case.cache_write}
usage = Usage(prompt_tokens=case.prompt, completion_tokens=case.completion, prompt_tokens_details=details)
return generic_cost_per_token(
model="synthetic",
usage=usage,
custom_llm_provider="openai",
model_info=info,
)
def main() -> None:
revision = subprocess.check_output(("git", "rev-parse", "HEAD"), text=True).strip()
rows = ("# Python reference commit: " + revision,) + tuple(
"\t".join(
str(value) if value is not None else ""
for value in (
case.name,
case.prompt,
case.completion,
case.cache_read,
case.cache_write,
case.input_rate,
case.output_rate,
case.cache_read_rate,
case.cache_write_rate,
case.threshold,
case.threshold_input_rate,
case.threshold_output_rate,
*reference(case),
)
)
for case in CASES
)
Path(__file__).with_name("python_reference.tsv").write_text("\n".join(rows) + "\n")
if __name__ == "__main__":
main()

View file

@ -0,0 +1,9 @@
# Python reference commit: dc4be2fd987c993aefcf16444e34f960c12c8627
ordinary 100 20 0 0 2.0 4.0 200.0 80.0
cache_fallback 100 20 25 10 2.0 4.0 200.0 80.0
cache_specific 100 20 25 10 2.0 4.0 0.5 3.0 172.5 80.0
free_cache 100 20 25 10 2.0 4.0 0.0 0.0 130.0 80.0
threshold_below 99 20 0 0 2.0 4.0 100 5.0 8.0 198.0 80.0
threshold_at 100 20 0 0 2.0 4.0 100 5.0 8.0 200.0 80.0
threshold_above 101 20 0 0 2.0 4.0 100 5.0 8.0 505.0 160.0
cache_threshold_above 101 20 25 10 2.0 4.0 100 5.0 8.0 505.0 160.0
Can't render this file because it has a wrong number of fields in line 2.

View file

@ -0,0 +1,25 @@
[package]
name = "litellm-model-catalog"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[features]
schema = ["dep:schemars"]
[dependencies]
indexmap = { version = "2.14.0", features = ["serde"] }
schemars = { version = "1.0", optional = true }
serde.workspace = true
serde_json.workspace = true
thiserror.workspace = true
[dev-dependencies]
criterion.workspace = true
rstest.workspace = true
litellm-model-catalog = { path = ".", features = ["schema"] }
[[bench]]
name = "catalog"
harness = false

View file

@ -0,0 +1,25 @@
# Model catalog
`litellm-model-catalog` builds an immutable snapshot from caller supplied JSON bytes. It has no network, Python, registration, or refresh behavior. The caller supplies optional source, revision, and ETag provenance. Parse and validation are separate so small synthetic catalogs can use explicit integrity limits
The parser treats `sample_spec` and `fallback_generalizations` as reserved top level metadata. `fallback_rules()` exposes the typed rule array when present; this crate does not execute regex generalizations. Model entries retain all JSON fields except `aliases`, including unknown fields. `field()` returns `None` for an absent key and a JSON null, false, or zero value for a present key. The returned values are borrowed, so callers cannot mutate the snapshot
Each entry also deserializes into `ModelInfo`, a typed mirror of `model_prices_and_context_window.schema.json`'s `modelEntry` definition, reachable via `ModelEntry::info()`. All schema fields are optional on `ModelInfo`, including `litellm_provider` which the schema marks required, so small synthetic catalogs still parse. Unknown fields are not part of `ModelInfo`; they remain on `fields()`. Building with the `schema` feature adds `schemars` derives and exposes `model_entry_json_schema()` for emitting the entry's JSON Schema. Parse and validation failures are reported by the `Error` enum in `error.rs`, while catalog logic lives in `catalog.rs`
The integration tests read the repository's catalog and schema files at test time, assert every entry round-trips through `ModelInfo`, and verify that the generated schema's properties match the repository schema
Aliases point to their canonical entries. An alias that exactly matches any canonical key is skipped; the first canonical entry claiming an alias wins. Invalid alias lists and nonstring names are skipped and reported by `alias_issues()`. Exact lookup wins. For a case insensitive miss, the last key with the same lowercase spelling wins, following Python's lowercase map built after aliases are appended. This uses Rust Unicode lowercasing, which can differ from Python for unusual Unicode model IDs
`validate()` counts canonical entries before alias expansion and excludes both reserved keys. It enforces an explicit minimum and backup shrink ratio, with Python defaults of 50 models and 0.5. Parsing rejects nonobject model entries and known fields with the wrong JSON type, but ignores unknown fields. It does not enforce every constraint in the JSON schema, calculate prices, resolve providers, or check provenance authenticity. The caller decides how to handle validation failures
This snapshot does not represent Python's live mutable `litellm.model_cost`, nested dict and list mutation, or mutation of dicts previously returned by Python APIs. It has no bridge or runtime integration
## Benchmarks
`cargo bench -p litellm-model-catalog --bench catalog` measures parsing plus alias indexing and exact lookup. For a local Python baseline on the same fixture, use:
```sh
python3 -m timeit -s 'import json, pathlib; body = pathlib.Path("../model_prices_and_context_window.json").read_bytes()' 'json.loads(body)'
```
Run these commands from `litellm-rust`. Python's command measures JSON loading only, without alias expansion or snapshot construction. The Rust benchmark does not include future Python object materialization, so these numbers are not an end to end runtime comparison

View file

@ -0,0 +1,21 @@
use criterion::{Criterion, criterion_group, criterion_main};
use litellm_model_catalog::{Catalog, Provenance};
use std::hint::black_box;
fn benchmarks(c: &mut Criterion) {
let body = include_bytes!("../../../../model_prices_and_context_window.json");
c.bench_function("parse_current_catalog", |b| {
b.iter(|| Catalog::parse(black_box(body), Provenance::default()).unwrap())
});
let catalog = Catalog::parse(body, Provenance::default()).unwrap();
let key = catalog
.model_names()
.next()
.expect("catalog must have a benchmark key");
c.bench_function("lookup_catalog_key", |b| {
b.iter(|| black_box(&catalog).lookup(black_box(key)))
});
}
criterion_group!(benches, benchmarks);
criterion_main!(benches);

View file

@ -0,0 +1,241 @@
use crate::error::Error;
use crate::model_info::{FallbackGeneralizations, FallbackRule, ModelInfo};
use indexmap::IndexMap;
use serde::Deserialize;
use serde_json::{Map, Value};
use std::collections::HashMap;
#[derive(Clone, Debug, Default, PartialEq, Eq)]
pub struct Provenance {
pub source: Option<String>,
pub revision: Option<String>,
pub etag: Option<String>,
}
#[derive(Clone, Copy, Debug, PartialEq)]
pub struct IntegrityLimits {
pub backup_model_count: usize,
pub min_model_count: usize,
pub min_backup_ratio: f64,
}
impl IntegrityLimits {
pub fn python_defaults(backup_model_count: usize) -> Self {
Self {
backup_model_count,
min_model_count: 50,
min_backup_ratio: 0.5,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum AliasIssue {
InvalidList { model: String },
InvalidName { model: String },
CanonicalCollision { model: String, alias: String },
AliasCollision { model: String, alias: String },
}
#[derive(Clone, Debug)]
pub struct ModelEntry {
fields: Map<String, Value>,
info: ModelInfo,
}
impl ModelEntry {
pub fn field(&self, name: &str) -> Option<&Value> {
self.fields.get(name)
}
pub fn fields(&self) -> &Map<String, Value> {
&self.fields
}
/// The entry deserialized into the typed mirror of the catalog schema.
pub fn info(&self) -> &ModelInfo {
&self.info
}
}
#[derive(Clone, Copy, Debug)]
pub struct ModelMatch<'a> {
pub matched_key: &'a str,
pub canonical_key: &'a str,
pub entry: &'a ModelEntry,
}
#[derive(Debug)]
pub struct Catalog {
entries: IndexMap<String, ModelEntry>,
aliases: IndexMap<String, String>,
lowercase_keys: HashMap<String, String>,
sample_spec: Option<Value>,
fallback_generalizations: Option<FallbackGeneralizations>,
provenance: Provenance,
alias_issues: Vec<AliasIssue>,
}
impl Catalog {
pub fn parse(body: &[u8], provenance: Provenance) -> Result<Self, Error> {
let root: IndexMap<String, Value> = serde_json::from_slice(body)?;
if root.is_empty() {
return Err(Error::Empty);
}
let mut entries = IndexMap::with_capacity(root.len());
let mut alias_lists = Vec::new();
let mut alias_issues = Vec::new();
let mut sample_spec = None;
let mut fallback_generalizations = None;
for (name, value) in root {
match name.as_str() {
"sample_spec" => {
sample_spec = Some(value);
continue;
}
"fallback_generalizations" => {
fallback_generalizations =
Some(serde_json::from_value::<FallbackGeneralizations>(value)?);
continue;
}
_ => {}
}
let Value::Object(ref object) = value else {
return Err(Error::EntryNotObject { model: name });
};
let info = ModelInfo::deserialize(object)?;
let Value::Object(mut fields) = value else {
unreachable!("value checked is_object above")
};
if let Some(aliases) = fields.remove("aliases")
&& !aliases.is_null()
{
match aliases {
Value::Array(names) => alias_lists.push((name.clone(), names)),
_ => alias_issues.push(AliasIssue::InvalidList {
model: name.clone(),
}),
}
}
entries.insert(name, ModelEntry { fields, info });
}
let mut aliases = IndexMap::new();
for (model, names) in alias_lists {
for name in names {
let Value::String(alias) = name else {
alias_issues.push(AliasIssue::InvalidName {
model: model.clone(),
});
continue;
};
if entries.contains_key(&alias) {
alias_issues.push(AliasIssue::CanonicalCollision {
model: model.clone(),
alias,
});
} else if aliases.contains_key(&alias) {
alias_issues.push(AliasIssue::AliasCollision {
model: model.clone(),
alias,
});
} else {
aliases.insert(alias, model.clone());
}
}
}
let lowercase_keys = entries
.keys()
.chain(aliases.keys())
.map(|key| (key.to_lowercase(), key.clone()))
.collect();
Ok(Self {
entries,
aliases,
lowercase_keys,
sample_spec,
fallback_generalizations,
provenance,
alias_issues,
})
}
pub fn validate(&self, limits: IntegrityLimits) -> Result<(), Error> {
if !limits.min_backup_ratio.is_finite() || !(0.0..=1.0).contains(&limits.min_backup_ratio) {
return Err(Error::InvalidRatio);
}
let actual = self.entries.len();
if actual < limits.min_model_count {
return Err(Error::BelowMinimum {
actual,
minimum: limits.min_model_count,
});
}
if limits.backup_model_count > 0
&& (actual as f64) < (limits.backup_model_count as f64) * limits.min_backup_ratio
{
return Err(Error::Shrunk {
actual,
backup: limits.backup_model_count,
ratio: limits.min_backup_ratio,
});
}
Ok(())
}
pub fn lookup(&self, key: &str) -> Option<ModelMatch<'_>> {
let matched_key = if self.entries.contains_key(key) || self.aliases.contains_key(key) {
key
} else {
self.lowercase_keys.get(&key.to_lowercase())?.as_str()
};
let canonical_key = self
.aliases
.get(matched_key)
.map(String::as_str)
.unwrap_or(matched_key);
let (canonical_key, entry) = self.entries.get_key_value(canonical_key)?;
let matched_key = self
.entries
.get_key_value(matched_key)
.map(|(key, _)| key.as_str())
.or_else(|| {
self.aliases
.get_key_value(matched_key)
.map(|(key, _)| key.as_str())
})?;
Some(ModelMatch {
matched_key,
canonical_key,
entry,
})
}
pub fn model_count(&self) -> usize {
self.entries.len()
}
pub fn model_names(&self) -> impl Iterator<Item = &str> {
self.entries.keys().map(String::as_str)
}
pub fn alias_count(&self) -> usize {
self.aliases.len()
}
pub fn aliases(&self) -> &IndexMap<String, String> {
&self.aliases
}
pub fn alias_issues(&self) -> &[AliasIssue] {
&self.alias_issues
}
pub fn sample_spec(&self) -> Option<&Value> {
self.sample_spec.as_ref()
}
pub fn fallback_generalizations(&self) -> Option<&FallbackGeneralizations> {
self.fallback_generalizations.as_ref()
}
pub fn fallback_rules(&self) -> Option<&[FallbackRule]> {
Some(self.fallback_generalizations.as_ref()?.rules.as_slice())
}
pub fn provenance(&self) -> &Provenance {
&self.provenance
}
}

View file

@ -0,0 +1,28 @@
use thiserror::Error;
/// Failures from parsing or validating a catalog snapshot.
#[derive(Debug, Error)]
pub enum Error {
/// The body is not valid JSON, or a model entry fails typed deserialization.
#[error("invalid JSON: {0}")]
Json(#[from] serde_json::Error),
/// The catalog has no entries at all.
#[error("catalog is empty")]
Empty,
/// A non-reserved top level value is not a JSON object.
#[error("model {model:?} must be an object")]
EntryNotObject { model: String },
/// Canonical entry count is under the configured minimum.
#[error("catalog has {actual} models, below minimum {minimum}")]
BelowMinimum { actual: usize, minimum: usize },
/// Canonical entry count is under the configured backup shrink ratio.
#[error("catalog has {actual} models, below {ratio} of backup count {backup}")]
Shrunk {
actual: usize,
backup: usize,
ratio: f64,
},
/// The configured minimum backup ratio is not finite or outside `[0, 1]`.
#[error("minimum backup ratio must be finite and between zero and one")]
InvalidRatio,
}

View file

@ -0,0 +1,16 @@
mod catalog;
mod error;
mod model_info;
#[cfg(feature = "schema")]
mod schema;
pub use catalog::{AliasIssue, Catalog, IntegrityLimits, ModelEntry, ModelMatch, Provenance};
pub use error::Error;
pub use model_info::{
AudioFormat, FallbackGeneralizations, FallbackRule, InputModality, Mode, ModelInfo,
OffPeakPricing, OffPeakWindow, OutputModality, ReasoningEffort, SearchContextCostPerQuery,
TieredRate, UtcHours, VertexAiAudioApi, WebSearchBillingUnit, Weekday,
};
#[cfg(feature = "schema")]
pub use schema::model_entry_json_schema;

View file

@ -0,0 +1,665 @@
use serde::{Deserialize, Serialize};
use serde_json::Value;
use std::collections::BTreeMap;
/// Primary API surface / task type of the model.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum Mode {
AudioSpeech,
AudioTranscription,
Chat,
Completion,
Embedding,
Evaluation,
Guardrail,
ImageEdit,
ImageGeneration,
Moderation,
Ocr,
Realtime,
Rerank,
Responses,
Search,
VectorStore,
VideoGeneration,
}
/// Reasoning effort level accepted or applied by the model.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum ReasoningEffort {
None,
Minimal,
Low,
Medium,
High,
Xhigh,
Max,
}
/// Gemini audio generation API the model is served through.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum VertexAiAudioApi {
LyriaPredict,
LyriaInteractions,
}
/// Whether web search is billed per query or per prompt.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum WebSearchBillingUnit {
PerQuery,
PerPrompt,
}
/// Audio container format the model can return.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum AudioFormat {
Mp3,
Wav,
}
/// Input modality the model accepts.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum InputModality {
Text,
Image,
Audio,
Video,
}
/// Output modality the model can produce.
#[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(rename_all = "snake_case")]
pub enum OutputModality {
Text,
Image,
Audio,
Video,
Code,
}
/// UTC "HH:MM-HH:MM" window, or a list of them; a window may wrap past midnight.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum UtcHours {
Single(String),
Multiple(Vec<String>),
}
/// ISO-8601 weekday number (1 = Monday .. 7 = Sunday) or English day name.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(untagged)]
pub enum Weekday {
Number(u8),
Name(String),
}
/// One off-peak window entry inside `windows`.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct OffPeakWindow {
pub hours_utc: UtcHours,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub weekdays: Option<Vec<Weekday>>,
}
/// Rates that replace the same-named base fields inside the stated UTC windows.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct OffPeakPricing {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub hours_utc: Option<UtcHours>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub windows: Option<Vec<OffPeakWindow>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub weekday_timezone: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
}
/// USD cost per web search query, keyed by search context size.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct SearchContextCostPerQuery {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub search_context_size_low: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub search_context_size_medium: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub search_context_size_high: Option<f64>,
}
/// One tier of a context-length or result-count tiered rate.
#[derive(Clone, Debug, Default, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct TieredRate {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub range: Option<[f64; 2]>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_results_range: Option<[f64; 2]>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_query: Option<f64>,
}
/// One regex rule generalizing unknown model ids to known families.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub struct FallbackRule {
pub name: String,
pub pattern: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub description: Option<String>,
#[serde(flatten)]
pub extra: BTreeMap<String, Value>,
}
/// Regex rules that generalize unknown model ids to known families; not a model entry.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
#[serde(deny_unknown_fields)]
pub struct FallbackGeneralizations {
pub rules: Vec<FallbackRule>,
}
/// Typed mirror of one catalog model entry.
#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))]
pub struct ModelInfo {
#[serde(default, skip_serializing_if = "Option::is_none")]
pub annotation_cost_per_page: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub annotation_cost_per_page_batches: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub audio_transcription_config: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bedrock_converse_supports_strict_tools: Option<bool>,
/// Highest reasoning effort the Bedrock output_config accepts for this model.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub bedrock_output_config_effort_ceiling: Option<ReasoningEffort>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_audio_token_cost: Option<f64>,
/// USD per token written to the provider's prompt cache.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_1hr_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_200k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_above_272k_tokens_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_creation_input_token_cost_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_audio_token_cost: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_image_token_cost: Option<f64>,
/// USD per prompt token served from the provider's prompt cache.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_200k_tokens: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_272k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_above_512k_tokens: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub cache_read_input_token_cost_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub citation_cost_per_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub code_interpreter_cost_per_session: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub comment: Option<String>,
/// Reasoning effort the provider applies when the request omits reasoning_effort. Gates whether a non-default temperature or the top_p/logprobs sampling params are accepted, which hold only when the effort resolves to 'none'.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub default_reasoning_effort: Option<ReasoningEffort>,
/// Date the provider deprecates the model, YYYY-MM-DD.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub deprecation_date: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gemini_audio_only_live: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub gemini_native_audio: Option<bool>,
/// USD per Grounding with Google Maps request; billed per query or per prompt per web_search_billing_unit.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub google_maps_grounding_cost_per_query: Option<f64>,
/// USD cost per billable guardrail unit, keyed by the provider's usage counter name (e.g. Bedrock's contentPolicyUnits).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub guardrail_cost_per_unit: Option<BTreeMap<String, f64>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_per_second: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_per_second_above_128k_tokens: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token_batches: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_audio_token_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_character: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_character_above_128k_tokens: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_image: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_above_128k_tokens: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_image_token_batches: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_pixel: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_query: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_request: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_second: Option<f64>,
/// USD per prompt token.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_272k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_above_512k_tokens: Option<f64>,
/// USD per prompt token via the provider's batch API.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_batches: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_cache_hit: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_token_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_15s_interval: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_per_second_above_8s_interval: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_cost_per_video_token_batches: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub input_dbu_cost_per_token: Option<f64>,
/// LiteLLM provider slug; one of https://docs.litellm.ai/docs/providers.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub litellm_provider: Option<String>,
/// Maximum prompt/context tokens the model accepts.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_input_tokens: Option<u64>,
/// Maximum tokens the model can generate in one response.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_output_tokens: Option<u64>,
/// Legacy field: max output tokens if the provider specifies it, else max input tokens.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_tokens: Option<u64>,
/// Free-form notes about the entry (e.g. pricing derivation).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub metadata: Option<BTreeMap<String, Value>>,
/// Primary API surface / task type of the model.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub mode: Option<Mode>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_credit: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_page: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub ocr_cost_per_page_batches: Option<f64>,
/// Rates that replace the same-named base fields while the request falls inside the stated UTC windows.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub off_peak_pricing: Option<OffPeakPricing>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_audio_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_character: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_character_above_128k_tokens: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_image: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_1024: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_1536: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_512: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_image_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_pixel: Option<f64>,
/// USD per reasoning/thinking token, when billed separately.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_reasoning_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_1080p: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_2k: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_480p: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_4k: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_720p: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_second_768p: Option<f64>,
/// USD per generated token.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_128k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_200k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_256k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_272k_tokens_priority: Option<f64>,
/// Rate applied once the prompt exceeds the token threshold in the field name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_above_512k_tokens: Option<f64>,
/// USD per generated token via the provider's batch API.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_batches: Option<f64>,
/// Flex service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_flex: Option<f64>,
/// Priority service-tier rate for the same-named base field.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_token_priority: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_video_per_second: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_cost_per_video_token: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_dbu_cost_per_token: Option<f64>,
/// Embedding dimension for embedding models.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub output_vector_size: Option<u64>,
/// Smallest prefix the provider will actually cache; absent means the provider default applies.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub prompt_cache_min_tokens: Option<u64>,
/// Provider-internal routing hints (e.g. bedrock_invocation_schema).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub provider_specific_entry: Option<BTreeMap<String, Value>>,
/// Exact reasoning_effort levels this deployment accepts; wins over supports_* flags.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub reasoning_effort_levels: Option<Vec<ReasoningEffort>>,
/// Multiplier applied to all token costs when served from a non-global Vertex AI endpoint (e.g. 1.10 = +10%).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub regional_endpoint_uplift_multiplier: Option<f64>,
/// Multiplier applied to all token costs for EU data residency (e.g. 1.10 = +10%).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub regional_processing_uplift_multiplier_eu: Option<f64>,
/// Multiplier applied to all token costs for US data residency (e.g. 1.10 = +10%).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub regional_processing_uplift_multiplier_us: Option<f64>,
/// Provider default requests-per-minute limit.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub rpm: Option<u64>,
/// USD cost per web search query, keyed by search context size.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub search_context_cost_per_query: Option<SearchContextCostPerQuery>,
/// URL of the provider pricing/model page this entry was taken from.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub source: Option<String>,
/// Audio container formats the model can return.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supported_audio_formats: Option<Vec<AudioFormat>>,
/// OpenAI-style API routes this model can be called through, e.g. /v1/chat/completions.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supported_endpoints: Option<Vec<String>>,
/// Input modalities the model accepts.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supported_modalities: Option<Vec<InputModality>>,
/// Output modalities the model can produce.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supported_output_modalities: Option<Vec<OutputModality>>,
/// Cloud regions the model is available in ('global' or region ids).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supported_regions: Option<Vec<String>>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_adaptive_thinking: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_anthropic_compaction: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_anthropic_thinking_payload: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_assistant_prefill: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_audio_input: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_audio_output: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_computer_use: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_embedding_image_input: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_fast_mode: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_forced_tool_use: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_function_calling: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_image_input: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_image_size: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_legacy_thinking: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_low_reasoning_effort: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_max_reasoning_effort: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_mid_conversation_system: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_minimal_reasoning_effort: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_multimodal: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_native_streaming: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_native_structured_output: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_none_reasoning_effort: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_nova_canvas_image_edit: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_output_config: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_parallel_function_calling: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_parallel_tool_use_config: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_pdf_input: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_prompt_cache_breakpoint: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_prompt_caching: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_reasoning: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_response_schema: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_sampling_params: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_speed: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_system_messages: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_thinking_cache_preservation: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tool_choice: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tool_search: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_url_context: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_video_input: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_vision: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_web_search: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_xhigh_reasoning_effort: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub thinking_always_on: Option<bool>,
/// Context-length or result-count tiered rates; each tier's costs apply within its range.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tiered_pricing: Option<Vec<TieredRate>>,
/// Provider default tokens-per-minute limit.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tpm: Option<u64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub use_openai_responses_path: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub uses_embed_content: Option<bool>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub vertex_ai_audio_api: Option<VertexAiAudioApi>,
/// Whether web search is billed per query or per prompt.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub web_search_billing_unit: Option<WebSearchBillingUnit>,
}

View file

@ -0,0 +1,7 @@
use crate::model_info::ModelInfo;
/// JSON Schema for one catalog model entry, mirroring
/// `model_prices_and_context_window.schema.json`'s `modelEntry` definition.
pub fn model_entry_json_schema() -> schemars::Schema {
schemars::schema_for!(ModelInfo)
}

View file

@ -0,0 +1,253 @@
use std::path::{Path, PathBuf};
use litellm_model_catalog::{AliasIssue, Catalog, Error, IntegrityLimits, Provenance};
use rstest::{fixture, rstest};
use serde_json::json;
const ALPHA_FIXTURE: &[u8] = br#"{
"sample_spec":{"explanation":"example"},
"fallback_generalizations":{"rules":[{"name":"family","pattern":"^new-","model_info":{"mode":"chat"}}]},
"Alpha":{"litellm_provider":"test","aliases":["short"],"price":0,"enabled":false,
"optional":null,"unknown":{"nested":[1,{"x":true}]}}
}"#;
#[fixture]
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
}
#[fixture]
fn current_catalog(repo_root: PathBuf) -> Catalog {
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
Catalog::parse(&body, Provenance::default()).unwrap()
}
#[fixture]
fn backup_catalog(repo_root: PathBuf) -> Catalog {
let body = std::fs::read(repo_root.join("litellm/model_prices_and_context_window_backup.json"))
.unwrap();
Catalog::parse(&body, Provenance::default()).unwrap()
}
#[fixture]
fn fixture_catalog() -> Catalog {
Catalog::parse(
ALPHA_FIXTURE,
Provenance {
source: Some("fixture".into()),
revision: Some("rev".into()),
etag: None,
},
)
.unwrap()
}
#[rstest]
fn preserves_fields_and_metadata(fixture_catalog: Catalog) {
let catalog = fixture_catalog;
let entry = catalog.lookup("SHORT").unwrap();
assert_eq!(entry.canonical_key, "Alpha");
assert_eq!(entry.matched_key, "short");
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
assert_eq!(entry.entry.field("enabled"), Some(&json!(false)));
assert_eq!(entry.entry.field("optional"), Some(&json!(null)));
assert_eq!(entry.entry.field("missing"), None);
assert_eq!(
entry.entry.field("unknown"),
Some(&json!({"nested":[1,{"x":true}]}))
);
assert_eq!(entry.entry.field("aliases"), None);
assert_eq!(entry.entry.info().litellm_provider.as_deref(), Some("test"));
assert_eq!(
catalog.sample_spec(),
Some(&json!({"explanation":"example"}))
);
assert_eq!(catalog.fallback_rules().unwrap().len(), 1);
assert_eq!(catalog.provenance().revision.as_deref(), Some("rev"));
assert_eq!(catalog.model_count(), 1);
}
#[rstest]
fn snapshot_does_not_borrow_source() {
let mut source = ALPHA_FIXTURE.to_vec();
let catalog = Catalog::parse(&source, Provenance::default()).unwrap();
source.fill(b' ');
let entry = catalog.lookup("short").unwrap();
assert_eq!(entry.canonical_key, "Alpha");
assert_eq!(entry.entry.field("price"), Some(&json!(0)));
}
#[rstest]
#[case("Shared", "First")]
#[case("Second", "Second")]
#[case("shared", "Second")]
#[case("FIRST", "First")]
#[case("sHaReD", "Second")]
fn alias_collisions_and_case_fallback_follow_python_order(
#[case] lookup: &str,
#[case] expected: &str,
) {
let catalog = Catalog::parse(
br#"{
"First":{"aliases":["Shared","Second","first"],"value":1},
"Second":{"aliases":["Shared","sHaReD"],"value":2},
"SHARED":{"value":3}
}"#,
Provenance::default(),
)
.unwrap();
assert_eq!(catalog.lookup(lookup).unwrap().canonical_key, expected);
assert_eq!(catalog.alias_count(), 3);
assert!(
catalog
.alias_issues()
.contains(&AliasIssue::CanonicalCollision {
model: "First".into(),
alias: "Second".into(),
})
);
assert!(
catalog
.alias_issues()
.contains(&AliasIssue::AliasCollision {
model: "Second".into(),
alias: "Shared".into(),
})
);
}
#[derive(Debug)]
enum ValidationOutcome {
Ok,
Shrunk,
BelowMinimum,
InvalidRatio,
}
#[rstest]
#[case(
IntegrityLimits {
backup_model_count: 2,
min_model_count: 1,
min_backup_ratio: 0.5,
},
ValidationOutcome::Ok
)]
#[case(
IntegrityLimits {
backup_model_count: 3,
min_model_count: 1,
min_backup_ratio: 0.5,
},
ValidationOutcome::Shrunk
)]
#[case(
IntegrityLimits {
backup_model_count: 0,
min_model_count: 2,
min_backup_ratio: 0.5,
},
ValidationOutcome::BelowMinimum
)]
#[case(
IntegrityLimits {
backup_model_count: 0,
min_model_count: 0,
min_backup_ratio: f64::NAN,
},
ValidationOutcome::InvalidRatio
)]
fn integrity_uses_canonical_count_and_strict_shrink_boundary(
#[case] limits: IntegrityLimits,
#[case] expected: ValidationOutcome,
) {
let catalog = Catalog::parse(
br#"{"sample_spec":{},"fallback_generalizations":{"rules":[]},"a":{"aliases":["b","c"]}}"#,
Provenance::default(),
)
.unwrap();
let actual = catalog.validate(limits);
match expected {
ValidationOutcome::Ok => assert!(actual.is_ok()),
ValidationOutcome::Shrunk => {
assert!(matches!(actual, Err(Error::Shrunk { actual: 1, .. })))
}
ValidationOutcome::BelowMinimum => {
assert!(matches!(actual, Err(Error::BelowMinimum { actual: 1, .. })))
}
ValidationOutcome::InvalidRatio => assert!(matches!(actual, Err(Error::InvalidRatio))),
}
}
#[derive(Debug)]
enum MalformedOutcome {
Empty,
Json,
EntryNotObject,
}
#[rstest]
#[case::empty(b"{}", MalformedOutcome::Empty)]
#[case::invalid_json(b"{", MalformedOutcome::Json)]
#[case::entry_not_object(br#"{"a":1}"#, MalformedOutcome::EntryNotObject)]
#[case::fallback_rules_missing(
br#"{"fallback_generalizations":{},"a":{}}"#,
MalformedOutcome::Json
)]
fn malformed_input_and_aliases_have_typed_outcomes(
#[case] body: &[u8],
#[case] expected: MalformedOutcome,
) {
let actual = Catalog::parse(body, Provenance::default());
match expected {
MalformedOutcome::Empty => assert!(matches!(actual, Err(Error::Empty))),
MalformedOutcome::Json => assert!(matches!(actual, Err(Error::Json(_)))),
MalformedOutcome::EntryNotObject => {
assert!(matches!(actual, Err(Error::EntryNotObject { .. })))
}
}
}
#[rstest]
fn invalid_aliases_are_reported_not_fatal() {
let catalog = Catalog::parse(
br#"{"a":{"aliases":"bad"},"b":{"aliases":[9,"ok"]}}"#,
Provenance::default(),
)
.unwrap();
assert_eq!(
catalog.alias_issues(),
&[
AliasIssue::InvalidList { model: "a".into() },
AliasIssue::InvalidName { model: "b".into() },
]
);
assert_eq!(catalog.lookup("ok").unwrap().canonical_key, "b");
assert!(catalog.lookup("missing").is_none());
}
#[rstest]
fn parses_current_and_packaged_catalogs_without_pinning_counts(
current_catalog: Catalog,
backup_catalog: Catalog,
) {
assert!(current_catalog.model_count() > 0);
assert!(backup_catalog.model_count() > 0);
assert!(current_catalog.sample_spec().is_some());
assert!(backup_catalog.sample_spec().is_some());
assert!(
current_catalog
.validate(IntegrityLimits::python_defaults(
backup_catalog.model_count()
))
.is_ok()
);
for name in current_catalog.model_names() {
let entry = current_catalog.lookup(name).unwrap().entry;
assert_eq!(
entry.info().litellm_provider.is_some(),
entry.field("litellm_provider").is_some()
);
}
}

View file

@ -0,0 +1,121 @@
use std::collections::{BTreeSet, HashSet};
use std::path::{Path, PathBuf};
use indexmap::IndexMap;
use litellm_model_catalog::{
Catalog, FallbackGeneralizations, ModelInfo, Provenance, model_entry_json_schema,
};
use rstest::{fixture, rstest};
use serde_json::{Map, Value};
#[fixture]
fn repo_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR")).join("../../..")
}
fn json_eq(left: &Value, right: &Value) -> bool {
match (left, right) {
(Value::Number(left), Value::Number(right)) => left.as_f64() == right.as_f64(),
(Value::Array(left), Value::Array(right)) => {
left.len() == right.len() && left.iter().zip(right).all(|(a, b)| json_eq(a, b))
}
(Value::Object(left), Value::Object(right)) => {
left.len() == right.len()
&& left
.iter()
.all(|(key, value)| right.get(key).is_some_and(|other| json_eq(value, other)))
}
_ => left == right,
}
}
fn keys(value: &Map<String, Value>) -> BTreeSet<String> {
value.keys().cloned().collect()
}
fn symmetric_difference(left: &BTreeSet<String>, right: &BTreeSet<String>) -> BTreeSet<String> {
left.symmetric_difference(right).cloned().collect()
}
#[rstest]
#[case("model_prices_and_context_window.json")]
#[case("litellm/model_prices_and_context_window_backup.json")]
fn every_entry_round_trips_through_model_info(repo_root: PathBuf, #[case] filename: &str) {
let body = std::fs::read(repo_root.join(filename)).unwrap();
let document: IndexMap<String, Value> = serde_json::from_slice(&body).unwrap();
for (model_name, value) in document {
if matches!(
model_name.as_str(),
"sample_spec" | "fallback_generalizations"
) {
continue;
}
let object = value
.as_object()
.unwrap_or_else(|| panic!("{model_name} is not an object"));
let info: ModelInfo = serde_json::from_value(value.clone())
.unwrap_or_else(|error| panic!("{model_name} does not deserialize: {error}"));
let serialized = serde_json::to_value(info).unwrap();
let serialized_object = serialized
.as_object()
.unwrap_or_else(|| panic!("{model_name} did not serialize as an object"));
let mut expected = object.clone();
expected.remove("aliases");
let expected_keys = keys(&expected);
let serialized_keys = keys(serialized_object);
assert_eq!(
expected_keys,
serialized_keys,
"{model_name} key difference: {:?}",
symmetric_difference(&expected_keys, &serialized_keys)
);
assert!(
json_eq(&Value::Object(expected), &serialized),
"{model_name} changed during ModelInfo round-trip"
);
}
}
#[rstest]
fn fallback_generalizations_are_typed(repo_root: PathBuf) {
let body = std::fs::read(repo_root.join("model_prices_and_context_window.json")).unwrap();
let document: Map<String, Value> = serde_json::from_slice(&body).unwrap();
let Some(raw_rules) = document.get("fallback_generalizations") else {
return;
};
let _: FallbackGeneralizations = serde_json::from_value(raw_rules.clone()).unwrap();
let catalog = Catalog::parse(&body, Provenance::default()).unwrap();
assert!(
catalog
.fallback_rules()
.is_some_and(|rules| !rules.is_empty())
);
}
#[rstest]
fn generated_schema_properties_match_repo_schema(repo_root: PathBuf) {
let body =
std::fs::read(repo_root.join("model_prices_and_context_window.schema.json")).unwrap();
let document: Value = serde_json::from_slice(&body).unwrap();
let repo_entry_properties = document["$defs"]["modelEntry"]["properties"]
.as_object()
.unwrap();
let generated = serde_json::to_value(model_entry_json_schema()).unwrap();
let generated_properties = generated["properties"].as_object().unwrap();
let expected = keys(repo_entry_properties);
let actual = keys(generated_properties);
assert_eq!(
expected,
actual,
"modelEntry property difference: {:?}",
symmetric_difference(&expected, &actual)
);
let repo_root_properties = document["properties"].as_object().unwrap();
let actual_root: HashSet<String> = repo_root_properties.keys().cloned().collect();
let expected_root: HashSet<String> = ["sample_spec", "fallback_generalizations"]
.into_iter()
.map(str::to_owned)
.collect();
assert_eq!(actual_root, expected_root);
}

View file

@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"]
tiktoken = ["litellm-token-counter/tiktoken"]
[dependencies]
fancy-regex.workspace = true
litellm-tracing.workspace = true
litellm-host.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-cache.workspace = true

View file

@ -1,6 +1,7 @@
use crate::logger::run_sync_value;
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use litellm_cache_redis_semantic::RedisSemanticConfig;
use litellm_host_python::{release_gil, run_sync_value};
use litellm_host_python::release_gil;
use litellm_http::ClientVariant;
use pyo3::prelude::*;

View file

@ -1,5 +1,6 @@
use crate::logger::run_async;
use litellm_cache_response::PartialHits;
use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py};
use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyRuntimeError, PyValueError},

View file

@ -1,10 +1,11 @@
use crate::logger::run_sync_value;
use litellm_auth_aws::AwsAuthConfig;
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization};
use litellm_cache_redis::{RedisNode, RedisTopology};
use litellm_cache_redis_semantic::RedisSemanticConfig;
use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
use litellm_host_python::{release_gil, run_sync_value};
use litellm_host_python::release_gil;
use litellm_http::ClientVariant;
use pyo3::{
PyTraverseError, PyVisit,

View file

@ -470,7 +470,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move {
service
@ -495,7 +495,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_lookup(&request, now()).await },
super::cache_error,
@ -550,7 +550,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_store(&request, response, now()).await },
super::cache_error,
@ -619,7 +619,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_store_batch(entries, now()).await },
super::cache_error,

View file

@ -1,7 +1,8 @@
use crate::logger::run_async;
use std::{collections::VecDeque, time::Duration};
use litellm_cache::Error;
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyException, PyRuntimeError},

View file

@ -102,7 +102,7 @@ pub(crate) fn call_config(
.without_missing_files(&|path: &Path| path.exists());
let resolution = Resolution::from(&settings);
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
PythonSettings::warn(py, &unsupported.to_string())?;
crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}"));
}
Ok(resolution.config)
}

View file

@ -4,6 +4,7 @@ mod credentials;
mod diagnostics;
mod errors;
mod http;
mod logger;
mod marshal;
mod python_settings;
mod routes;
@ -25,6 +26,8 @@ mod _native {
#[pymodule_export]
use crate::errors::{RustBridgeDeclined, RustUpstreamError};
#[pymodule_export]
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{
@ -87,6 +90,7 @@ mod tests {
"chat_completions",
"achat_completions",
"ResponsesWebSocketConnection",
"NativeDiagnosticProcessor",
"TokenCounter",
"Tokenizer",
"gil_stats",

View file

@ -0,0 +1,46 @@
use std::future::Future;
use pyo3::prelude::*;
use serde::Serialize;
pub(crate) fn run_sync<T, E, F>(
py: Python<'_>,
future: F,
map_error: fn(E) -> PyErr,
) -> PyResult<Py<PyAny>>
where
T: Serialize + Send + 'static,
E: Send + 'static,
F: Future<Output = Result<T, E>> + Send + 'static,
{
litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
}
pub(crate) fn run_async<T, E, F>(
py: Python<'_>,
future: F,
map_error: fn(E) -> PyErr,
) -> PyResult<Bound<'_, PyAny>>
where
T: Serialize + Send + 'static,
E: Send + 'static,
F: Future<Output = Result<T, E>> + Send + 'static,
{
litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
}
pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
where
T: Send + 'static,
F: Future<Output = PyResult<T>> + Send + 'static,
{
litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
}
pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
where
T: for<'py> IntoPyObject<'py> + Send + 'static,
F: Future<Output = PyResult<T>> + Send + 'static,
{
litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
}

View file

@ -0,0 +1,41 @@
use std::sync::OnceLock;
use litellm_host::{
host::HostResult,
machine::{HostFailure, Interrupted, Machine, Step},
route::Route,
};
use litellm_tracing::Logger;
use pyo3::Python;
pub(crate) struct LoggedMachine<M> {
machine: M,
logger: OnceLock<Logger>,
}
impl<M> LoggedMachine<M> {
pub(crate) fn new(machine: M) -> Self {
Self {
machine,
logger: OnceLock::new(),
}
}
}
impl<M: Machine> Machine for LoggedMachine<M> {
type Route = M::Route;
type Complete = M::Complete;
fn resume(&mut self, result: Option<HostResult<Self::Route>>) -> Step<'_, Self> {
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result))))
}
fn interrupt(
&mut self,
failure: HostFailure<<Self::Route as Route>::Error>,
) -> Interrupted<'_, Self> {
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
}
}

View file

@ -0,0 +1,166 @@
mod execution;
mod machine;
pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
pub(crate) use machine::LoggedMachine;
use litellm_host_python::Pythonized;
use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink};
use pyo3::exceptions::PyRuntimeError;
use pyo3::prelude::*;
const MODULE: &str = "litellm.rust_bridge.logger";
type NativeDiagnosticOutput = (String, Option<String>, Option<String>, Vec<String>, bool);
#[pyclass]
pub(crate) struct NativeDiagnosticProcessor {
inner: Processor,
}
#[pymethods]
impl NativeDiagnosticProcessor {
#[new]
fn new(minimum_custom_key_length: usize) -> Self {
Self {
inner: Processor::new(minimum_custom_key_length),
}
}
fn redact_text(&self, text: &str) -> PyResult<String> {
self.inner.redact_text(text).map_err(processing_error)
}
fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult<String> {
self.inner
.redact_structured_text(key, text)
.map_err(processing_error)
}
fn redact_client_message(&self, text: &str) -> PyResult<String> {
self.inner
.redact_client_message(text)
.map_err(processing_error)
}
#[pyo3(signature = (message, exception, stack, leaves, policy))]
fn process_diagnostic(
&self,
message: String,
exception: Option<String>,
stack: Option<String>,
leaves: Vec<(Option<String>, String)>,
policy: (bool, i64, i64),
) -> PyResult<NativeDiagnosticOutput> {
let input = DiagnosticInput {
message,
exception,
stack,
leaves,
};
let policy = Policy {
redact: policy.0,
base64_limit: policy.1,
text_limit: policy.2,
};
self.inner
.process_diagnostic(&input, policy)
.map(|output| {
(
output.message,
output.exception,
output.stack,
output.leaves,
output.changed,
)
})
.map_err(processing_error)
}
fn scrub_access_arguments(&self, arguments: Vec<String>) -> PyResult<Vec<String>> {
self.inner
.scrub_access_arguments(&arguments)
.map_err(processing_error)
}
}
fn processing_error(_: fancy_regex::Error) -> PyErr {
PyRuntimeError::new_err("diagnostic processing failed")
}
struct PythonSink {
correlation: (String, String),
}
fn level(level: &Level) -> u8 {
match *level {
Level::ERROR => 40,
Level::WARN => 30,
Level::INFO => 20,
Level::DEBUG | Level::TRACE => 10,
}
}
fn report<T: Default>(py: Python<'_>, result: PyResult<T>) -> T {
match result {
Ok(value) => value,
Err(error) => {
error.write_unraisable(py, None);
T::default()
}
}
}
impl Sink for PythonSink {
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::")
{
return false;
}
Python::try_attach(|py| {
report(
py,
py.import(MODULE)
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
.and_then(|enabled| enabled.extract()),
)
})
.unwrap_or(false)
}
fn emit(&self, record: &Record) {
Python::try_attach(|py| {
report(
py,
py.import(MODULE).and_then(|module| {
module
.call_method1(
"emit",
(
level(record.metadata.level()),
&record.message,
record.metadata.file().unwrap_or_default(),
record.metadata.line().unwrap_or_default(),
record.metadata.target(),
Pythonized(&record.fields),
(&self.correlation.0, &self.correlation.1),
),
)
.map(|_| ())
}),
);
});
}
}
pub(crate) fn capture(py: Python<'_>) -> Logger {
report(
py,
py.import(MODULE)
.and_then(|module| module.call_method0("context"))
.and_then(|value| value.extract())
.map(|correlation| Logger::new(PythonSink { correlation })),
)
}
#[cfg(test)]
mod tests;

View file

@ -0,0 +1,295 @@
use std::{process::Command, task::Poll};
use litellm_host::{
host::HostResult,
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
route::Route,
};
use pyo3::{prelude::*, types::PyDict};
struct DiagnosticMachine;
impl Route for DiagnosticMachine {
type Response = ();
type Error = String;
type Op = ();
type OpResult = ();
type Chunk = ();
type StreamHead = ();
}
impl Machine for DiagnosticMachine {
type Route = Self;
type Complete = ();
fn resume(&mut self, _: Option<HostResult<Self>>) -> Step<'_, Self> {
litellm_tracing::warn!("machine started");
Box::pin(async {
tokio::task::yield_now().await;
litellm_tracing::warn!("machine warning");
Ok(MachineStep::Complete(()))
})
}
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
Box::pin(async {
litellm_tracing::warn!("machine interrupted");
Ok(())
})
}
}
#[pyfunction]
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
let mut future = Box::pin(async move {
machine
.resume(None)
.await
.map_err(pyo3::exceptions::PyValueError::new_err)?;
machine
.interrupt(HostFailure::Error("stop".into()))
.await
.map_err(pyo3::exceptions::PyValueError::new_err)
});
assert!(matches!(
litellm_host_python::poll_async_value(py, future.as_mut())?,
Poll::Pending
));
litellm_host_python::run_async_value(py, future)
}
#[pyfunction]
fn warning(py: Python<'_>) {
super::capture(py).scope(|| {
litellm_tracing::warn!(attempt = 3, retry = true, "native warning");
});
}
#[pyfunction]
fn levels(py: Python<'_>) {
super::capture(py).scope(|| {
litellm_tracing::trace!("trace");
litellm_tracing::debug!("debug");
litellm_tracing::info!("info");
litellm_tracing::warn!("warn");
litellm_tracing::error!("error");
litellm_tracing::warn!(target: "unrelated_transport", "private wire data");
});
}
#[pyfunction]
fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
super::run_async_value(py, async {
tokio::task::yield_now().await;
litellm_tracing::warn!("async warning");
Ok(())
})
}
#[pyfunction]
fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
super::run_sync_value(py, async {
tokio::task::yield_now().await;
litellm_tracing::warn!("sync warning");
Ok(())
})
}
#[pyfunction]
fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
super::run_sync_value(py, async {
litellm_tracing::warn!("failure diagnostic");
Err(pyo3::exceptions::PyValueError::new_err("request failed"))
})
}
#[pyfunction]
fn http_warning(py: Python<'_>) -> PyResult<()> {
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
}
#[test]
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
let output = Command::new(std::env::current_exe().unwrap())
.args([
"--exact",
std::thread::current().name().unwrap(),
"--nocapture",
])
.env("LITELLM_LOGGER_TEST_PROCESS", "1")
.output()
.unwrap();
assert!(
output.status.success(),
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
return;
}
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
locals
.set_item(
"repo_root",
concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."),
)
.unwrap();
locals
.set_item(
"machine_warning",
wrap_pyfunction!(machine_warning, py).unwrap(),
)
.unwrap();
locals
.set_item(
"synchronous_failure",
wrap_pyfunction!(synchronous_failure, py).unwrap(),
)
.unwrap();
locals
.set_item("levels", wrap_pyfunction!(levels, py).unwrap())
.unwrap();
locals
.set_item("warning", wrap_pyfunction!(warning, py).unwrap())
.unwrap();
locals
.set_item(
"asynchronous_warning",
wrap_pyfunction!(asynchronous_warning, py).unwrap(),
)
.unwrap();
locals
.set_item(
"synchronous_warning",
wrap_pyfunction!(synchronous_warning, py).unwrap(),
)
.unwrap();
locals
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
.unwrap();
let importable = py
.eval(
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
Some(&locals),
Some(&locals),
)
.unwrap()
.is_truthy()
.unwrap();
if !importable {
eprintln!("SKIP: litellm package dependencies are not importable in this interpreter");
return;
}
py.run(c"
import asyncio
import logging
import sys
sys.path.insert(0, repo_root)
import litellm
from litellm._logging import verbose_logger, session_id_var, trace_id_var
class Capture(logging.Handler):
def __init__(self):
super().__init__()
self.records = []
def emit(self, record):
self.records.append(record)
warning()
class Broken(logging.Handler):
def emit(self, record):
raise ValueError('handler failed')
capture = Capture()
old_handlers = verbose_logger.handlers
old_level = verbose_logger.level
old_correlation = litellm.request_correlation_in_logs
old_curve = litellm.ssl_ecdh_curve
old_unraisable = sys.unraisablehook
failures = []
try:
verbose_logger.handlers = [capture]
litellm.request_correlation_in_logs = True
verbose_logger.setLevel(logging.ERROR)
warning()
assert capture.records == []
verbose_logger.setLevel(logging.WARNING)
warning()
assert len(capture.records) == 1
record = capture.records[0]
assert record.getMessage() == 'native warning'
assert record.levelno == logging.WARNING
assert record.rust_fields == {'attempt': 3, 'retry': True}
assert record.pathname.endswith('logger/tests.rs')
assert record.lineno > 0
assert record.rust_target.endswith('logger::tests')
verbose_logger.setLevel(logging.ERROR)
warning()
assert len(capture.records) == 1
verbose_logger.setLevel(logging.WARNING)
async def request(name):
session = session_id_var.set(name)
trace = trace_id_var.set('trace-' + name)
try:
await asynchronous_warning()
await machine_warning()
synchronous_warning()
assert session_id_var.get() == name
assert trace_id_var.get() == 'trace-' + name
finally:
trace_id_var.reset(trace)
session_id_var.reset(session)
async def concurrent():
await asyncio.gather(request('first'), request('second'))
asyncio.run(concurrent())
assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted(
(message, name, 'trace-' + name)
for name in ('first', 'second')
for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted')
)
verbose_logger.setLevel(logging.DEBUG)
before_levels = len(capture.records)
levels()
assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [
('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO),
('warn', logging.WARNING), ('error', logging.ERROR),
]
before = len(capture.records)
litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve'
http_warning()
http_warning()
assert len(capture.records) == before + 1
assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage()
assert capture.records[-1].pathname.endswith('http.rs')
verbose_logger.handlers = [Broken()]
sys.unraisablehook = failures.append
warning()
assert len(failures) == 1
assert str(failures[0].exc_value) == 'handler failed'
try:
synchronous_failure()
except ValueError as error:
assert str(error) == 'request failed'
else:
raise AssertionError('request failure was lost')
assert len(failures) == 2
finally:
sys.unraisablehook = old_unraisable
verbose_logger.handlers = old_handlers
verbose_logger.setLevel(old_level)
litellm.request_correlation_in_logs = old_correlation
litellm.ssl_ecdh_curve = old_curve
", Some(&locals), Some(&locals)).unwrap();
});
}

View file

@ -44,11 +44,6 @@ impl PythonSettings {
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
Snapshot { group: self, value }
}
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
py.import(MODULE)?.getattr("warn")?.call1((message,))?;
Ok(())
}
}
#[cfg(test)]

View file

@ -1,7 +1,8 @@
use crate::logger::{run_async, run_sync};
use litellm_core::audio_transcription::{
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use litellm_host_python::from_py_argument;
use pyo3::prelude::*;
use serde_json::{Map, Value};

View file

@ -1,8 +1,9 @@
use crate::logger::{run_async, run_sync};
use litellm_core::chat_completions::{
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
types::ChatCompletionsRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use litellm_host_python::from_py_argument;
use litellm_types::utils::ChatCompletionsResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};

View file

@ -43,7 +43,7 @@ fn run_messages(
py,
SURFACE,
PublicCall::capture(&request, &args, &kwargs)?,
messages_machine(),
crate::logger::LoggedMachine::new(messages_machine()),
MessagesRouteHost::new(request.unbind()),
asynchronous,
)

View file

@ -73,7 +73,7 @@ fn run_ocr(
py,
if asynchronous { ASYNC_SURFACE } else { SURFACE },
PublicCall::capture(&request, &args, &kwargs)?,
ocr_machine(client),
crate::logger::LoggedMachine::new(ocr_machine(client)),
OcrRouteHost::new(request.unbind()),
asynchronous,
)

View file

@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection {
) -> PyResult<Bound<'py, PyAny>> {
let headers = marshal_headers(headers)?;
let timeout = optional_timeout(timeout_seconds);
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
.await
.map_err(responses_error_to_pyerr)?;
@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection {
fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner
.send_text(text)
.await
@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection {
fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner.recv_text().await.map_err(responses_error_to_pyerr)
})
}
fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner.close().await.map_err(responses_error_to_pyerr)
})
}

View file

@ -1,7 +1,8 @@
use crate::logger::run_async;
use std::sync::Arc;
use std::{num::NonZero, thread::available_parallelism};
use litellm_host_python::{enter_native, run_async};
use litellm_host_python::enter_native;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};

View file

@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true
litellm-core-utils.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tracing = "0.1"
litellm-tracing.workspace = true
veil.workspace = true
aws-sdk-kms = "1.120.0"
aws-sdk-secretsmanager = "1.117.0"

View file

@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 {
.await
.is_err()
{
tracing::warn!("secret created but replication failed");
litellm_tracing::warn!("secret created but replication failed");
}
Ok(response)
}

View file

@ -14,7 +14,7 @@ reqwest.workspace = true
serde_json.workspace = true
thiserror.workspace = true
veil.workspace = true
tracing = "0.1"
litellm-tracing.workspace = true
percent-encoding = "2.3"
tokio = { workspace = true, features = ["sync"] }

View file

@ -92,7 +92,7 @@ impl CyberArkSecretManager {
.unwrap_or(true);
let mut builder = reqwest::Client::builder();
if !verify {
tracing::warn!(
litellm_tracing::warn!(
"CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
);
builder = builder.danger_accept_invalid_certs(true);
@ -259,11 +259,13 @@ impl CyberArkSecretManager {
.endpoint
.join(&format!("policies/{}/policy/root", self.account));
let Ok(policy_url) = policy_url else {
tracing::warn!("Could not build CyberArk policy endpoint");
litellm_tracing::warn!("Could not build CyberArk policy endpoint");
return;
};
let Ok(authorization) = self.authorization_header(context).await else {
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
litellm_tracing::warn!(
"Could not authenticate while ensuring CyberArk variable exists"
);
return;
};
let body = format!(
@ -288,19 +290,19 @@ impl CyberArkSecretManager {
reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
) =>
{
tracing::debug!(
litellm_tracing::debug!(
"CyberArk variable policy already exists or conflicts: {}",
response.status()
);
}
Ok(response) => {
tracing::warn!(
litellm_tracing::warn!(
"Could not ensure CyberArk variable exists: {}",
response.status()
);
}
Err(error) => {
tracing::warn!("Error ensuring CyberArk variable exists: {error}");
litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}");
}
}
}
@ -324,7 +326,7 @@ impl CyberArkSecretManager {
_recovery_window_in_days: Option<u32>,
_context: &SecretOperationContext,
) -> Result<DeleteOutcome, Error> {
tracing::warn!(
litellm_tracing::warn!(
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
);
self.secrets.invalidate(name).await;

View file

@ -0,0 +1,17 @@
[package]
name = "litellm-tracing"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
fancy-regex.workspace = true
percent-encoding.workspace = true
serde_json.workspace = true
tracing.workspace = true
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
[dev-dependencies]
rstest.workspace = true
tokio.workspace = true

View file

@ -0,0 +1,26 @@
# Native diagnostic tracing
`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber
Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work
Bindings implement `litellm_tracing::Sink` to connect events to their host runtime:
```rust
pub trait Sink: Send + Sync + 'static {
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
fn emit(&self, record: &Record);
}
```
Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value
Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion
The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger
Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events
A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue
This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate

View file

@ -0,0 +1,146 @@
use std::{
cell::Cell,
fmt,
future::{Future, poll_fn},
pin::pin,
};
use serde_json::{Map, Value};
use tracing::{
Dispatch, Event, Subscriber,
field::{Field, Visit},
subscriber::Interest,
};
use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
mod processing;
mod redaction;
pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor};
pub use redaction::{REDACTED, SecretRedactor};
pub use tracing::{Level, Metadata, debug, error, info, trace, warn};
pub trait Sink: Send + Sync + 'static {
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
fn emit(&self, record: &Record);
}
#[derive(Debug)]
pub struct Record {
pub metadata: &'static Metadata<'static>,
pub message: String,
pub fields: Map<String, Value>,
}
#[derive(Clone, Default)]
pub struct Logger {
dispatch: Dispatch,
}
impl Logger {
pub fn new(sink: impl Sink) -> Self {
Self {
dispatch: Dispatch::new(Registry::default().with(Output(sink))),
}
}
pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
if EMITTING.get() {
return operation();
}
tracing::dispatcher::with_default(&self.dispatch, operation)
}
pub fn instrument<F: Future>(&self, future: F) -> impl Future<Output = F::Output> + use<F> {
let logger = self.clone();
async move {
let mut future = pin!(future);
poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await
}
}
}
thread_local! {
static EMITTING: Cell<bool> = const { Cell::new(false) };
}
struct Emitting;
impl Emitting {
fn enter() -> Option<Self> {
EMITTING.with(|active| (!active.replace(true)).then_some(Self))
}
}
impl Drop for Emitting {
fn drop(&mut self) {
EMITTING.set(false);
}
}
struct Output<S>(S);
impl<S: Sink, R: Subscriber> Layer<R> for Output<S> {
fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {
Interest::sometimes()
}
fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool {
let Some(_guard) = Emitting::enter() else {
return false;
};
self.0.enabled(metadata)
}
fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) {
let Some(_guard) = Emitting::enter() else {
return;
};
let mut record = Record {
metadata: event.metadata(),
message: String::new(),
fields: Map::new(),
};
event.record(&mut record);
self.0.emit(&record);
}
}
impl Record {
fn field(&mut self, field: &Field, value: Value) {
if field.name() == "message" {
self.message = match value {
Value::String(message) => message,
value => value.to_string(),
};
} else {
self.fields.insert(field.name().to_owned(), value);
}
}
}
impl Visit for Record {
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
self.field(field, format!("{value:?}").into());
}
fn record_str(&mut self, field: &Field, value: &str) {
self.field(field, value.into());
}
fn record_bool(&mut self, field: &Field, value: bool) {
self.field(field, value.into());
}
fn record_i64(&mut self, field: &Field, value: i64) {
self.field(field, value.into());
}
fn record_u64(&mut self, field: &Field, value: u64) {
self.field(field, value.into());
}
fn record_f64(&mut self, field: &Field, value: f64) {
self.field(field, value.into());
}
}

View file

@ -0,0 +1,333 @@
use fancy_regex::Result;
use percent_encoding::percent_decode_str;
use crate::{REDACTED, SecretRedactor};
#[derive(Clone, Copy, Debug)]
pub struct Policy {
pub redact: bool,
pub base64_limit: i64,
pub text_limit: i64,
}
#[derive(Clone, Debug)]
pub struct DiagnosticInput {
pub message: String,
pub exception: Option<String>,
pub stack: Option<String>,
pub leaves: Vec<(Option<String>, String)>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DiagnosticOutput {
pub message: String,
pub exception: Option<String>,
pub stack: Option<String>,
pub leaves: Vec<String>,
pub changed: bool,
}
pub struct Processor {
redactor: SecretRedactor,
}
impl Processor {
pub fn new(minimum_custom_key_length: usize) -> Self {
Self {
redactor: SecretRedactor::new(minimum_custom_key_length),
}
}
pub fn redact_text(&self, text: &str) -> Result<String> {
self.redactor.try_redact(text)
}
pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result<String> {
self.redactor.try_redact_structured(key, text)
}
pub fn redact_client_message(&self, text: &str) -> Result<String> {
self.redactor.try_redact_internal(text)
}
pub fn process_diagnostic(
&self,
input: &DiagnosticInput,
policy: Policy,
) -> Result<DiagnosticOutput> {
let message = self.process_text(&input.message, policy)?;
let exception = input
.exception
.as_deref()
.map(|text| self.process_text(text, policy))
.transpose()?;
let stack = input
.stack
.as_deref()
.map(|text| {
if policy.redact {
self.redact_text(text)
} else {
Ok(text.to_owned())
}
})
.transpose()?;
let leaves = input
.leaves
.iter()
.map(|(key, text)| {
if policy.redact {
self.redact_structured_text(key.as_deref(), text)
} else {
Ok(text.clone())
}
})
.collect::<Result<Vec<_>>>()?;
let changed = message != input.message
|| exception != input.exception
|| stack != input.stack
|| leaves
.iter()
.zip(&input.leaves)
.any(|(processed, (_, original))| processed != original);
Ok(DiagnosticOutput {
message,
exception,
stack,
leaves,
changed,
})
}
pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result<Vec<String>> {
arguments
.iter()
.map(|argument| self.scrub_access_arg(argument))
.collect()
}
fn process_text(&self, text: &str, policy: Policy) -> Result<String> {
let collapsed = if policy.base64_limit > 0 {
collapse_base64(text, policy.base64_limit as usize)
} else {
text.to_owned()
};
let redacted = if policy.redact {
self.redact_text(&collapsed)?
} else {
collapsed
};
Ok(
if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize {
truncate_text(&redacted, policy.text_limit as usize)
} else {
redacted
},
)
}
fn scrub_access_arg(&self, value: &str) -> Result<String> {
let length = value.chars().count();
let scanned = if length <= 512 {
value
} else {
let head = &value[..char_offset(value, 512)];
if head.contains('?') {
&head[..head.rfind(['?', '&']).unwrap_or(0)]
} else {
head
}
};
let scrubbed = self.redact_text(scanned)?;
let (path, query) = scrubbed
.split_once('?')
.map_or((scrubbed.as_str(), None), |(path, query)| {
(path, Some(query))
});
let safe = if self.hides_encoded_credential(path)? {
REDACTED.to_owned()
} else if query.is_some() && self.hides_encoded_credential(&scrubbed)? {
format!("{path}?{REDACTED}")
} else {
scrubbed
};
Ok(if length > 512 {
format!(
"{safe}... ({} more chars truncated) ...",
length - scanned.chars().count()
)
} else {
safe
})
}
fn hides_encoded_credential(&self, value: &str) -> Result<bool> {
if !value.as_bytes().contains(&b'%') {
return Ok(false);
}
let decoded = percent_decode_str(value).decode_utf8_lossy();
Ok(self.redact_text(&decoded)? != decoded)
}
}
fn char_offset(text: &str, count: usize) -> usize {
text.char_indices()
.nth(count)
.map_or(text.len(), |(index, _)| index)
}
fn marker(skipped_chars: usize) -> String {
format!(
"... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..."
)
}
fn truncate_text(text: &str, limit: usize) -> String {
let length = text.chars().count();
let kept = limit.saturating_sub(marker(length).len());
if kept == 0 {
return text[..char_offset(text, limit)].to_owned();
}
let head = kept / 2;
let tail = kept - head;
format!(
"{}{}{}",
&text[..char_offset(text, head)],
marker(length - kept),
&text[char_offset(text, length - tail)..]
)
}
fn base64_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/'
}
fn looks_like_base64(run: &str) -> bool {
let unpadded = run.trim_end_matches('=');
let lower_hex = unpadded
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
let upper_hex = unpadded
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]);
(!lower_hex && !upper_hex) || repeated
}
fn base64_size(chars: usize) -> String {
let bytes = chars as f64 * 3.0 / 4.0;
if bytes >= 1024.0 * 1024.0 {
return format!("{:.2}MB", bytes / (1024.0 * 1024.0));
}
if bytes >= 1024.0 {
return format!("{:.1}KB", bytes / 1024.0);
}
format!("{}B", bytes as usize)
}
fn collapse_base64(text: &str, limit: usize) -> String {
let bytes = text.as_bytes();
let mut position = 0;
let mut previous = 0;
let mut output = String::new();
while position < bytes.len() {
if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) {
position += 1;
continue;
}
let start = position;
while position < bytes.len() && base64_byte(bytes[position]) {
position += 1;
}
let run_end = position;
while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' {
position += 1;
}
let run = &text[start..position];
if run_end - start > limit && looks_like_base64(run) {
output.push_str(&text[previous..start]);
output.push_str(&format!(
"[base64_data truncated: {}]",
base64_size(run.len())
));
previous = position;
}
}
output.push_str(&text[previous..]);
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() {
let processor = Processor::new(16);
let secret = format!("sk-{}", "q".repeat(48));
let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000));
let input = DiagnosticInput {
message: text,
exception: None,
stack: None,
leaves: vec![],
};
let output = processor
.process_diagnostic(
&input,
Policy {
redact: true,
base64_limit: 0,
text_limit: 500,
},
)
.unwrap();
assert!(output.message.chars().count() <= 500);
assert!(!output.message.contains("sk-qq"));
assert!(output.changed);
}
#[test]
fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() {
let processor = Processor::new(16);
let input = DiagnosticInput {
message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)),
exception: Some(format!("upload failed: {}", "Q".repeat(100))),
stack: Some("api_key=secret123".to_owned()),
leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())],
};
let output = processor
.process_diagnostic(
&input,
Policy {
redact: true,
base64_limit: 20,
text_limit: 0,
},
)
.unwrap();
assert!(output.message.contains("[base64_data truncated: 75B]"));
assert!(output.message.contains(&"a1".repeat(50)));
assert!(
output
.exception
.unwrap()
.contains("[base64_data truncated: 75B]")
);
assert_eq!(output.stack.as_deref(), Some(REDACTED));
assert_eq!(output.leaves, vec![REDACTED]);
}
#[test]
fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() {
let processor = Processor::new(16);
let arguments = vec![
"/v1/models?filter=gpt%2D4o&page=2".to_owned(),
"/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(),
];
assert_eq!(
processor.scrub_access_arguments(&arguments).unwrap(),
vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()]
);
}
}

View file

@ -0,0 +1,140 @@
use fancy_regex::{NoExpand, Regex};
pub const REDACTED: &str = "REDACTED";
#[cfg(test)]
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
fn secret_patterns(minimum_custom_key_length: usize) -> String {
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
[
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
r"\bya29\.[A-Za-z0-9_.~+/-]+",
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r"x-ak-[A-Za-z0-9\-_]{20,}",
r"AIza[0-9A-Za-z\-_]{35}",
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
r"dapi[0-9a-f]{32}",
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
concat!(
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|",
r"signing_key|encryption_key|",
r"auth_token|access_token|refresh_token|",
r"slack_webhook_url|webhook_url|",
r"database_connection_string|",
r"huggingface_token|jwt_secret)",
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
),
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
]
.join("|")
}
#[derive(Clone, Debug)]
pub struct SecretRedactor {
pattern: Regex,
internal_pattern: Regex,
}
impl SecretRedactor {
pub fn new(minimum_custom_key_length: usize) -> Self {
let pattern = Regex::new(&format!(
"(?i){}",
secret_patterns(minimum_custom_key_length)
))
.expect("secret redaction patterns compile");
let internal_pattern = Regex::new(concat!(
r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#,
r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#,
r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|",
r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|",
r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b",
))
.expect("internal detail patterns compile");
Self {
pattern,
internal_pattern,
}
}
pub fn redact(&self, value: &str) -> String {
self.try_redact(value)
.unwrap_or_else(|_| REDACTED.to_owned())
}
pub fn try_redact(&self, value: &str) -> fancy_regex::Result<String> {
self.pattern
.try_replacen(value, 0, NoExpand(REDACTED))
.map(|value| value.into_owned())
}
pub fn try_redact_structured(
&self,
key: Option<&str>,
value: &str,
) -> fancy_regex::Result<String> {
let scrubbed = self.try_redact(value)?;
if scrubbed != value || key.is_none() {
return Ok(scrubbed);
}
let rendered = format!("'{}': '{value}'", key.unwrap_or_default());
Ok(if self.try_redact(&rendered)? != rendered {
REDACTED.to_owned()
} else {
value.to_owned()
})
}
pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result<String> {
let without_traceback = value
.split_once("Traceback (most recent call last):")
.map_or(value, |(prefix, _)| prefix.trim_end());
self.internal_pattern
.try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED))
.map(|value| value.into_owned())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
expected
);
}
#[test]
fn sk_threshold_follows_the_minimum_custom_key_length() {
let redactor = SecretRedactor::new(8);
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
}
}

View file

@ -0,0 +1,122 @@
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
mpsc,
};
use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn};
use serde_json::{Value, json};
struct Output {
enabled: Arc<AtomicBool>,
sender: mpsc::Sender<(String, Value, Level, &'static str, Option<u32>)>,
}
impl Sink for Output {
fn enabled(&self, _: &Metadata<'_>) -> bool {
self.enabled.load(Ordering::Relaxed)
}
fn emit(&self, record: &Record) {
self.sender
.send((
record.message.clone(),
Value::Object(record.fields.clone()),
*record.metadata.level(),
record.metadata.target(),
record.metadata.line(),
))
.unwrap();
Logger::default().scope(|| warn!("a sink must not recursively emit"));
}
}
fn emit() {
warn!(
attempt = 3_u64,
elapsed = 1.5,
retry = true,
reason = "timeout",
"retry {}",
3
);
}
#[test]
fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() {
let (sender, receiver) = mpsc::channel();
let enabled = Arc::new(AtomicBool::new(false));
let logger = Logger::new(Output {
enabled: enabled.clone(),
sender,
});
logger.scope(emit);
assert!(receiver.try_recv().is_err());
enabled.store(true, Ordering::Relaxed);
logger.scope(emit);
let (message, fields, level, target, line) = receiver.try_recv().unwrap();
assert_eq!(message, "retry 3");
assert_eq!(
fields,
json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"})
);
assert_eq!(level, Level::WARN);
assert_eq!(target, module_path!());
assert!(line.is_some());
enabled.store(false, Ordering::Relaxed);
logger.scope(emit);
assert!(receiver.try_recv().is_err());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() {
let tasks = (0..2)
.map(|id| {
let (sender, receiver) = mpsc::channel();
let logger = Logger::new(Output {
enabled: Arc::new(AtomicBool::new(true)),
sender,
});
let task = tokio::spawn(logger.instrument(async move {
tokio::task::yield_now().await;
info!(id, "worker");
}));
(id, task, receiver)
})
.collect::<Vec<_>>();
for (id, task, receiver) in tasks {
task.await.unwrap();
let (message, fields, level, _, _) = receiver.try_recv().unwrap();
assert_eq!(message, "worker");
assert_eq!(fields, json!({"id": id}));
assert_eq!(level, Level::INFO);
assert!(receiver.try_recv().is_err());
}
}
#[test]
fn nested_scopes_restore_the_previous_sink() {
let (outer_sender, outer) = mpsc::channel();
let (inner_sender, inner) = mpsc::channel();
let logger = |sender| {
Logger::new(Output {
enabled: Arc::new(AtomicBool::new(true)),
sender,
})
};
let outside = logger(outer_sender);
let inside = logger(inner_sender);
outside.scope(|| {
info!("before");
inside.scope(|| info!("inside"));
info!("after");
});
assert_eq!(
outer.try_iter().map(|event| event.0).collect::<Vec<_>>(),
["before", "after"]
);
assert_eq!(
inner.try_iter().map(|event| event.0).collect::<Vec<_>>(),
["inside"]
);
}

View file

@ -1402,6 +1402,7 @@ from .exceptions import (
JSONSchemaValidationError,
LITELLM_EXCEPTION_TYPES,
MockException,
ModelNotMappedError as ModelNotMappedError,
)
from .budget_manager import BudgetManager
from .proxy.proxy_cli import run_server

View file

@ -1,11 +1,12 @@
import ast
import contextvars
import functools
import itertools
import logging
import os
import re
import sys
from collections.abc import Sequence
from collections.abc import Iterator
from datetime import datetime
from logging import Formatter
from typing import Any, Final, TextIO
@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.secret_redaction import (
_python_redact_string,
_python_redact_structured_value,
redact_internal_details,
redact_string,
redact_structured_value,
)
from litellm.rust_bridge import diagnostics
set_verbose = False
@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str:
pass through credential redaction.
"""
stripped: Final = "".join(ch for ch in value if ch.isprintable())
return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
def set_session_id(session_id: str) -> "contextvars.Token[str]":
@ -74,12 +77,6 @@ def _redact_string(value: str) -> str:
return redact_string(value)
def _redact_structured_value(key: str | None, value: str) -> str:
if not _ENABLE_SECRET_REDACTION:
return value
return redact_structured_value(key, value)
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
_REDACTED_STAMP: Final = object()
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
@ -103,14 +100,6 @@ def _plain_text(value: object) -> str:
return UNSERIALIZABLE_OBJECT
def _redact_extra_value(key: str, value: object) -> object:
try:
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
except Exception:
return _redact_string(_plain_text(value))
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None:
return None
try:
return color_message % record.args
except TypeError:
except Exception:
return color_message
@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
return True
# Runs before args are cleared, and before the extra-field loop below
# that redacts the substituted result.
substituted_color_message: Final = _substituted_color_message(record)
if substituted_color_message is not None:
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
try:
record.msg = _redact_string(record.getMessage())
record.args = None
except Exception:
if isinstance(record.msg, str):
record.msg = _redact_string(record.msg)
# Redact exception tracebacks
if record.exc_info and record.exc_info[1] is not None:
try:
record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
except Exception:
pass
if isinstance(record.stack_info, str):
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
# Redact extra fields passed via logger.debug("msg", extra={...})
record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items())
for key, value in record_items:
if key in _STANDARD_RECORD_ATTRS:
continue
if isinstance(value, str):
setattr(record, key, _redact_structured_value(key, value))
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
setattr(record, key, _redact_extra_value(key, value))
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
return True
return _process_record(record, base64_limit=0, text_limit=0, redact=True)
_secret_filter: Final = SecretRedactionFilter()
@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
def _hides_a_credential(value: str) -> bool:
"""Whether *value* only looks clean until it is percent-decoded."""
decoded: Final = unquote(value)
return _redact_string(decoded) != decoded
return _python_redact_string(decoded) != decoded
def _drop_encoded_credential(scrubbed: str) -> str:
@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str:
pattern and would then be logged raw.
"""
if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
return _drop_encoded_credential(_redact_string(value))
return _drop_encoded_credential(_python_redact_string(value))
head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
scrubbed: Final = _drop_encoded_credential(_redact_string(kept))
scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter):
if not _ENABLE_SECRET_REDACTION:
return True
if isinstance(record.args, tuple) and record.args:
strings: Final = tuple(arg for arg in record.args if isinstance(arg, str))
candidate: Final = diagnostics.run(
lambda native: native.scrub_access_arguments(strings),
lambda: tuple(_scrub_access_arg(arg) for arg in strings),
)
scrubbed: Final = (
candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings)
)
values: Final = iter(scrubbed)
record.args = tuple( # rebind-ok: a Filter scrubs records in place
_scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args
next(values) if isinstance(arg, str) else arg for arg in record.args
)
return True
# No positional args means everything is in msg, where collapsing is correct.
@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str:
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
def _extra_structure(key: str, value: object) -> object:
if isinstance(value, str):
return value
try:
return safe_json_structure(value, key=key)
except Exception:
return _plain_text(value)
def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]:
if isinstance(value, str):
yield key, value
elif isinstance(value, dict):
yield from itertools.chain.from_iterable(
_string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str)
)
elif isinstance(value, (list, tuple)):
yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value)
def _replace_string_leaves(value: object, values: Iterator[str]) -> object:
if isinstance(value, str):
return next(values)
if isinstance(value, dict):
return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers
key: _replace_string_leaves(child, values) for key, child in value.items()
}
if isinstance(value, list):
return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers
_replace_string_leaves(child, values) for child in value
]
if isinstance(value, tuple):
return tuple(_replace_string_leaves(child, values) for child in value)
return value
def _sort_processed_sets(original: object, processed: object) -> object:
if isinstance(original, set) and isinstance(processed, list):
return sorted(processed)
if isinstance(original, dict) and isinstance(processed, dict):
return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict
key: _sort_processed_sets(original.get(key), value) for key, value in processed.items()
}
if isinstance(original, list) and isinstance(processed, list):
return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list
_sort_processed_sets(before, after) for before, after in zip(original, processed)
]
if isinstance(original, tuple) and isinstance(processed, tuple):
return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed))
return processed
def _python_process_diagnostic(
message: str,
exception: str | None,
stack: str | None,
leaves: tuple[tuple[str | None, str], ...],
redact: bool,
base64_limit: int,
text_limit: int,
) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
def process_text(text: str) -> str:
collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
processed_message: Final = process_text(message)
processed_exception: Final = process_text(exception) if exception is not None else None
processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
processed_leaves: Final = tuple(
_python_redact_structured_value(key, text) if redact else text for key, text in leaves
)
changed: Final = (
processed_message != message
or processed_exception != exception
or processed_stack != stack
or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves))
)
return processed_message, processed_exception, processed_stack, processed_leaves, changed
def _render_message(record: logging.LogRecord) -> str:
try:
return record.getMessage()
except Exception:
return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT
def _render_exception(record: logging.LogRecord) -> str | None:
if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None:
return None
try:
return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info)
except Exception:
return "REDACTED"
def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool:
if _is_redacted(record):
return True
message: Final = _render_message(record)
exception: Final = _render_exception(record)
stack: Final = record.stack_info if isinstance(record.stack_info, str) else None
substituted_color: Final = _substituted_color_message(record)
extras: Final = (
tuple(
(
key,
value,
_extra_structure(
key, substituted_color if key == "color_message" and substituted_color is not None else value
),
)
for key, value in record.__dict__.items()
if key not in _STANDARD_RECORD_ATTRS
and key != _REDACTED_RECORD_ATTR
and not isinstance(value, _UNREDACTED_SCALAR_TYPES)
)
if redact
else ()
)
extra_leaves: Final = tuple(
itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras)
)
raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None
color_template: Final = record.__dict__.get("color_message")
raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None
leaves: Final = (
extra_leaves
+ (((None, raw_template),) if raw_template is not None else ())
+ (((None, raw_color),) if raw_color is not None else ())
)
candidate: Final = diagnostics.run(
lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)),
lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit),
)
processed_message, processed_exception, processed_stack, processed_leaves, _ = (
candidate
if len(candidate[3]) == len(leaves)
else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit)
)
raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template
safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message
if redact or safe_message != message:
record.msg = safe_message # rebind-ok: the Filter interface mutates the record
record.args = None # rebind-ok: the rendered message replaces interpolation inputs
if processed_exception is not None:
record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record
if processed_stack is not None:
record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record
processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
for key, original, prepared in extras:
replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
if not _scrubbing_changed_nothing(replacement, original):
setattr(record, key, replacement)
raw_color_changed: Final = (
raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color
)
if raw_color_changed and getattr(record, "color_message", None) == substituted_color:
setattr(record, "color_message", "REDACTED")
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
return True
def _redact_json_record(value: object) -> object:
prepared: Final = safe_json_structure(value)
leaves: Final = tuple(_string_leaves(None, prepared))
candidate: Final = diagnostics.run(
lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
)
replacements: Final = (
candidate
if len(candidate) == len(leaves)
else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
)
return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
class StdoutLogTruncationFilter(logging.Filter):
"""Bounds how much of an oversized log line reaches stdout.
@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter):
return True
_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
class DiagnosticProcessingFilter(StdoutLogTruncationFilter):
def filter(self, record: logging.LogRecord) -> bool:
return _process_record(
record,
base64_limit=_get_max_base64_length_stdout_log(),
text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0,
redact=_ENABLE_SECRET_REDACTION,
)
_diagnostic_filter: Final = DiagnosticProcessingFilter()
class CorrelationContextFilter(logging.Filter):
@ -633,7 +785,9 @@ class JsonFormatter(Formatter):
if record.exc_info:
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
return safe_dumps(
json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record)
)
class CorrelationPlainFormatter(logging.Formatter):
@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter):
# Create a handler with JSON formatting for exceptions
error_handler: Final = logging.StreamHandler()
error_handler.setFormatter(formatter)
error_handler.addFilter(_stdout_truncation_filter)
error_handler.addFilter(_diagnostic_filter)
error_handler.addFilter(_secret_filter)
error_handler.addFilter(_correlation_filter)
@ -734,10 +888,10 @@ verbose_logger.addHandler(handler)
# Filters attached to the logger, not the handler, survive callers swapping in their own
# handlers (JSON mode, uvicorn log config, a host app's root handler).
verbose_router_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
verbose_logger.addFilter(_stdout_truncation_filter)
verbose_router_logger.addFilter(_diagnostic_filter)
verbose_proxy_logger.addFilter(_diagnostic_filter)
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
verbose_logger.addFilter(_diagnostic_filter)
def _suppress_loggers():

View file

@ -991,6 +991,10 @@ LITELLM_EXCEPTION_TYPES: Final = [
]
class ModelNotMappedError(Exception):
pass
class BudgetExceededError(Exception):
def __init__(
self,

View file

@ -277,7 +277,7 @@ def get_s3_object_key(
start_time: datetime,
s3_file_name: str,
) -> str:
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_")
sanitized_s3_file_name: Final = s3_file_name.replace("/", "_").replace(":", "_")
configured_prefix: Final = (s3_path.rstrip("/") + "/" if s3_path else "") + prefix
date_segment: Final = start_time.strftime("%Y-%m-%d") + "/"
# we need the s3 key to include the time, so we log cache hits too

View file

@ -0,0 +1,195 @@
"""
Map any exception litellm logs to one stable ``normalized_error`` code so dashboards can cluster
failures without parsing free-text messages that embed team names, token counts, model names, etc.
"""
import re
from collections.abc import Mapping
from types import MappingProxyType
from typing import Final, Protocol, runtime_checkable
from litellm.exceptions import (
APIConnectionError,
AuthenticationError,
BadGatewayError,
BadRequestError,
BlockedPiiEntityError,
BudgetExceededError,
ContentPolicyViolationError,
ContextWindowExceededError,
GuardrailRaisedException,
InternalServerError,
MidStreamFallbackError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitType,
ServiceUnavailableError,
Timeout,
UnprocessableEntityError,
UnsupportedParamsError,
)
RATE_LIMIT_EXCEEDED: Final = "429_RATE_LIMIT_EXCEEDED"
BUDGET_EXCEEDED: Final = "429_BUDGET_EXCEEDED"
NO_HEALTHY_DEPLOYMENTS: Final = "429_NO_HEALTHY_DEPLOYMENTS"
AUTHENTICATION_FAILED: Final = "401_AUTHENTICATION_FAILED"
MODEL_ACCESS_DENIED: Final = "403_MODEL_ACCESS_DENIED"
PERMISSION_DENIED: Final = "403_PERMISSION_DENIED"
MISSING_REQUIRED_PARAMETER: Final = "400_MISSING_REQUIRED_PARAMETER"
INVALID_PARAMETER_VALUE: Final = "400_INVALID_PARAMETER_VALUE"
CONTEXT_WINDOW_EXCEEDED: Final = "400_CONTEXT_WINDOW_EXCEEDED"
CONTENT_POLICY_VIOLATION: Final = "400_CONTENT_POLICY_VIOLATION"
INVALID_REQUEST: Final = "400_INVALID_REQUEST"
RESOURCE_NOT_FOUND: Final = "404_RESOURCE_NOT_FOUND"
UPSTREAM_TIMEOUT: Final = "408_UPSTREAM_TIMEOUT"
PROVIDER_CONNECTION_ERROR: Final = "500_PROVIDER_CONNECTION_ERROR"
PROVIDER_OVERLOADED: Final = "503_PROVIDER_OVERLOADED"
PROVIDER_INTERNAL_ERROR: Final = "500_PROVIDER_INTERNAL_ERROR"
ROUTER_NO_FALLBACK: Final = "500_ROUTER_NO_FALLBACK"
ROUTER_FALLBACK_FAILURE: Final = "500_ROUTER_FALLBACK_FAILURE"
UPSTREAM_PASSTHROUGH: Final = "500_UPSTREAM_PASSTHROUGH"
UNSUPPORTED_OPERATION: Final = "500_UNSUPPORTED_OPERATION"
INTERNAL_STATE_ERROR: Final = "500_INTERNAL_STATE_ERROR"
UNCLASSIFIED: Final = "UNCLASSIFIED"
@runtime_checkable
class _HasProxyErrorType(Protocol):
type: str
_MESSAGE_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
(
re.compile(r"budget has been exceeded|max budget|exceeded.*budget|crossed budget", re.IGNORECASE),
BUDGET_EXCEEDED,
),
(re.compile(r"no healthy deployments?|no deployments available", re.IGNORECASE), NO_HEALTHY_DEPLOYMENTS),
(re.compile(r"not allowed to access model due to tags configuration", re.IGNORECASE), MODEL_ACCESS_DENIED),
(re.compile(r"upstream passthrough request failed", re.IGNORECASE), UPSTREAM_PASSTHROUGH),
(re.compile(r"is not supported for provider|not implemented", re.IGNORECASE), UNSUPPORTED_OPERATION),
(
re.compile(r"context window|context length|(prompt|input) is too long|tokens? ?> ?\d+ ?maximum", re.IGNORECASE),
CONTEXT_WINDOW_EXCEEDED,
),
(re.compile(r"missing required parameter|field required", re.IGNORECASE), MISSING_REQUIRED_PARAMETER),
(re.compile(r"overloaded|unable to process your request", re.IGNORECASE), PROVIDER_OVERLOADED),
(
re.compile(
r"connection error|APIConnectionError|TransferEncodingError|payload is not completed|connection reset"
r"|peer closed connection|incomplete chunked read",
re.IGNORECASE,
),
PROVIDER_CONNECTION_ERROR,
),
(re.compile(r"timed? ?out", re.IGNORECASE), UPSTREAM_TIMEOUT),
)
_ROUTER_WRAPPER_PATTERNS: Final[tuple[tuple[re.Pattern[str], str], ...]] = (
(re.compile(r"no fallback model group found", re.IGNORECASE), ROUTER_NO_FALLBACK),
(re.compile(r"error doing the fallback|MidStreamFallbackError", re.IGNORECASE), ROUTER_FALLBACK_FAILURE),
)
_PROXY_ERROR_TYPE_MAP: Final[Mapping[str, str]] = MappingProxyType(
{
"budget_exceeded": BUDGET_EXCEEDED,
"auth_error": AUTHENTICATION_FAILED,
"expired_key": AUTHENTICATION_FAILED,
"token_not_found_in_db": AUTHENTICATION_FAILED,
"auth_provider_unavailable": AUTHENTICATION_FAILED,
"key_model_access_denied": MODEL_ACCESS_DENIED,
"team_model_access_denied": MODEL_ACCESS_DENIED,
"user_model_access_denied": MODEL_ACCESS_DENIED,
"org_model_access_denied": MODEL_ACCESS_DENIED,
"project_model_access_denied": MODEL_ACCESS_DENIED,
"agent_model_access_denied": MODEL_ACCESS_DENIED,
"key_vector_store_access_denied": PERMISSION_DENIED,
"team_vector_store_access_denied": PERMISSION_DENIED,
"org_vector_store_access_denied": PERMISSION_DENIED,
"tool_access_denied": PERMISSION_DENIED,
"team_member_permission_error": PERMISSION_DENIED,
"not_found_error": RESOURCE_NOT_FOUND,
}
)
_STATUS_CODE_MAP: Final[Mapping[str, str]] = MappingProxyType(
{
"400": INVALID_REQUEST,
"401": AUTHENTICATION_FAILED,
"403": PERMISSION_DENIED,
"404": RESOURCE_NOT_FOUND,
"408": UPSTREAM_TIMEOUT,
"422": INVALID_PARAMETER_VALUE,
"429": RATE_LIMIT_EXCEEDED,
"500": PROVIDER_INTERNAL_ERROR,
"502": PROVIDER_INTERNAL_ERROR,
"503": PROVIDER_OVERLOADED,
"504": UPSTREAM_TIMEOUT,
}
)
_INTERNAL_STATE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = (
TypeError,
KeyError,
AttributeError,
IndexError,
RuntimeError,
AssertionError,
ZeroDivisionError,
)
_CLASS_CODE_TABLE: Final[tuple[tuple[tuple[type[BaseException], ...], str], ...]] = (
((AuthenticationError,), AUTHENTICATION_FAILED),
((PermissionDeniedError,), PERMISSION_DENIED),
((ContextWindowExceededError,), CONTEXT_WINDOW_EXCEEDED),
((ContentPolicyViolationError, GuardrailRaisedException, BlockedPiiEntityError), CONTENT_POLICY_VIOLATION),
((UnsupportedParamsError,), INVALID_PARAMETER_VALUE),
((NotFoundError,), RESOURCE_NOT_FOUND),
((Timeout,), UPSTREAM_TIMEOUT),
((MidStreamFallbackError,), ROUTER_FALLBACK_FAILURE),
((APIConnectionError,), PROVIDER_CONNECTION_ERROR),
((ServiceUnavailableError,), PROVIDER_OVERLOADED),
((InternalServerError, BadGatewayError), PROVIDER_INTERNAL_ERROR),
((BadRequestError, UnprocessableEntityError), INVALID_REQUEST),
((NotImplementedError,), UNSUPPORTED_OPERATION),
)
def _classify_by_message(message: str, patterns: tuple[tuple[re.Pattern[str], str], ...]) -> str | None:
return next((code for pattern, code in patterns if pattern.search(message)), None)
def _classify_by_class(exc: Exception) -> str | None:
if isinstance(exc, BudgetExceededError):
return BUDGET_EXCEEDED
if isinstance(exc, RateLimitError):
return BUDGET_EXCEEDED if exc.rate_limit_type == RateLimitType.BUDGET.value else RATE_LIMIT_EXCEEDED
for exc_types, code in _CLASS_CODE_TABLE:
if isinstance(exc, exc_types):
return code
if isinstance(exc, _INTERNAL_STATE_EXCEPTIONS):
return INTERNAL_STATE_ERROR
return None
def normalize_error(exc: Exception | None, status_code: str, message: str) -> str | None:
"""
Return a stable cluster key for ``exc``. ``status_code`` and ``message`` are the values
``get_error_information`` already extracted, so the same exception always yields the same code.
"""
if exc is None:
return None
proxy_type: Final = exc.type if isinstance(exc, _HasProxyErrorType) else None
by_proxy_type: Final = _PROXY_ERROR_TYPE_MAP.get(proxy_type) if isinstance(proxy_type, str) else None
if by_proxy_type is not None:
return by_proxy_type
by_message: Final = _classify_by_message(message, _MESSAGE_PATTERNS)
if by_message is not None:
return by_message
by_class: Final = _classify_by_class(exc)
if by_class is not None:
return by_class
by_router_wrapper: Final = _classify_by_message(message, _ROUTER_WRAPPER_PATTERNS)
if by_router_wrapper is not None:
return by_router_wrapper
return _STATUS_CODE_MAP.get(status_code, UNCLASSIFIED)

View file

@ -76,6 +76,7 @@ from litellm.litellm_core_utils.core_helpers import (
reconstruct_model_name,
set_response_cost_in_hidden_params,
)
from litellm.litellm_core_utils.error_normalization import normalize_error
from litellm.litellm_core_utils.get_litellm_params import get_litellm_params
from litellm.litellm_core_utils.internal_call_metadata import (
MODEL_ACCESS_GROUP_METADATA_KEY,
@ -6100,6 +6101,7 @@ class StandardLoggingPayloadSetup:
error_budget_entity_id=budget_error.entity_id if budget_error else None,
error_budget_limit=budget_error.max_budget if budget_error else None,
error_budget_spend=budget_error.current_cost if budget_error else None,
normalized_error=normalize_error(original_exception, error_status, error_message),
)
@staticmethod

View file

@ -3,10 +3,30 @@ from typing import Final
import litellm
from litellm import verbose_logger
from ...litellm_core_utils.get_llm_provider_logic import get_llm_provider
from ...litellm_core_utils.get_llm_provider_logic import (
declared_authenticating_provider,
get_llm_provider,
)
from ...types.router import LiteLLM_Params
def _api_base_without_login(provider: str) -> str | None:
if provider == "github_copilot":
return litellm.GithubCopilotConfig().api_base_without_login()
if provider == "chatgpt":
return litellm.ChatGPTConfig().api_base_without_login()
return None
def _provider_default_api_base(model: str, custom_llm_provider: str | None, stream: bool) -> str | None:
if custom_llm_provider == "gemini":
action: Final = "streamGenerateContent" if stream else "generateContent"
return f"https://generativelanguage.googleapis.com/v1beta/models/{model}:{action}"
if custom_llm_provider == "openai":
return "https://api.openai.com"
return None
def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | None:
"""
Returns the api base used for calling the model.
@ -42,6 +62,9 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No
if litellm.model_alias_map and model in litellm.model_alias_map:
model = litellm.model_alias_map[model]
declared: Final = declared_authenticating_provider(model, _optional_params.custom_llm_provider)
if declared is not None:
return _api_base_without_login(declared)
try:
(
model,
@ -83,16 +106,4 @@ def get_api_base(model: str, optional_params: dict | LiteLLM_Params) -> str | No
_api_base = f"{_optional_params.vertex_location}-aiplatform.googleapis.com/v1/projects/{_optional_params.vertex_project}/locations/{_optional_params.vertex_location}/publishers/google/models/{model}:generateContent"
return _api_base
if custom_llm_provider is None:
return None
if custom_llm_provider == "gemini":
if stream:
_api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:streamGenerateContent"
else:
_api_base = f"https://generativelanguage.googleapis.com/v1beta/models/{model}:generateContent"
return _api_base
elif custom_llm_provider == "openai":
_api_base = "https://api.openai.com"
return _api_base
return None
return _provider_default_api_base(model, custom_llm_provider, stream)

View file

@ -1657,7 +1657,7 @@ def get_file_ids_from_messages(messages: list[AllMessageValues]) -> list[str]:
if isinstance(content, str):
continue
for c in content:
if c["type"] == "file":
if isinstance(c, dict) and c["type"] == "file":
file_object = cast(ChatCompletionFileObject, c)
file_object_file_field = file_object.get("file")
if not isinstance(file_object_file_field, dict):

View file

@ -10,6 +10,7 @@ import re
from typing import Final
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
from litellm.rust_bridge import diagnostics
REDACTED: Final = "REDACTED"
@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]":
_SECRET_RE: Final = _build_secret_patterns()
def _python_redact_string(value: str) -> str:
return _SECRET_RE.sub(REDACTED, value)
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(REDACTED, value)
return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value))
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile(
_TRACEBACK_MARKER: Final = "Traceback (most recent call last):"
def redact_internal_details(value: str) -> str:
def _python_redact_internal_details(value: str) -> str:
"""Drop an embedded traceback and scrub filesystem paths and internal hostnames,
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
marker_index: Final = value.find(_TRACEBACK_MARKER)
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback))
def redact_structured_value(key: str | None, value: str) -> str:
def redact_internal_details(value: str) -> str:
return diagnostics.run(
lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value)
)
def _python_redact_structured_value(key: str | None, value: str) -> str:
"""Scrub *value* as it appeared under *key* inside a structured record.
redact_string() replaces a whole ``key: value`` span with REDACTED, which is
@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str:
repr would, so the key-name patterns still fire, but collapses only the value
so the caller's structure survives.
"""
scrubbed: Final = redact_string(value)
scrubbed: Final = _python_redact_string(value)
if scrubbed != value or key is None:
return scrubbed
rendered: Final = f"'{key}': '{value}'"
return REDACTED if redact_string(rendered) != rendered else value
return REDACTED if _python_redact_string(rendered) != rendered else value
def redact_structured_value(key: str | None, value: str) -> str:
return diagnostics.run(
lambda native: native.redact_structured_text(key, value),
lambda: _python_redact_structured_value(key, value),
)

View file

@ -314,7 +314,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
_message_content = message.get("content")
if _message_content is not None and isinstance(_message_content, list):
for content in _message_content:
if "cache_control" in content:
if isinstance(content, dict) and "cache_control" in content:
return True
return False
@ -359,7 +359,7 @@ class AnthropicModelInfo(BaseLLMModelInfo):
for message in messages:
if "content" in message and message["content"] is not None and isinstance(message["content"], list):
for content in message["content"]:
if "type" in content and content["type"] != "text":
if isinstance(content, dict) and "type" in content and content["type"] != "text":
return True
return False

View file

@ -1593,7 +1593,7 @@ def _resolve_s3_setting(
source.get(param_name) for source in (litellm_params, optional_params) if source is not None
)
explicit: Final = next((value for value in candidates if isinstance(value, str) and value), None)
return explicit or get_secret_str(env_var)
return explicit or get_secret_str(env_var) or None
class CommonBatchFilesUtils:

View file

@ -23,6 +23,9 @@ class ChatGPTConfig(OpenAIConfig):
super().__init__()
self.authenticator = Authenticator()
def api_base_without_login(self) -> str:
return self.authenticator.get_api_base()
def _get_openai_compatible_provider_info(
self,
model: str,
@ -30,7 +33,7 @@ class ChatGPTConfig(OpenAIConfig):
api_key: str | None,
custom_llm_provider: str,
) -> tuple[str | None, str | None, str]:
dynamic_api_base: Final = self.authenticator.get_api_base()
dynamic_api_base: Final = self.api_base_without_login()
try:
dynamic_api_key: Final = self.authenticator.get_access_token()
except GetAccessTokenError as e:

View file

@ -31,6 +31,14 @@ class GithubCopilotConfig(OpenAIConfig):
super().__init__()
self.authenticator = Authenticator()
def api_base_without_login(self, api_base: str | None = None) -> str:
return (
api_base
or self.authenticator.get_api_base()
or os.getenv("GITHUB_COPILOT_API_BASE")
or DEFAULT_GITHUB_COPILOT_API_BASE
)
def _get_openai_compatible_provider_info(
self,
model: str,
@ -38,12 +46,7 @@ class GithubCopilotConfig(OpenAIConfig):
api_key: str | None,
custom_llm_provider: str,
) -> tuple[str | None, str | None, str]:
dynamic_api_base: Final = (
api_base
or self.authenticator.get_api_base()
or os.getenv("GITHUB_COPILOT_API_BASE")
or DEFAULT_GITHUB_COPILOT_API_BASE
)
dynamic_api_base: Final = self.api_base_without_login(api_base)
try:
dynamic_api_key: Final = self.authenticator.get_api_key()
except GetAPIKeyError as e:

View file

@ -1,3 +1,5 @@
import base64
import io
from typing import Any, Final
import httpx
@ -11,37 +13,35 @@ class OllamaError(BaseLLMException):
super().__init__(status_code=status_code, message=message, headers=headers)
def _convert_image(image):
"""
Convert image to base64 encoded image if not already in base64 format
_JPEG_AND_PNG_SIGNATURES: Final = (b"\xff\xd8\xff", b"\x89PNG\r\n\x1a\n")
If image is already in base64 format AND is a jpeg/png, return it
If image is not JPEG/PNG, convert it to JPEG base64 format
"""
import base64
import io
def _reencode_as_jpeg(raw_image: bytes, original: str) -> str:
try:
from PIL import Image
except Exception:
raise Exception("ollama image conversion failed please run `pip install Pillow`")
orig: Final = image
if image.startswith("data:"):
image = image.split(",")[-1]
try:
image_data: Final = Image.open(io.BytesIO(base64.b64decode(image)))
if image_data.format in ["JPEG", "PNG"]:
return image
picture: Final = Image.open(io.BytesIO(raw_image))
except Exception:
return orig
return original
jpeg_image: Final = io.BytesIO()
image_data.convert("RGB").save(jpeg_image, "JPEG")
jpeg_image.seek(0)
picture.convert("RGB").save(jpeg_image, "JPEG")
return base64.b64encode(jpeg_image.getvalue()).decode("utf-8")
def _convert_image(image: str) -> str:
payload: Final = image.split(",")[-1] if image.startswith("data:") else image
try:
raw_image: Final = base64.b64decode(payload)
except ValueError:
return image
if raw_image.startswith(_JPEG_AND_PNG_SIGNATURES):
return payload
return _reencode_as_jpeg(raw_image, original=image)
from litellm.llms.base_llm.base_utils import BaseLLMModelInfo

View file

@ -25314,8 +25314,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"cache_read_input_token_cost_batches": 1e-07
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -25401,8 +25400,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"cache_read_input_token_cost_batches": 2.5e-08
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -25482,8 +25480,7 @@
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_vision": true
},
"gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
@ -25593,8 +25590,7 @@
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
"cache_read_input_token_cost_batches": 1.25e-08
"input_cost_per_audio_token_batches": 2.5e-07
},
"gemini-3.5-flash-lite": {
"deprecation_date": "2027-07-21",
@ -25652,8 +25648,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 1.5e-08
"google_maps_grounding_cost_per_query": 0.014
},
"deep-research-pro-preview-12-2025": {
"cache_read_input_token_cost": 2e-07,
@ -25689,8 +25684,7 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"cache_read_input_token_cost_batches": 1e-07
"supports_web_search": true
},
"gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
@ -26321,8 +26315,7 @@
"output_cost_per_token_batches": 4.5e-06,
"input_cost_per_token_flex": 7.5e-07,
"output_cost_per_token_flex": 4.5e-06,
"cache_read_input_token_cost_flex": 7.5e-08,
"cache_read_input_token_cost_batches": 7.5e-08
"cache_read_input_token_cost_flex": 7.5e-08
},
"vertex_ai/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
@ -26380,8 +26373,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
@ -26440,8 +26432,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
@ -26500,8 +26491,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
@ -28250,8 +28240,7 @@
"output_cost_per_token_batches": 4.5e-06,
"input_cost_per_token_flex": 7.5e-07,
"output_cost_per_token_flex": 4.5e-06,
"cache_read_input_token_cost_flex": 7.5e-08,
"cache_read_input_token_cost_batches": 7.5e-08
"cache_read_input_token_cost_flex": 7.5e-08
},
"gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
@ -28309,8 +28298,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
@ -28369,8 +28357,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
@ -28429,8 +28416,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
@ -33110,8 +33096,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 6.25e-08
"supports_minimal_reasoning_effort": true
},
"gpt-5-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -33292,8 +33277,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_minimal_reasoning_effort": true
},
"gpt-5-nano": {
"cache_read_input_token_cost": 5e-09,
@ -33392,8 +33376,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 2.5e-09
"supports_minimal_reasoning_effort": true
},
"gpt-image-1": {
"cache_read_input_token_cost": 1.25e-06,
@ -47531,8 +47514,7 @@
"output_cost_per_token_flex": 6e-06,
"output_cost_per_token_priority": 2.16e-05,
"supports_reasoning": false,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 1e-07
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -47570,8 +47552,7 @@
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_reasoning": false,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 2.5e-08
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -47627,8 +47608,7 @@
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_vision": true
},
"vertex_ai/gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
@ -47739,8 +47719,7 @@
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
"cache_read_input_token_cost_batches": 1.25e-08
"input_cost_per_audio_token_batches": 2.5e-07
},
"vertex_ai/gemini-3.5-flash-lite": {
"deprecation_date": "2027-07-21",
@ -47799,8 +47778,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 1.5e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"cache_read_input_token_cost": 2e-07,
@ -47817,8 +47795,7 @@
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 1e-07
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/jamba-1.5": {
"input_cost_per_token": 2e-07,

View file

@ -42,6 +42,7 @@ from litellm.proxy._types import (
SpecialMCPServerName,
SpecialMCPServerNames,
UserAPIKeyAuth,
hash_token,
user_api_key_has_admin_view,
)
from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
@ -182,6 +183,20 @@ def _is_litellm_auth_admission_error(exc: Exception) -> bool:
return False
def _explicit_credential_matches_envelope(
explicit_auth: UserAPIKeyAuth,
presented_token: str,
identity: EnvelopeIdentity,
) -> bool:
"""Match the stored key hash or user ID, including token-only mapped JWT keys."""
match identity.subject_type:
case "key_hash":
return identity.subject in (hash_token(presented_token), explicit_auth.token)
case "user_id":
return explicit_auth.user_id is not None and explicit_auth.user_id == identity.subject
return assert_never(identity.subject_type)
def _has_client_supplied_mcp_auth(
mcp_auth_header: str | None,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
@ -475,6 +490,31 @@ class MCPRequestHandler:
# Only OAuth metadata routes registered under /.well-known/ are public.
if request_route.startswith("/.well-known/"):
validated_user_api_key_auth = UserAPIKeyAuth()
elif (
has_explicit_litellm_key
and oauth2_headers
and is_bridge_envelope_shaped(oauth2_headers["Authorization"])
and (
dual_bridge_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
path=request_route,
mcp_servers=mcp_servers,
client_ip=IPAddressUtils.get_mcp_client_ip(request),
)
)
is not None
):
(
validated_user_api_key_auth,
mcp_server_auth_headers,
) = await MCPRequestHandler._admit_dcr_bridge_dual_credential(
server=dual_bridge_target.server,
requested_name=dual_bridge_target.requested_name,
authorization_value=oauth2_headers["Authorization"],
litellm_api_key=litellm_api_key,
mcp_server_auth_headers=mcp_server_auth_headers,
request=request,
route=request_route,
)
elif has_explicit_litellm_key:
# An explicit x-litellm-api-key is always a LiteLLM credential, even
# for a delegated server, so validate it: identity / spend / rate
@ -782,6 +822,45 @@ class MCPRequestHandler:
higher-priority alias slot, pairing the admitted identity with an attacker's upstream
credential; the alias-keyed injection overwrites any such caller value.
"""
result: Final = await MCPRequestHandler._open_dcr_bridge_envelope(
server=server,
requested_name=requested_name,
authorization_value=authorization_value,
request=request,
route=route,
)
header_key: Final = server.alias or server.server_name
if header_key is None:
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
admitted: Final = await MCPRequestHandler._reload_admitted_principal(result.identity)
await MCPRequestHandler._enforce_admitted_live_policy(admitted=admitted, request=request, route=route)
injected: Final = { # mutable-ok: mcp_server_auth_headers contract requires concrete dicts
header_key: { # mutable-ok: concrete dict header payload
"Authorization": result.upstream_authorization.get_secret_value()
}
}
new_headers: Final = { # mutable-ok: merged header map must stay a concrete dict
**(mcp_server_auth_headers or {}), # mutable-ok: empty-dict fallback for the merge
**injected,
}
return admitted, new_headers
@staticmethod
async def _open_dcr_bridge_envelope(
server: MCPServer,
requested_name: str,
authorization_value: str,
request: Request,
route: str,
) -> BridgeEnvelopeAdmitted:
"""Open a bridge envelope after the pre-DB gates, or fail closed with the scope's challenge.
Shared by the envelope-only arm (:meth:`_admit_dcr_bridge_delegate`) and the dual-credential
arm (:meth:`_admit_dcr_bridge_dual_credential`): both require master_key, run the same
proxy-wide pre-DB checks the standard pipeline applies before any key lookup, and resolve
the envelope's crypto. Returns only the ``BridgeEnvelopeAdmitted`` result; an invalid,
expired, tampered, or non-envelope value raises the requested scope's ``invalid_token``
challenge instead."""
from litellm.proxy.proxy_server import master_key
if not master_key:
@ -793,20 +872,67 @@ class MCPRequestHandler:
result: Final = resolve_bridge_envelope(authorization_value, keys, datetime.now(timezone.utc), server.server_id)
match result:
case BridgeEnvelopeAdmitted():
header_key: Final = server.alias or server.server_name
if header_key is None:
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
admitted: Final = await MCPRequestHandler._reload_admitted_principal(result.identity)
await MCPRequestHandler._enforce_admitted_live_policy(admitted=admitted, request=request, route=route)
injected: Final = {header_key: {"Authorization": result.upstream_authorization.get_secret_value()}}
new_headers: Final = {**(mcp_server_auth_headers or {}), **injected}
return admitted, new_headers
return result
case BridgeEnvelopeInvalid() | NotBridgeEnvelope():
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
requested_name=requested_name, request=request
)
case _:
assert_never(result)
return assert_never(result)
@staticmethod
async def _admit_dcr_bridge_dual_credential(
server: MCPServer,
requested_name: str,
authorization_value: str,
litellm_api_key: str,
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
request: Request,
route: str,
) -> tuple[UserAPIKeyAuth, dict[str, dict[str, str]] | None]:
"""Admit a request carrying BOTH an explicit litellm credential and a bridge envelope.
MCP clients send ``x-litellm-api-key`` on every request, including the ``tools/list`` that
follows the ``/{server}/token`` mint, so the envelope arrives alongside the key rather than
alone. The explicit credential is validated first (its own pipeline, so a bad key keeps the
normal 401/403), then the envelope is opened and its sealed identity must match the explicit
credential's principal — a mismatch is a 403, never a fallback onto either credential alone.
On a match the explicit credential's ``UserAPIKeyAuth`` is the admission context (key
permissions, budgets, rate limits) and the sealed upstream token is injected under the
server's per-server auth-header key, while the leak-defense chokepoint strips the envelope
``Authorization`` itself from egress."""
presented_token: Final = _get_bearer_token_or_received_api_key(litellm_api_key)
explicit_auth: Final = await user_api_key_auth(api_key=f"Bearer {presented_token}", request=request)
result: Final = await MCPRequestHandler._open_dcr_bridge_envelope(
server=server,
requested_name=requested_name,
authorization_value=authorization_value,
request=request,
route=route,
)
if not _explicit_credential_matches_envelope(
explicit_auth=explicit_auth,
presented_token=presented_token,
identity=result.identity,
):
raise HTTPException(
status_code=403,
detail={ # mutable-ok: HTTPException detail payload requires a concrete dict
"error": "oauth_principal_mismatch"
},
)
header_key: Final = server.alias or server.server_name
if header_key is None:
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
injected: Final = { # mutable-ok: mcp_server_auth_headers contract requires concrete dicts
header_key: { # mutable-ok: concrete dict header payload
"Authorization": result.upstream_authorization.get_secret_value()
}
}
new_headers: Final = { # mutable-ok: merged header map must stay a concrete dict
**(mcp_server_auth_headers or {}), # mutable-ok: empty-dict fallback for the merge
**injected,
}
return explicit_auth, new_headers
@staticmethod
async def _admit_dcr_bridge_authorization(
@ -1091,9 +1217,15 @@ class MCPRequestHandler:
project, org, and budget state are NOT re-checked here; the caller runs the admitted
identity through ``_enforce_admitted_live_policy`` for those.
"""
from litellm.proxy._experimental.mcp_server.bridge_token_flow import (
master_key_admin_auth, # noqa: PLC0415 # inline import avoids a module-load circular import
)
from litellm.proxy.auth.auth_checks import get_key_object
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
admin: Final = master_key_admin_auth(key_hash)
if admin is not None:
return admin
if prisma_client is None:
raise HTTPException(status_code=500, detail="Server misconfigured: no database connection")
try:

View file

@ -209,6 +209,31 @@ async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyR
return await _reload_active_key_by_hash(hash_token(token))
def master_key_admin_auth(key_hash: str) -> "UserAPIKeyAuth | None":
from litellm.constants import ( # noqa: PLC0415 # inline import avoids a module-load circular import
LITELLM_PROXY_MASTER_KEY_ALIAS,
)
from litellm.proxy._types import ( # noqa: PLC0415 # inline import avoids a module-load circular import
LitellmUserRoles,
UserAPIKeyAuth,
hash_token,
)
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # inline import avoids a module-load circular import
litellm_proxy_admin_name,
master_key,
)
if not master_key or not secrets.compare_digest(key_hash, hash_token(master_key)):
return None
auth: Final = UserAPIKeyAuth(
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
user_role=LitellmUserRoles.PROXY_ADMIN,
user_id=litellm_proxy_admin_name,
)
auth.via_virtual_key = True
return auth
async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResolutionFailure":
"""Reload the live key record for ``key_hash`` (cache first, then DB) and gate it on active state,
returning the resolved key or a precise failure. Shared by the token request's presented-key
@ -233,6 +258,8 @@ async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResol
user_api_key_cache,
)
if (admin := master_key_admin_auth(key_hash)) is not None:
return _ResolvedKey(key_hash=key_hash, key=admin)
if prisma_client is None:
return "unresolvable"
try:
@ -475,7 +502,7 @@ async def _resolve_jwt_auth(
proxy_logging_obj=proxy_logging_obj,
)
if isinstance(mapped, UserAPIKeyAuth):
return None if await _key_owner_scim_deactivated(mapped) or not _active_key_user_id(mapped) else mapped
return None if await _key_owner_scim_deactivated(mapped) or not _key_is_active(mapped) else mapped
if mapped is not None:
return None
if write_route is None:
@ -593,6 +620,7 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
_BridgeMintError = Literal[
"no_identity",
"jwt_client_policy_unsupported",
"invalid_refresh",
"identity_unavailable",
"identity_faulted",
@ -633,6 +661,13 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
"this server issues a gateway-bound credential; complete the interactive sign-in, or "
"send a litellm credential (x-litellm-api-key or Authorization) on the token request",
)
case "jwt_client_policy_unsupported":
status, code, desc = (
400,
"invalid_request",
"JWT bridge minting is not supported with a claim-based MCP client allowlist; "
"the bridge credential cannot preserve the signed client identity",
)
case "invalid_refresh":
status, code, desc = (
400,
@ -736,11 +771,18 @@ async def _prepare_bridge_mint(
Two identity sources, one envelope. The interactive DCR client authenticates via SSO at the bridged
authorize, so its identity arrives as ``bridge_identity`` (the user recovered from the gateway
authorization code) and mints a user subject. The scripted two-header client presents a litellm key
on the token request instead, so its identity is the active key's hash and mints a key_hash subject.
A missing or invalid presented key keeps its resolution origin so the mapper statuses it truthfully;
neither source present is ``no_identity``. The refresh_token grant has its own phase-1
authorization code) and mints a user subject. The scripted two-header client presents a litellm
credential (a virtual key or a JWT) on the token request instead: a key mints a key_hash subject,
while a JWT resolves through the same auth path as admission and mints a key_hash subject when it
maps to a virtual key. An unmapped JWT is rejected because a user subject cannot preserve its
JWT-specific authorization restrictions. A JWT client-claim allowlist also prevents JWT minting:
the envelope cannot retain the signed client identity for subsequent allowlist checks. A missing or invalid
presented key keeps its resolution origin so the mapper statuses it truthfully; neither source
present is ``no_identity``. The refresh_token grant has its own phase-1
(:func:`_prepare_bridge_refresh`), which recovers identity from the presented refresh envelope."""
from litellm.proxy._experimental.mcp_server.client_allowlist import ( # noqa: PLC0415 # keep mint policy dependencies local
load_mcp_client_allowlist,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import
envelope_keys_from_master_key,
)
@ -748,7 +790,10 @@ async def _prepare_bridge_mint(
key_hash_identity,
user_identity,
)
from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # inline import avoids a module-load circular import
general_settings,
master_key,
)
@ -758,6 +803,16 @@ async def _prepare_bridge_mint(
if bridge_identity is not None:
identity = user_identity(server_id=mcp_server.server_id, user_id=bridge_identity.litellm_user_id)
return _BridgeMintReady(identity=identity, keys=keys)
presented_token: Final = _litellm_key_from_request(request)
if presented_token is not None and JWTHandler.is_jwt(presented_token):
client_allowlist: Final = load_mcp_client_allowlist(general_settings)
if client_allowlist is not None and client_allowlist.jwt_field is not None:
return "jwt_client_policy_unsupported"
resolved_jwt: Final = await _resolve_jwt_auth(request, presented_token, None)
if isinstance(resolved_jwt, UserAPIKeyAuth) and resolved_jwt.token:
identity = key_hash_identity(server_id=mcp_server.server_id, key_hash=resolved_jwt.token)
return _BridgeMintReady(identity=identity, keys=keys)
return "no_identity"
resolved: Final = await _resolve_active_litellm_key(request)
if not isinstance(resolved, _ResolvedKey):
return _key_resolution_failure_to_mint_error(resolved)

View file

@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum):
# proxy admin, or team admin naming their own team via team_id
"/auto_router/test_routing",
"/auto_router/validate_complexity_router_config",
"/auto_router/availability",
# Per-session auto-router read - the endpoint scopes the row to the caller's own key hash
"/auto_router/session",
"/cost/predict-cache",

View file

@ -1606,13 +1606,21 @@ class DBSpendUpdateWriter:
proxy_logging_obj: ProxyLogging,
) -> None:
transactions: Final = await queue.flush_and_get_aggregated_daily_spend_update_transactions()
try:
await commit(
commit_task: Final = asyncio.ensure_future(
commit(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
daily_spend_transactions=cast(dict[str, _DailySpendTransactionT], transactions),
)
)
try:
await asyncio.shield(commit_task)
except asyncio.CancelledError:
commit_task.cancel()
if transactions:
await queue.add_update(transactions)
raise
except Exception as e: # noqa: BLE001 # whatever failed here, the other tables must still flush
if not transactions:
return
@ -1839,14 +1847,18 @@ class DBSpendUpdateWriter:
if not daily_tag_spend_update_transactions:
return
try:
await DBSpendUpdateWriter.update_daily_tag_spend(
commit_task: Final = asyncio.ensure_future(
DBSpendUpdateWriter.update_daily_tag_spend(
n_retry_times=n_retry_times,
prisma_client=prisma_client,
proxy_logging_obj=proxy_logging_obj,
daily_spend_transactions=daily_tag_spend_update_transactions,
)
except Exception:
)
try:
await asyncio.shield(commit_task)
except BaseException: # noqa: BLE001 # a cancel must restore the drained rows before its rollback returns
commit_task.cancel()
await self.redis_update_buffer.restore_transactions_to_redis(
daily_tag_spend_update_transactions=daily_tag_spend_update_transactions,
)
@ -2368,7 +2380,8 @@ class DBSpendUpdateWriter:
table=table, transactions=tuple(transactions_to_process.values())
)
sql, params = build_bulk_upsert(table=table, batch=merged_batch)
await prisma_client.db.execute_raw(sql, *params)
async with _spend_update_tx(prisma_client) as transaction:
await transaction.execute_raw(sql, *params)
except Exception as batch_error:
if _spend_commit_failure_is_requeue_safe(batch_error):
spend_log_error(

View file

@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import (
)
from litellm.types.management_endpoints.auto_router_endpoints import (
SHADOW_EVAL_TURN_VALVE,
AutoRouterAvailabilityRequest,
AutoRouterAvailabilityResponse,
AutoRouterBenchmarkGroup,
AutoRouterBenchmarksResponse,
AutoRouterBenchmarkTotals,
@ -391,6 +393,54 @@ async def validate_complexity_router_config(
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
@router.post(
"/auto_router/availability",
tags=["model management"], # mutable-ok: FastAPI requires a list
response_model=AutoRouterAvailabilityResponse,
)
async def get_auto_router_availability(
data: AutoRouterAvailabilityRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> AutoRouterAvailabilityResponse:
from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability
from litellm.proxy.proxy_server import (
_license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate
heuristic_v1_tuning_baselines,
llm_router,
proxy_config,
)
member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id)
rows: Final = proxy_config.auto_router_db_catalog
if rows is None or llm_router is None:
raise HTTPException(status_code=503, detail="Auto-router availability is unavailable")
saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None)
if data.saved_model_id is not None:
if saved is None:
raise HTTPException(status_code=404, detail="Saved auto router is unavailable")
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and (
saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id)
):
raise HTTPException(status_code=403, detail="Cannot check another user's auto router")
existing: Final = saved.deployment if saved is not None else None
others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments())
candidate: Final = MappingProxyType(
{
"litellm_params": MappingProxyType(
{"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config}
),
"model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}),
}
)
return auto_router_availability(
others=others,
existing=existing,
candidate=candidate,
baselines=heuristic_v1_tuning_baselines,
limit=_license_check.auto_router_capability_limit(),
)
async def _resolve_saved_routing_test(
data: AutoRouterRoutingTestRequest,
user_api_key_dict: UserAPIKeyAuth,

View file

@ -2741,6 +2741,10 @@ async def _resolve_team_org_filter(
async def ui_view_users(
user_id: str | None = fastapi.Query(default=None, description="User ID in the request parameters"),
user_email: str | None = fastapi.Query(default=None, description="User email in the request parameters"),
search: str | None = fastapi.Query(
default=None,
description="Combined search: matches users whose 'user_id' or 'user_email' contains the value (case-insensitive).",
),
team_id: str | None = fastapi.Query(
default=None,
description="Team ID — used when a team admin searches for users to add to their team",
@ -2750,7 +2754,7 @@ async def ui_view_users(
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
Filter users based on partial match of user_id or email with pagination.
Filter users based on partial match of user_id or email, or combined ``search``, with pagination.
Behaviour depends on the ``scope_user_search_to_org`` UI-setting flag
(stored in the ``litellm_uisettings`` table):
@ -2802,9 +2806,15 @@ async def ui_view_users(
if org_filter_ids is not None:
where_conditions["organization_memberships"] = {"some": {"organization_id": {"in": org_filter_ids}}}
where: Final[Mapping[str, object]] = { # mutable-ok: prisma serializes `where`, keep it a plain dict
key: value
for key, value in (*where_conditions.items(), *_user_search_where(search).items())
if value is not None
}
# Query users with pagination and filters
users: Final = await _user_table(prisma_client).find_many(
where=where_conditions,
where=where,
skip=skip,
take=page_size,
order={"created_at": "desc"},

View file

@ -1782,10 +1782,11 @@ async def delete_team_models(
# Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
# gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
# onto this pod. The lock orders the eviction after any in-flight reconcile.
if llm_router is not None:
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config
async with MODEL_RECONCILE_LOCK:
async with MODEL_RECONCILE_LOCK:
proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids))
if llm_router is not None:
for model_id in deleted_model_ids:
llm_router.delete_deployment(id=model_id)
@ -2194,6 +2195,7 @@ async def delete_model(
llm_router,
premium_user,
prisma_client,
proxy_config,
proxy_logging_obj,
store_model_in_db,
user_api_key_cache,
@ -2245,8 +2247,9 @@ async def delete_model(
# this pod serving a model the database no longer has, until the next
# reconcile. Taking the lock orders this eviction after any such in-flight
# reconcile's re-add, so the eviction is the last word.
if llm_router is not None:
async with MODEL_RECONCILE_LOCK:
async with MODEL_RECONCILE_LOCK:
proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id}))
if llm_router is not None:
llm_router.delete_deployment(id=model_info.id)
# Runs after the row delete so the sibling check sees post-delete state.

View file

@ -0,0 +1,148 @@
from collections.abc import Mapping, Sequence
from copy import deepcopy
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, Json, TypeAdapter, ValidationError
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
from litellm.router_utils.auto_router_model_naming import (
GATED_AUTO_ROUTER_CAPABILITIES,
capability_limit_violation,
classify_strategy_router_model,
count_capability_routers,
gated_capability_of,
)
from litellm.router_utils.auto_router_tuning_baseline import (
is_mutable_tuned_candidate,
mutable_tuned_identities,
tuning_quota_violation,
)
from litellm.types.management_endpoints.auto_router_endpoints import (
AutoRouterAllowance,
AutoRouterAvailabilityResponse,
)
class _CatalogModelInfo(BaseModel):
team_id: str | None = None
class _CatalogSource(BaseModel):
model_id: str
created_by: str | None = None
litellm_params: Json[dict[str, object]] | dict[str, object]
model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None
@dataclass(frozen=True, slots=True)
class AutoRouterCatalogEntry:
model_id: str
team_id: str | None
created_by: str | None
deployment: Mapping[str, object]
def _catalog_field(value: object, key: str) -> object:
if not isinstance(value, str):
return deepcopy(value)
return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True)
def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None:
try:
sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True)
except ValidationError:
return None
return tuple(
AutoRouterCatalogEntry(
model_id=row.model_id,
team_id=row.model_info.team_id if row.model_info is not None else None,
created_by=row.created_by,
deployment=MappingProxyType(
{
"litellm_params": MappingProxyType(
{
"model": model,
"complexity_router_config": _catalog_field(
row.litellm_params.get("complexity_router_config"), "complexity_router_config"
),
}
),
"model_info": MappingProxyType({"id": row.model_id, "db_model": True}),
}
),
)
for row in sources
if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str)
and classify_strategy_router_model(model) == "complexity"
)
def auto_router_availability(
*,
others: Sequence[Mapping[str, object]],
existing: Mapping[str, object] | None,
candidate: Mapping[str, object],
baselines: Mapping[str, str] | None,
limit: int | None,
) -> AutoRouterAvailabilityResponse:
existing_params: Final = None if existing is None else existing.get("litellm_params")
candidate_params: Final = candidate.get("litellm_params")
owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
counts: Final = tuple(
(capability, count_capability_routers(others, capability=capability))
for capability in GATED_AUTO_ROUTER_CAPABILITIES
)
tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0
allowances: Final = tuple(
AutoRouterAllowance(
key=capability.key,
limit=limit,
remaining=None if limit is None else max(0, limit - held),
used_by_this_router=owned is capability,
)
for capability, held in counts
)
capability_error: Final = next(
(
capability_limit_violation(capability=capability, held=held + 1, limit=limit)
for capability, held in counts
if capability is claimed
),
None,
)
tuning_error: Final = (
tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit)
if baselines is not None
else None
)
capability_labels: Final = {
"heuristic_v2": "Heuristic v2",
"capability": "Capability",
"llm_v2": "Fuse v2",
"tier_or_classifier_prompt": "Custom tiers or classifier instructions",
}
return AutoRouterAvailabilityResponse(
allowances=(
*allowances,
AutoRouterAllowance(
key="heuristic_tuning",
limit=limit,
remaining=None if limit is None or baselines is None else max(0, limit - tuned_count),
available=limit is None or baselines is not None,
used_by_this_router=bool(
existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines)
),
),
),
error=(
f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
if capability_error is not None and claimed is not None
else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
if tuning_error is not None
else None
),
)

View file

@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import (
strip_callback_config,
)
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
from litellm.router_utils.access_windows import access_windows_config_error
from litellm.router_utils.add_retry_fallback_headers import (
get_fallback_errors_from_headers,
@ -5068,6 +5069,7 @@ class ProxyConfig:
def __init__(self) -> None:
self.config: Mapping[str, object] = MappingProxyType({})
self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None
self._last_semantic_filter_config: dict[str, object] | None = None
self._last_websearch_interception_config: dict[str, object] | None = None
self._last_hashicorp_vault_config: dict[str, object] | None = None
@ -7691,6 +7693,12 @@ class ProxyConfig:
def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
return should_load_db_object(object_type=object_type)
def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None:
if self.auto_router_db_catalog is not None:
self.auto_router_db_catalog = tuple(
row for row in self.auto_router_db_catalog if row.model_id not in model_ids
)
async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None:
"""
Fetch all model deployments from the DB.
@ -7711,6 +7719,7 @@ class ProxyConfig:
new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(
WriterPinnedClient(prisma_client.db)
).table.find_many()
self.auto_router_db_catalog = build_auto_router_catalog(new_models)
return new_models
except Exception as e:
verbose_proxy_logger.exception(

View file

@ -1,18 +1,18 @@
{
"1m_context": {
"label": "1M Context",
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.",
"description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gpt-5.6-luna"],
"SIMPLE": ["gpt-6-luna"],
"MEDIUM": ["gpt-5.6-terra"],
"COMPLEX": ["gpt-5.6-sol"],
"REASONING": ["claude-opus-5"]
"COMPLEX": ["gpt-6-sol"],
"REASONING": ["claude-opus-5-5"]
},
"tier_model_configs": {
"REASONING": [
{
"model_name": "claude-opus-5",
"model_name": "claude-opus-5-5",
"litellm_params": { "reasoning_effort": "high" }
}
]
@ -28,12 +28,12 @@
},
"anthropic_family": {
"label": "Anthropic Family",
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.",
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["claude-haiku-4-5"],
"MEDIUM": ["claude-sonnet-5"],
"COMPLEX": ["claude-opus-5"],
"COMPLEX": ["claude-opus-5-5"],
"REASONING": ["claude-fable-5-1"]
},
"tier_model_configs": {
@ -55,12 +55,12 @@
},
"gemini_family": {
"label": "Gemini Family",
"description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.",
"description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gemini-2.5-flash-lite"],
"MEDIUM": ["gemini-3.1-flash-lite"],
"COMPLEX": ["gemini-3.7-flash"],
"SIMPLE": ["gemini-3.5-flash-lite"],
"MEDIUM": ["gemini-3.8-flash"],
"COMPLEX": ["gemini-3.8-flash"],
"REASONING": ["gemini-3.1-pro-preview"]
},
"classifier_type": "heuristic",
@ -74,18 +74,18 @@
},
"lite": {
"label": "Lite",
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["deepseek-v4-flash"],
"MEDIUM": ["muse-spark-1.2"],
"MEDIUM": ["muse-spark-1.3"],
"COMPLEX": ["kimi-k3"],
"REASONING": ["claude-opus-5"]
"REASONING": ["claude-opus-5-5"]
},
"tier_model_configs": {
"MEDIUM": [
{
"model_name": "muse-spark-1.2",
"model_name": "muse-spark-1.3",
"litellm_params": { "reasoning_effort": "xhigh" }
}
],
@ -113,12 +113,12 @@
},
"openai_family": {
"label": "OpenAI Family",
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.",
"description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.",
"complexity_router_config": {
"tiers": {
"SIMPLE": ["gpt-5.6-luna"],
"SIMPLE": ["gpt-6-luna"],
"MEDIUM": ["gpt-5.6-terra"],
"COMPLEX": ["gpt-5.6-sol"],
"COMPLEX": ["gpt-6-sol"],
"REASONING": ["gpt-6-astra"]
},
"tier_model_configs": {

View file

@ -2323,6 +2323,13 @@ async def calculate_spend(request: SpendCalculateRequest):
param=getattr(e, "param", "None"),
code=getattr(e, "status_code", status.HTTP_400_BAD_REQUEST),
)
if isinstance(e, litellm.exceptions.ModelNotMappedError):
raise ProxyException(
message=str(e),
type="invalid_request_error",
param="model",
code=status.HTTP_400_BAD_REQUEST,
)
error_msg: Final = f"{e}"
raise ProxyException(
message=getattr(e, "message", error_msg),

View file

@ -1,5 +1,5 @@
{
"version": "2026-09-17-v1",
"version": "2026-09-22-v1",
"models": [
{
"id": "gpt-6-astra-v1",
@ -8,6 +8,20 @@
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
},
{
"id": "gpt-6-sol-v1",
"label": "GPT-6 Sol",
"model": "gpt-6-sol",
"text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API",
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"]
},
{
"id": "gpt-6-luna-v1",
"label": "GPT-6 Luna",
"model": "gpt-6-luna",
"text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API",
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"]
},
{
"id": "gpt-5.6-sol-v1",
"label": "GPT-5.6 Sol",
@ -63,6 +77,13 @@
"model": "claude-fable-5-1",
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
},
{
"id": "claude-opus-5-5-v1",
"label": "Claude Opus 5.5",
"model": "claude-opus-5-5",
"text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use",
"sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"]
}
],
"harnesses": [

View file

@ -10,12 +10,10 @@ from pydantic import ValidationError
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2"
# v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
HEURISTIC_V1_TUNING_FIELDS: Final = (
"tiers",
"tier_model_configs",
"classifier_type",
"tier_boundaries",
"reasoning_override_min_score",
"token_thresholds",
@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
validated: Final = ComplexityRouterConfig.model_validate(raw)
except ValidationError:
return None
supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
# The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes.
default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"])
supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - (
frozenset(("escalation_keywords",)) if default_escalation else frozenset()
)
payload: Final = validated.model_dump(
mode="json",
@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None:
if limit is None or held <= limit:
return None
return (
f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified "
f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified "
"without an auto-router license. Keep this router on its recorded settings, or revert the other changed "
"router to its baseline, or remove one of them."
"router to its baseline, or remove one of them. Selecting models does not use this allowance."
)

View file

@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ...
class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
class ProcessReservedForForking(RuntimeError): ...
@final
class NativeDiagnosticProcessor:
def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
def redact_text(self, text: str) -> str: ...
def redact_structured_text(self, key: str | None, text: str) -> str: ...
def redact_client_message(self, text: str) -> str: ...
def process_diagnostic(
self,
message: str,
exception: str | None,
stack: str | None,
leaves: Sequence[tuple[str | None, str]],
policy: tuple[bool, int, int],
) -> tuple[str, str | None, str | None, list[str], bool]: ...
def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ...
def ocr(
request: LiteLLMOcrRequest,
args: tuple[object, ...],
@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ...
__all__ = [
"ForkedAfterNativeRuntimeStarted",
"HuggingFaceEncoding",
"NativeDiagnosticProcessor",
"ProcessReservedForForking",
"ResponsesWebSocketConnection",
"RustBridgeDeclined",

View file

@ -86,11 +86,25 @@ class SecretManagerRule:
return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems)
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule
@dataclass(frozen=True, slots=True)
class LoggerContext:
pass
@dataclass(frozen=True, slots=True)
class LoggerRule:
rollout: Rollout
def matches(self, context: Context) -> bool:
return isinstance(context, LoggerContext)
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule
Rules: TypeAlias = tuple[Rule, ...]
RULES: Final[Rules] = (
LoggerRule(Rollout.RUST_OPT_IN),
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),

View file

@ -0,0 +1,59 @@
from __future__ import annotations
from collections.abc import Callable, Sequence
from functools import lru_cache
from typing import Final, Protocol, TypeVar, cast
from litellm.rust_bridge.bindings import NativeBinding
ResultT: Final = TypeVar("ResultT")
class NativeDiagnosticProcessor(Protocol):
def redact_text(self, text: str) -> str: ...
def redact_structured_text(self, key: str | None, text: str) -> str: ...
def redact_client_message(self, text: str) -> str: ...
def process_diagnostic(
self,
message: str,
exception: str | None,
stack: str | None,
leaves: tuple[tuple[str | None, str], ...],
policy: tuple[bool, int, int],
) -> tuple[str, str | None, str | None, Sequence[str], bool]: ...
def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ...
class NativeDiagnosticFactory(Protocol):
def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
def _as_factory(value: object) -> NativeDiagnosticFactory | None:
if not isinstance(value, type):
return None
return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type
PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory)
@lru_cache(maxsize=4)
def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor:
return factory(minimum_custom_key_length)
def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT:
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
from litellm.rust_bridge.catalog import LoggerContext, decision
from litellm.rust_bridge.configuration import Decision
selected: Final = decision(LoggerContext())
if selected is Decision.PYTHON:
return python()
factory: Final = PROCESSOR.load()
if factory is None:
return python()
try:
return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH))
except Exception:
return python()

View file

@ -0,0 +1,63 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Final
from pydantic import JsonValue
from litellm._logging import (
CorrelationContextFilter,
DiagnosticProcessingFilter,
session_id_var,
set_session_id,
set_trace_id,
trace_id_var,
verbose_logger,
)
_REDACTION: Final = DiagnosticProcessingFilter()
_CORRELATION: Final = CorrelationContextFilter()
def context() -> tuple[str, str]:
return session_id_var.get(), trace_id_var.get()
def enabled(level: int) -> bool:
return verbose_logger.isEnabledFor(level)
def emit(
level: int,
message: str,
pathname: str,
lineno: int,
target: str,
fields: Mapping[str, JsonValue],
correlation: tuple[str, str],
) -> None:
if not enabled(level):
return
session_token: Final = set_session_id(correlation[0])
trace_token: Final = set_trace_id(correlation[1])
try:
record: Final = verbose_logger.makeRecord(
verbose_logger.name,
level,
pathname,
lineno,
message,
(),
None,
func=target,
extra={
"rust_target": target,
"rust_fields": dict(fields),
}, # mutable-ok: LogRecord requires JSON dict extras
)
_REDACTION.filter(record)
_CORRELATION.filter(record)
verbose_logger.handle(record)
finally:
trace_id_var.reset(trace_token)
session_id_var.reset(session_token)

View file

@ -58,12 +58,6 @@ class SecretManagerBinding:
settings_object: object
def warn(message: str) -> None:
from litellm._logging import verbose_logger
verbose_logger.warning("%s", message)
def secret_manager() -> SecretManager:
from litellm.secret_managers.main import (
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private

View file

@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
error: str | None = None
class AutoRouterAvailabilityRequest(BaseModel):
team_id: str | None = None
saved_model_id: str | None = None
complexity_router_config: Mapping[str, object] | None = None
class AutoRouterAllowance(BaseModel):
key: str
limit: int | None
remaining: int | None
used_by_this_router: bool = False
available: bool = True
class AutoRouterAvailabilityResponse(BaseModel):
allowances: tuple[AutoRouterAllowance, ...]
error: str | None = None
class AutoRouterRoutingTestRequest(BaseModel):
"""A single request to classify against a complexity-router config that need not be saved yet.

View file

@ -3256,6 +3256,7 @@ class StandardLoggingPayloadErrorInformation(TypedDict, total=False):
error_budget_entity_id: str | None
error_budget_limit: float | None
error_budget_spend: float | None
normalized_error: ReadOnly[str | None]
class GuardrailMode(TypedDict, total=False):

View file

@ -472,6 +472,7 @@ from .exceptions import (
BudgetExceededError,
ContentPolicyViolationError,
ContextWindowExceededError,
ModelNotMappedError,
NotFoundError,
OpenAIError,
PermissionDeniedError,
@ -1439,11 +1440,22 @@ async def async_post_call_success_deployment_hook(
modified_response = response
CustomLogger: Final = _get_cached_custom_logger()
CustomGuardrail: Final = _get_cached_custom_guardrail()
for callback in litellm.callbacks:
if isinstance(callback, CustomLogger):
result = await callback.async_post_call_success_deployment_hook(
request_data, cast(LLMResponseTypes, modified_response), typed_call_type
)
try:
result = await callback.async_post_call_success_deployment_hook(
request_data, cast(LLMResponseTypes, modified_response), typed_call_type
)
except Exception: # noqa: BLE001 # a broken callback must not fail a completed request
if isinstance(callback, CustomGuardrail):
raise
verbose_logger.exception(
"async_post_call_success_deployment_hook error in %s for call_type=%s",
type(callback).__name__,
typed_call_type,
)
continue
if result is not None:
modified_response = result
@ -5830,6 +5842,13 @@ def _is_potential_model_name_in_model_cost(
_ABOVE_THRESHOLD_COST_KEY: Final = ABOVE_THRESHOLD_COST_KEY_PATTERN
def _model_not_mapped_message(model: str, custom_llm_provider: str | None) -> str:
return (
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. "
"Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json."
)
def _get_model_info_helper(
model: str,
custom_llm_provider: str | None = None,
@ -6012,9 +6031,7 @@ def _get_model_info_helper(
key, _model_info = generalization
if _model_info is None or key is None:
raise ValueError(
"This model isn't mapped yet. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json"
)
raise ModelNotMappedError(_model_not_mapped_message(model, custom_llm_provider))
_input_cost_per_token: float | None = _model_info.get("input_cost_per_token")
if _input_cost_per_token is None:
# default value to 0, be noisy about this
@ -6249,11 +6266,11 @@ def _get_model_info_helper(
if cost_key not in returned_model_info and _ABOVE_THRESHOLD_COST_KEY.search(cost_key) is not None:
returned_model_info[cost_key] = cost_value
return returned_model_info
except ModelNotMappedError:
raise
except Exception as e:
verbose_logger.debug("Error getting model info: %s", e)
raise Exception(
f"This model isn't mapped yet. model={model}, custom_llm_provider={custom_llm_provider}. Add it here - https://github.com/BerriAI/litellm/blob/main/model_prices_and_context_window.json."
)
raise Exception(_model_not_mapped_message(model, custom_llm_provider))
def _build_model_info(

View file

@ -25314,8 +25314,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"cache_read_input_token_cost_batches": 1e-07
"web_search_billing_unit": "per_query"
},
"gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -25401,8 +25400,7 @@
"search_context_size_medium": 0.014,
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"cache_read_input_token_cost_batches": 2.5e-08
"web_search_billing_unit": "per_query"
},
"gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -25482,8 +25480,7 @@
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_vision": true
},
"gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
@ -25593,8 +25590,7 @@
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
"cache_read_input_token_cost_batches": 1.25e-08
"input_cost_per_audio_token_batches": 2.5e-07
},
"gemini-3.5-flash-lite": {
"deprecation_date": "2027-07-21",
@ -25652,8 +25648,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 1.5e-08
"google_maps_grounding_cost_per_query": 0.014
},
"deep-research-pro-preview-12-2025": {
"cache_read_input_token_cost": 2e-07,
@ -25689,8 +25684,7 @@
"supports_response_schema": true,
"supports_system_messages": true,
"supports_vision": true,
"supports_web_search": true,
"cache_read_input_token_cost_batches": 1e-07
"supports_web_search": true
},
"gemini-2.5-flash-lite": {
"cache_read_input_audio_token_cost": 3e-08,
@ -26321,8 +26315,7 @@
"output_cost_per_token_batches": 4.5e-06,
"input_cost_per_token_flex": 7.5e-07,
"output_cost_per_token_flex": 4.5e-06,
"cache_read_input_token_cost_flex": 7.5e-08,
"cache_read_input_token_cost_batches": 7.5e-08
"cache_read_input_token_cost_flex": 7.5e-08
},
"vertex_ai/gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
@ -26380,8 +26373,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
@ -26440,8 +26432,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
@ -26500,8 +26491,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/gemini-3.1-pro-preview": {
"prompt_cache_min_tokens": 4096,
@ -28250,8 +28240,7 @@
"output_cost_per_token_batches": 4.5e-06,
"input_cost_per_token_flex": 7.5e-07,
"output_cost_per_token_flex": 4.5e-06,
"cache_read_input_token_cost_flex": 7.5e-08,
"cache_read_input_token_cost_batches": 7.5e-08
"cache_read_input_token_cost_flex": 7.5e-08
},
"gemini-3.6-flash": {
"prompt_cache_min_tokens": 4096,
@ -28309,8 +28298,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.7-flash": {
"prompt_cache_min_tokens": 4096,
@ -28369,8 +28357,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini-3.8-flash": {
"prompt_cache_min_tokens": 4096,
@ -28429,8 +28416,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 3.75e-08
"google_maps_grounding_cost_per_query": 0.014
},
"gemini/gemini-2.5-pro-preview-tts": {
"cache_read_input_token_cost": 1.25e-07,
@ -33110,8 +33096,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 6.25e-08
"supports_minimal_reasoning_effort": true
},
"gpt-5-chat": {
"cache_read_input_token_cost": 1.25e-07,
@ -33292,8 +33277,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_minimal_reasoning_effort": true
},
"gpt-5-nano": {
"cache_read_input_token_cost": 5e-09,
@ -33392,8 +33376,7 @@
"supports_web_search": true,
"supports_none_reasoning_effort": false,
"supports_xhigh_reasoning_effort": false,
"supports_minimal_reasoning_effort": true,
"cache_read_input_token_cost_batches": 2.5e-09
"supports_minimal_reasoning_effort": true
},
"gpt-image-1": {
"cache_read_input_token_cost": 1.25e-06,
@ -47531,8 +47514,7 @@
"output_cost_per_token_flex": 6e-06,
"output_cost_per_token_priority": 2.16e-05,
"supports_reasoning": false,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 1e-07
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/gemini-3-pro-image-preview": {
"input_cost_per_image": 0.0011,
@ -47570,8 +47552,7 @@
"output_cost_per_token_batches": 1.5e-06,
"output_cost_per_token_flex": 1.5e-06,
"supports_reasoning": false,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 2.5e-08
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/gemini-3.1-flash-image-preview": {
"input_cost_per_image": 0.00056,
@ -47627,8 +47608,7 @@
"supports_response_schema": false,
"supports_system_messages": true,
"supports_video_input": true,
"supports_vision": true,
"cache_read_input_token_cost_batches": 1.25e-08
"supports_vision": true
},
"vertex_ai/gemini-3.1-flash-lite-preview": {
"cache_read_input_token_cost": 2.5e-08,
@ -47739,8 +47719,7 @@
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"input_cost_per_audio_token_batches": 2.5e-07,
"cache_read_input_token_cost_batches": 1.25e-08
"input_cost_per_audio_token_batches": 2.5e-07
},
"vertex_ai/gemini-3.5-flash-lite": {
"deprecation_date": "2027-07-21",
@ -47799,8 +47778,7 @@
"search_context_size_high": 0.014
},
"web_search_billing_unit": "per_query",
"google_maps_grounding_cost_per_query": 0.014,
"cache_read_input_token_cost_batches": 1.5e-08
"google_maps_grounding_cost_per_query": 0.014
},
"vertex_ai/deep-research-pro-preview-12-2025": {
"cache_read_input_token_cost": 2e-07,
@ -47817,8 +47795,7 @@
"output_cost_per_image_token": 0.00012,
"output_cost_per_token": 1.2e-05,
"output_cost_per_token_batches": 6e-06,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
"cache_read_input_token_cost_batches": 1e-07
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"vertex_ai/jamba-1.5": {
"input_cost_per_token": 2e-07,

View file

@ -3,62 +3,140 @@ import { describe, expect, test } from "bun:test";
import type { Comment, GitHubApi } from "./auto-close-duplicates";
import {
FIXED_MARKER,
OPEN_PULL_REQUESTS_QUERY,
SUPERSEDED_MARKER,
closeVerdict,
closerOf,
commentFixedIssue,
describeIssue,
describeSweep,
fixOf,
fixedBody,
handleFixedIssue,
nextMinor,
parseVersion,
placement,
readConfig,
releaseCandidate,
supersededBody,
sweep,
type ClosedIssue,
type FixedConfig,
type IssueClosure,
type LinkedIssue,
type LinkedPullRequest,
type PullRequestsPage,
} from "./comment-fixed-issue";
const MERGE_COMMIT = "68c4c82ac977b48b2b81ee8d633d5771307c6162";
const ISSUE = 41750;
const mergedPr = {
__typename: "PullRequest" as const,
number: 41767,
merged: true,
baseRefName: "main",
repository: { nameWithOwner: "BerriAI/litellm" },
mergeCommit: { oid: MERGE_COMMIT },
};
type Closer = ClosedIssue["timelineItems"]["nodes"][number]["closer"];
const commitCloser = { __typename: "Commit" as const, oid: MERGE_COMMIT, repository: { nameWithOwner: "BerriAI/litellm" } };
const closedBy = (closer: Closer, state: ClosedIssue["state"] = "CLOSED"): ClosedIssue => ({
type Closer = IssueClosure["timelineItems"]["nodes"][number]["closer"];
const closure = (closer: Closer, state: IssueClosure["state"] = "CLOSED"): IssueClosure => ({
state,
timelineItems: { nodes: [{ closer }] },
});
const page = (pages: readonly (readonly LinkedPullRequest[])[], index: number): PullRequestsPage => ({
pageInfo: { hasNextPage: index + 1 < pages.length, endCursor: String(index + 1) },
nodes: pages[index] ?? [],
});
const cursorIndex = (after: string | null): number => (after === null ? 0 : Number(after));
const closedBy = (closer: Closer, state?: IssueClosure["state"], linked: readonly LinkedPullRequest[] = []): ClosedIssue => ({
...closure(closer, state),
closedByPullRequestsReferences: page([linked], 0),
});
const reopenedAt = (createdAt: string): LinkedPullRequest["reopens"] => ({ nodes: [{ createdAt }] });
const linkedIssue = (number: number, closer: Closer = mergedPr, state?: IssueClosure["state"]): LinkedIssue => ({
number,
repository: { nameWithOwner: "BerriAI/litellm" },
...closure(closer, state),
});
const links = (...issues: readonly LinkedIssue[]): LinkedPullRequest["closingIssuesReferences"] => ({ totalCount: issues.length, nodes: issues });
const openPr = (number: number, overrides: Partial<LinkedPullRequest> = {}): LinkedPullRequest => ({
number,
state: "OPEN",
baseRefName: "main",
repository: { nameWithOwner: "BerriAI/litellm" },
closingIssuesReferences: links(linkedIssue(ISSUE)),
reopens: { nodes: [] },
...overrides,
});
const pyproject = (version: string): string =>
`[project]\nname = "litellm"\nversion = "${version}"\n\n[tool.commitizen]\nversion = "${version}"\n`;
const config: FixedConfig = { repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false };
const config: FixedConfig = { repo: "BerriAI/litellm", defaultBranch: "main", commentDryRun: false, closeDryRun: false };
const prFix = (issue = ISSUE, number = 41767) => ({ issue, source: { kind: "pull_request" as const, number, oid: MERGE_COMMIT } });
const commitFix = (issue = ISSUE) => ({ issue, source: { kind: "commit" as const, oid: MERGE_COMMIT } });
const oneFixBody = supersededBody([prFix()], "main");
const noPause = async (): Promise<void> => {};
const supersededComment: Comment = {
id: 2,
body: `${SUPERSEDED_MARKER}\n#41750 was fixed by #41767 on main, so this pull request is closed. Reopen it if something was missed.`,
created_at: "2026-09-18T00:00:00Z",
user: { type: "Bot", login: "github-actions[bot]" },
};
interface World {
readonly issue?: ClosedIssue | null;
readonly comments?: readonly Comment[];
readonly pullRequestComments?: Readonly<Record<number, readonly Comment[]>>;
readonly openPullRequests?: readonly (readonly LinkedPullRequest[])[];
readonly linkedPullRequests?: readonly (readonly LinkedPullRequest[])[];
readonly version?: string;
// Which existing rc.1 tags contain the merge commit; a tag absent from the map does not exist
readonly tags?: Readonly<Record<string, boolean>>;
readonly reachable?: Readonly<Record<string, readonly string[]>>;
}
function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes: string[] } {
const writes: string[] = [];
const tags = world.tags ?? {};
const reachable = world.reachable ?? { main: [MERGE_COMMIT] };
const pages = world.openPullRequests ?? [];
const api: GitHubApi = {
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
if (method === "POST" && path === "/graphql") {
return { data: { repository: { issue: world.issue === undefined ? closedBy(mergedPr) : world.issue } } } as T;
const { query, variables } = body as { query: string; variables: { after: string | null } };
if (query === OPEN_PULL_REQUESTS_QUERY) {
return { data: { repository: { pullRequests: page(pages, cursorIndex(variables.after)) } } } as T;
}
const issue = world.issue === undefined ? closedBy(mergedPr) : world.issue;
if (issue === null || world.linkedPullRequests === undefined) {
return { data: { repository: { issue } } } as T;
}
const closedByPullRequestsReferences = page(world.linkedPullRequests, cursorIndex(variables.after));
return { data: { repository: { issue: { ...issue, closedByPullRequestsReferences } } } } as T;
}
if (method !== "GET") {
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
return {} as T;
}
if (path.startsWith("/repos/BerriAI/litellm/issues/41750/comments")) {
return (world.comments ?? []) as T;
const comments = /^\/repos\/BerriAI\/litellm\/issues\/(\d+)\/comments/.exec(path);
if (comments !== null) {
const number = Number(comments[1]);
return ((number === ISSUE ? world.comments : world.pullRequestComments?.[number]) ?? []) as T;
}
if (path === `/repos/BerriAI/litellm/contents/pyproject.toml?ref=${MERGE_COMMIT}`) {
return { content: btoa(pyproject(world.version ?? "1.103.0")).replace(/(.{60})/g, "$1\n") } as T;
@ -68,6 +146,10 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes:
return (matching[1] in tags ? [{ ref: `refs/tags/${matching[1]}` }] : []) as T;
}
const compare = /^\/repos\/BerriAI\/litellm\/compare\/(.+)\.\.\.(.+)$/.exec(path);
const branch = compare === null ? undefined : reachable[compare[1] ?? ""];
if (compare !== null && branch !== undefined) {
return { status: branch.includes(compare[2] ?? "") ? "behind" : "diverged" } as T;
}
if (compare !== null && compare[2] === MERGE_COMMIT) {
return { status: tags[compare[1]] ? "behind" : "ahead" } as T;
}
@ -79,23 +161,28 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes:
describe("closerOf", () => {
test("a pull request merged into the default branch is the fix", () => {
expect(closerOf(closedBy(mergedPr), "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT });
expect(closerOf(closedBy(mergedPr), "BerriAI/litellm", "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT });
});
test("an issue closed by hand, by a commit, or by an unmerged pull request gets no comment", () => {
expect(closerOf(closedBy(null), "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
expect(closerOf(closedBy({ __typename: "Commit", oid: MERGE_COMMIT }), "main").kind).toBe("skip");
expect(closerOf(closedBy({ ...mergedPr, merged: false }), "main").kind).toBe("skip");
expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "main").kind).toBe("skip");
expect(closerOf(closedBy(null), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
expect(closerOf(closedBy(commitCloser), "BerriAI/litellm", "main").kind).toBe("skip");
expect(closerOf(closedBy({ ...mergedPr, merged: false }), "BerriAI/litellm", "main").kind).toBe("skip");
expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm", "main").kind).toBe("skip");
});
test("a pull request merged into a release branch is not a fix on main", () => {
const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "main");
const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "BerriAI/litellm", "main");
expect(verdict).toEqual({ kind: "skip", reason: "#41767 merged into release/1.102.0rc2, not main" });
});
test("an issue reopened after the close event is left alone", () => {
expect(closerOf(closedBy(mergedPr, "OPEN"), "main")).toEqual({ kind: "skip", reason: "the issue is open again" });
expect(closerOf(closedBy(mergedPr, "OPEN"), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "the issue is open again" });
});
test("a pull request merged in a fork closes the issue on GitHub but is no fix here", () => {
const forkPr = { ...mergedPr, number: 9, repository: { nameWithOwner: "someone/litellm" } };
expect(closerOf(closedBy(forkPr), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by someone/litellm#9, a pull request in another repository" });
});
});
@ -173,44 +260,327 @@ describe("fixedBody", () => {
});
});
describe("commentFixedIssue", () => {
describe("fixOf", () => {
test("a merged pull request or a commit is a fix, whatever branch it was merged into", () => {
expect(fixOf(closure(mergedPr), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT });
expect(fixOf(closure({ ...mergedPr, baseRefName: "litellm_internal_staging" }), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT });
expect(fixOf(closure(commitCloser), "BerriAI/litellm")).toEqual({ kind: "commit", oid: MERGE_COMMIT });
});
test("a hand close, a fork's pull request, an unmerged pull request, or a reopened issue is no fix", () => {
expect(fixOf(closure(null), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by hand" });
expect(fixOf(closure({ ...mergedPr, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" });
expect(fixOf(closure({ ...commitCloser, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" });
expect(fixOf(closure({ ...mergedPr, merged: false }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by #41767, which is not merged" });
expect(fixOf(closure({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm").kind).toBe("skip");
expect(fixOf(closure(mergedPr, "OPEN"), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "is open again" });
});
});
describe("closeVerdict", () => {
test("an open pull request whose only linked issue was fixed by a merged pull request is a candidate", () => {
expect(closeVerdict(openPr(41760), config)).toEqual({ kind: "candidate", fixes: [prFix()] });
});
test("every linked issue has to be fixed, and each fix is named", () => {
const both = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, commitCloser)) });
expect(closeVerdict(both, config)).toEqual({ kind: "candidate", fixes: [prFix(), commitFix(41751)] });
});
test("a pull request that still links an open issue keeps its work", () => {
const stillOpen = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) });
expect(closeVerdict(stillOpen, config)).toEqual({ kind: "skip", reason: "still linked to open #41751" });
});
test("a linked issue closed by hand or by an unmerged pull request is not a fix that supersedes the pull request", () => {
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) }), config)).toEqual({
kind: "skip",
reason: `#${ISSUE} was closed by hand`,
});
const unmerged = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, { ...mergedPr, merged: false })) });
expect(closeVerdict(unmerged, config)).toEqual({ kind: "skip", reason: `#${ISSUE} was closed by #41767, which is not merged` });
});
test("a pull request linking an issue in another repository, or more issues than the query reads, is left alone", () => {
const foreign = { ...linkedIssue(41751), repository: { nameWithOwner: "mlflow/mlflow" } };
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), foreign) }), config)).toEqual({
kind: "skip",
reason: "links mlflow/mlflow#41751",
});
const truncated = openPr(41760, { closingIssuesReferences: { totalCount: 11, nodes: [linkedIssue(ISSUE)] } });
expect(closeVerdict(truncated, config)).toEqual({ kind: "skip", reason: "links 11 issues, more than the 10 this workflow reads" });
});
test("a pull request against a release line is a backport and stays open, one against a retired development branch does not", () => {
for (const base of ["release/v1.102.0-rc.2", "stable/v1.83.14", "v_1_83_3_stable_patch"]) {
expect(closeVerdict(openPr(41760, { baseRefName: base }), config)).toEqual({ kind: "skip", reason: `targets the release line ${base}` });
}
expect(closeVerdict(openPr(41760, { baseRefName: "litellm_internal_staging" }), config).kind).toBe("candidate");
});
test("a pull request in a fork, one that is not open, or one linking no issue is left alone", () => {
expect(closeVerdict(openPr(1, { repository: { nameWithOwner: "someone/litellm" } }), config)).toEqual({ kind: "skip", reason: "lives in someone/litellm" });
expect(closeVerdict(openPr(41767, { state: "MERGED" }), config)).toEqual({ kind: "skip", reason: "is merged" });
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links() }), config)).toEqual({ kind: "skip", reason: "links no issue" });
});
});
describe("supersededBody", () => {
test("names each issue with the pull request or commit that fixed it, invites a reopen, and carries the marker the rerun looks for", () => {
expect(oneFixBody.startsWith(SUPERSEDED_MARKER)).toBe(true);
expect(oneFixBody).toContain("#41750 was fixed by #41767 on main");
expect(oneFixBody).toContain("Reopen it");
expect(supersededBody([prFix(), commitFix(41751)], "main")).toContain("#41750 was fixed by #41767 and #41751 by commit 68c4c82ac9 on main");
});
test("stays within the 25-word comment rule for one and two fixes", () => {
for (const fixes of [[prFix()], [commitFix()], [prFix(), commitFix(41751)]]) {
const words = supersededBody(fixes, "main").replace(SUPERSEDED_MARKER, "").trim().split(/\s+/);
expect(words.length).toBeGreaterThanOrEqual(15);
expect(words.length).toBeLessThanOrEqual(25);
}
});
});
describe("handleFixedIssue", () => {
test("a real run posts one comment naming the pull request and the release", async () => {
const { api, writes } = fakeApi();
const verdict = await commentFixedIssue(api, config);
expect(verdict).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" });
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(comment).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" });
expect(pullRequests).toEqual([]);
expect(writes).toHaveLength(1);
expect(writes[0]).toContain("POST /repos/BerriAI/litellm/issues/41750/comments");
expect(writes[0]).toContain("Fixed by #41767. This ships in v1.103.0-rc.1 and up");
});
test("a dry run renders the comment and writes nothing", async () => {
const { api, writes } = fakeApi();
const verdict = await commentFixedIssue(api, { ...config, dryRun: true });
expect(verdict.kind).toBe("commented");
expect(writes).toEqual([]);
test("every other open pull request linked to the fixed issue is commented on and then closed, a pause before every write", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760), openPr(41761)]) });
let pauses = 0;
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, async () => {
pauses += 1;
});
expect(pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed", "closed"]);
expect(writes).toEqual([
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41750/comments"),
`POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`,
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"),
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
]);
expect(pauses).toBe(4);
});
test("an issue that already carries the comment is not commented twice", async () => {
test("the fixed-in comment and the closing are gated separately: a close dry run still posts the comment and closes nothing", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
let pauses = 0;
const outcome = await handleFixedIssue(api, { ...config, closeDryRun: true }, ISSUE, async () => {
pauses += 1;
});
expect(outcome.comment.kind).toBe("commented");
expect(outcome.pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
expect(writes).toHaveLength(1);
expect(writes[0]).toContain("/issues/41750/comments");
expect(pauses).toBe(0);
});
test("a comment dry run still closes the pull requests for real", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
const outcome = await handleFixedIssue(api, { ...config, commentDryRun: true }, ISSUE, noPause);
expect(outcome.comment.kind).toBe("commented");
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
});
test("an issue that already carries the fixed-in comment is not commented twice, and its linked pull requests still get closed", async () => {
const existing: Comment = {
id: 1,
body: `${FIXED_MARKER}\nFixed by #41767. This ships in v1.103.0-rc.1 and up.`,
created_at: "2026-09-18T00:00:00Z",
user: { type: "Bot", login: "github-actions[bot]" },
};
const { api, writes } = fakeApi({ comments: [existing] });
expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" });
const { api, writes } = fakeApi({ comments: [existing], issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
expect(outcome.comment).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" });
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
});
test("a fix merged into the retired development branch or pushed as a commit counts once it is on the default branch", async () => {
const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" };
const staging = openPr(41760, { baseRefName: "litellm_internal_staging", closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) });
const byCommit = openPr(41761, { closingIssuesReferences: links(linkedIssue(41751, commitCloser)) });
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [staging, byCommit]) });
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([
{ kind: "closed", number: 41760, body: oneFixBody },
{ kind: "closed", number: 41761, body: supersededBody([commitFix(41751)], "main") },
]);
expect(writes).toHaveLength(5);
expect(writes[3]).toContain("#41751 was fixed by commit 68c4c82ac9 on main");
});
test("a fix whose commit never reached the default branch supersedes nothing", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]), reachable: { main: [] } });
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([{ kind: "skip", number: 41760, reason: "#41750 was fixed by #41767, which is not on main" }]);
expect(writes).toHaveLength(1);
expect(writes[0]).toContain("/issues/41750/comments");
});
test("the default branch comes from the config for the containment check and the comment alike", async () => {
const stagingConfig = { ...config, defaultBranch: "litellm_internal_staging" };
const closer = { ...mergedPr, baseRefName: "litellm_internal_staging" };
const { api, writes } = fakeApi({
issue: closedBy(closer, "CLOSED", [openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, closer)) })]),
reachable: { litellm_internal_staging: [MERGE_COMMIT] },
});
const { pullRequests } = await handleFixedIssue(api, stagingConfig, ISSUE, noPause);
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([prFix()], "litellm_internal_staging") }]);
expect(writes[1]).toContain("on litellm_internal_staging, so this pull request is closed");
});
test("the closer sits in the linked list as merged and gets neither a line nor a write", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41767, { state: "MERGED" }), openPr(41760)]) });
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
expect(writes.map((write) => write.split(" ")[1])).toEqual([
"/repos/BerriAI/litellm/issues/41750/comments",
"/repos/BerriAI/litellm/issues/41760/comments",
"/repos/BerriAI/litellm/pulls/41760",
]);
});
test("a pull request this workflow closed once and its author reopened stays open", async () => {
const reopened = openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") });
const { api, writes } = fakeApi({
issue: closedBy(mergedPr, "CLOSED", [reopened, openPr(41761)]),
pullRequestComments: { 41760: [supersededComment] },
});
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests[0]).toEqual({ kind: "skip", number: 41760, reason: "was closed by this workflow once and reopened" });
expect(pullRequests[1]?.kind).toBe("closed");
expect(writes.filter((write) => write.includes("41760"))).toEqual([]);
});
test("a pull request whose comment landed but whose close failed is closed on the next run without a second comment", async () => {
const reopenedBeforeTheComment = openPr(41761, { reopens: reopenedAt("2026-09-17T00:00:00Z") });
const { api, writes } = fakeApi({
issue: closedBy(mergedPr, "CLOSED", [openPr(41760), reopenedBeforeTheComment]),
pullRequestComments: { 41760: [supersededComment], 41761: [supersededComment] },
});
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([
{ kind: "closed", number: 41760, body: supersededComment.body },
{ kind: "closed", number: 41761, body: supersededComment.body },
]);
expect(writes.filter((write) => write.includes("/4176"))).toEqual([
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
]);
});
test("a superseded marker pasted by anyone but the workflow neither keeps a pull request open nor replaces its comment", async () => {
const forged: Comment = { ...supersededComment, id: 3, user: { type: "User", login: "someone" } };
const { api, writes } = fakeApi({
issue: closedBy(mergedPr, "CLOSED", [openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") })]),
pullRequestComments: { 41760: [forged] },
});
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
expect(writes.filter((write) => write.includes("/41760"))).toEqual([
`POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`,
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
]);
});
test("every page of linked pull requests is read, not just the first", async () => {
const { api } = fakeApi({ linkedPullRequests: [[openPr(41760)], [openPr(41761)], [openPr(41762)]] });
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([
{ kind: "closed", number: 41760, body: oneFixBody },
{ kind: "closed", number: 41761, body: oneFixBody },
{ kind: "closed", number: 41762, body: oneFixBody },
]);
});
test("a linked pull request from a fork or one still tied to another open issue is reported, not closed", async () => {
const fork = openPr(1, { repository: { nameWithOwner: "someone/litellm" } });
const busy = openPr(41762, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) });
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [fork, busy]) });
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(pullRequests).toEqual([
{ kind: "skip", number: 1, reason: "lives in someone/litellm" },
{ kind: "skip", number: 41762, reason: "still linked to open #41751" },
]);
expect(writes).toHaveLength(1);
});
test("a hand-closed issue gets no comment and leaves its linked pull requests open with the reason on each", async () => {
const byHand = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) });
const { api, writes } = fakeApi({ issue: closedBy(null, "CLOSED", [byHand]) });
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
expect(outcome.comment).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
expect(outcome.pullRequests).toEqual([{ kind: "skip", number: 41760, reason: `#${ISSUE} was closed by hand` }]);
expect(writes).toEqual([]);
});
test("a hand-closed issue never reaches the release lookup or the API writes", async () => {
const { api, writes } = fakeApi({ issue: closedBy(null) });
expect((await commentFixedIssue(api, config)).kind).toBe("skip");
test("an issue closed by a commit on the default branch gets no comment but still closes its linked pull requests", async () => {
const byCommit = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, commitCloser)) });
const { api, writes } = fakeApi({ issue: closedBy(commitCloser, "CLOSED", [byCommit]) });
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(comment).toEqual({ kind: "skip", reason: "closed by commit 68c4c82ac9, not by a pull request" });
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([commitFix()], "main") }]);
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
expect(writes[0]).toContain("#41750 was fixed by commit 68c4c82ac9 on main");
});
test("an issue closed from the retired development branch gets no comment but still closes its linked pull requests once the fix is on the default branch", async () => {
const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" };
const staging = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) });
const { api, writes } = fakeApi({ issue: closedBy(stagingPr, "CLOSED", [staging]) });
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
expect(comment).toEqual({ kind: "skip", reason: "#41767 merged into litellm_internal_staging, not main" });
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
expect(writes.map((write) => write.split(" ")[1])).toEqual(["/repos/BerriAI/litellm/issues/41760/comments", "/repos/BerriAI/litellm/pulls/41760"]);
});
test("an issue that is open again gets no comment and its linked pull requests are neither read nor touched", async () => {
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "OPEN", [openPr(41760)]) });
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
expect(outcome).toEqual({ comment: { kind: "skip", reason: "the issue is open again" }, pullRequests: [] });
expect(writes).toEqual([]);
});
test("a number that is not an issue in the repository is a skip", async () => {
const { api, writes } = fakeApi({ issue: null });
expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "not an issue in this repository" });
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
expect(outcome.comment).toEqual({ kind: "skip", reason: "not an issue in this repository" });
expect(writes).toEqual([]);
});
});
describe("sweep", () => {
test("walks every page of open pull requests and closes the ones whose linked issues were all fixed", async () => {
const unlinked = openPr(41700, { closingIssuesReferences: links() });
const busy = openPr(41701, { closingIssuesReferences: links(linkedIssue(41751, null, "OPEN")) });
const { api, writes } = fakeApi({ openPullRequests: [[unlinked, openPr(41760)], [busy, openPr(41761)]] });
const outcome = await sweep(api, { ...config, commentDryRun: true }, noPause);
expect(outcome.considered).toBe(4);
expect(outcome.pullRequests).toEqual([
{ kind: "closed", number: 41760, body: oneFixBody },
{ kind: "skip", number: 41701, reason: "still linked to open #41751" },
{ kind: "closed", number: 41761, body: oneFixBody },
]);
expect(writes).toEqual([
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41760/comments"),
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"),
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
]);
});
test("a sweep dry run lists what it would close and writes nothing", async () => {
const { api, writes } = fakeApi({ openPullRequests: [[openPr(41760)]] });
const outcome = await sweep(api, { ...config, closeDryRun: true }, noPause);
expect(outcome.pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed"]);
expect(writes).toEqual([]);
});
});
@ -218,10 +588,24 @@ describe("commentFixedIssue", () => {
describe("readConfig", () => {
const env = { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm", ISSUE_NUMBER: "41750", DEFAULT_BRANCH: "main" };
test("reads the four inputs and treats anything but the literal true as a real run", () => {
expect(readConfig(env)).toEqual({ token: "t", repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false });
expect(readConfig({ ...env, DRY_RUN: "true" }).dryRun).toBe(true);
expect(readConfig({ ...env, DRY_RUN: "false" }).dryRun).toBe(false);
test("reads the inputs and treats anything but the literal true as a real run for each gate", () => {
expect(readConfig(env)).toEqual({
token: "t",
repo: "BerriAI/litellm",
defaultBranch: "main",
commentDryRun: false,
closeDryRun: false,
run: { kind: "issue", number: 41750 },
});
expect(readConfig({ ...env, DRY_RUN: "true" })).toMatchObject({ commentDryRun: true, closeDryRun: false });
expect(readConfig({ ...env, CLOSE_PRS_DRY_RUN: "true" })).toMatchObject({ commentDryRun: false, closeDryRun: true });
expect(readConfig({ ...env, DRY_RUN: "false", CLOSE_PRS_DRY_RUN: "false" })).toMatchObject({ commentDryRun: false, closeDryRun: false });
});
test("a sweep needs no issue number and anything but the literal true is an issue run", () => {
expect(readConfig({ ...env, ISSUE_NUMBER: undefined, SWEEP: "true" }).run).toEqual({ kind: "sweep" });
expect(readConfig({ ...env, SWEEP: "false" }).run).toEqual({ kind: "issue", number: 41750 });
expect(() => readConfig({ ...env, ISSUE_NUMBER: "", SWEEP: "false" })).toThrow("dispatch with an issue_number or with sweep ticked");
});
test("refuses a missing token, repo, branch or a bad issue number", () => {
@ -232,3 +616,31 @@ describe("readConfig", () => {
expect(() => readConfig({ ...env, ISSUE_NUMBER: "abc" })).toThrow("ISSUE_NUMBER");
});
});
describe("step summary", () => {
const closed = { kind: "closed" as const, number: 41760, body: oneFixBody };
const left = { kind: "skip" as const, number: 41762, reason: "still linked to open #41751" };
const commented = { kind: "commented" as const, pullRequest: 41767, tag: "v1.103.0-rc.1", body: fixedBody(41767, { tag: "v1.103.0-rc.1", shipped: false }) };
test("an issue run names the comment, each close, and each pull request left open", () => {
const summary = describeIssue(config, ISSUE, { comment: commented, pullRequests: [closed, left] });
expect(summary).toContain("#41750: commented, fixed by #41767 in v1.103.0-rc.1");
expect(summary).toContain("#41760: closed with: #41750 was fixed by #41767 on main");
expect(summary).toContain("#41762: left open, still linked to open #41751");
expect(summary).not.toContain("DRY RUN");
});
test("a close dry run names the repo variable that turns closing on", () => {
const summary = describeIssue({ ...config, closeDryRun: true }, ISSUE, { comment: commented, pullRequests: [closed] });
expect(summary).toContain("ISSUE_FIXED_CLOSE_PRS_ENABLED");
expect(summary).toContain("#41760: DRY RUN, would close with: #41750 was fixed by #41767 on main");
});
test("a sweep summary counts what it saw and lists only the closes", () => {
const summary = describeSweep(config, { considered: 3522, pullRequests: [closed, left] });
expect(summary).toContain("Swept 3522 open pull requests, 2 linked to an issue, 1 closed");
expect(summary).toContain("#41760: closed with:");
expect(summary).not.toContain("#41762");
expect(describeSweep({ ...config, closeDryRun: true }, { considered: 3522, pullRequests: [closed] })).toContain("1 would be closed, DRY RUN, set the ISSUE_FIXED_CLOSE_PRS_ENABLED");
});
});

View file

@ -6,35 +6,66 @@ declare const process: { readonly env: Readonly<Record<string, string | undefine
export interface FixedConfig {
readonly repo: string;
readonly issueNumber: number;
readonly defaultBranch: string;
readonly dryRun: boolean;
readonly commentDryRun: boolean;
readonly closeDryRun: boolean;
}
export type Run = { readonly kind: "issue"; readonly number: number } | { readonly kind: "sweep" };
interface PullRequestCloser {
readonly __typename: "PullRequest";
readonly number: number;
readonly merged: boolean;
readonly baseRefName: string;
readonly repository: { readonly nameWithOwner: string };
readonly mergeCommit: { readonly oid: string } | null;
}
interface CommitCloser {
readonly __typename: "Commit";
readonly oid: string;
readonly repository: { readonly nameWithOwner: string };
}
export interface ClosedIssue {
export interface IssueClosure {
readonly state: "OPEN" | "CLOSED";
readonly timelineItems: {
readonly nodes: readonly { readonly closer: PullRequestCloser | CommitCloser | null }[];
};
}
export interface LinkedIssue extends IssueClosure {
readonly number: number;
readonly repository: { readonly nameWithOwner: string };
}
export interface LinkedPullRequest {
readonly number: number;
readonly state: "OPEN" | "CLOSED" | "MERGED";
readonly baseRefName: string;
readonly repository: { readonly nameWithOwner: string };
readonly closingIssuesReferences: { readonly totalCount: number; readonly nodes: readonly LinkedIssue[] };
readonly reopens: { readonly nodes: readonly { readonly createdAt: string }[] };
}
export interface PullRequestsPage {
readonly pageInfo: { readonly hasNextPage: boolean; readonly endCursor: string | null };
readonly nodes: readonly LinkedPullRequest[];
}
export interface ClosedIssue extends IssueClosure {
readonly closedByPullRequestsReferences: PullRequestsPage;
}
interface TimelineResponse {
readonly data?: { readonly repository?: { readonly issue: ClosedIssue | null } };
}
interface OpenPullRequestsResponse {
readonly data?: { readonly repository?: { readonly pullRequests: PullRequestsPage } };
}
interface MatchingRef {
readonly ref: string;
}
@ -59,33 +90,100 @@ export type FixedVerdict =
| { readonly kind: "commented"; readonly pullRequest: number; readonly tag: string; readonly body: string }
| { readonly kind: "skip"; readonly reason: string };
export const FIXED_MARKER = "<!-- litellm:fixed-in -->";
const MAX_MINOR_BUMPS = 3;
export type FixSource =
| { readonly kind: "pull_request"; readonly number: number; readonly oid: string }
| { readonly kind: "commit"; readonly oid: string };
export const CLOSER_QUERY = `query($owner: String!, $name: String!, $number: Int!) {
repository(owner: $owner, name: $name) {
issue(number: $number) {
state
timelineItems(last: 1, itemTypes: [CLOSED_EVENT]) {
nodes {
... on ClosedEvent {
closer {
__typename
... on PullRequest { number merged baseRefName mergeCommit { oid } }
... on Commit { oid }
}
}
export type FixVerdict = FixSource | { readonly kind: "skip"; readonly reason: string };
export interface Fix {
readonly issue: number;
readonly source: FixSource;
}
export type CloseVerdict =
| { readonly kind: "candidate"; readonly fixes: readonly Fix[] }
| { readonly kind: "skip"; readonly reason: string };
export type CloseOutcome =
| { readonly kind: "closed"; readonly number: number; readonly body: string }
| { readonly kind: "skip"; readonly number: number; readonly reason: string };
export interface IssueOutcome {
readonly comment: FixedVerdict;
readonly pullRequests: readonly CloseOutcome[];
}
export interface SweepOutcome {
readonly considered: number;
readonly pullRequests: readonly CloseOutcome[];
}
export const FIXED_MARKER = "<!-- litellm:fixed-in -->";
export const SUPERSEDED_MARKER = "<!-- litellm:superseded -->";
const MAX_MINOR_BUMPS = 3;
const MAX_LINKED_PULL_REQUESTS = 50;
const MAX_LINKED_ISSUES = 10;
const SWEEP_PAGE_SIZE = 100;
const CLOSE_PAUSE_MS = 1000;
const WORKFLOW_LOGIN = "github-actions[bot]";
const OPEN_AGAIN = "the issue is open again";
const isReleaseLine = (base: string): boolean => base.startsWith("release/") || base.includes("stable");
const CLOSURE_FRAGMENT = `fragment Closure on Issue {
state
timelineItems(last: 1, itemTypes: [CLOSED_EVENT]) {
nodes {
... on ClosedEvent {
closer {
__typename
... on PullRequest { number merged baseRefName repository { nameWithOwner } mergeCommit { oid } }
... on Commit { oid repository { nameWithOwner } }
}
}
}
}
}`;
const LINKED_PULL_REQUEST_FRAGMENT = `fragment Linked on PullRequest {
number
state
baseRefName
repository { nameWithOwner }
closingIssuesReferences(first: ${MAX_LINKED_ISSUES}) { totalCount nodes { number repository { nameWithOwner } ...Closure } }
reopens: timelineItems(last: 1, itemTypes: [REOPENED_EVENT]) { nodes { ... on ReopenedEvent { createdAt } } }
}`;
export const CLOSER_QUERY = `query($owner: String!, $name: String!, $number: Int!, $after: String) {
repository(owner: $owner, name: $name) {
issue(number: $number) {
...Closure
closedByPullRequestsReferences(first: ${MAX_LINKED_PULL_REQUESTS}, after: $after) {
pageInfo { hasNextPage endCursor }
nodes { ...Linked }
}
}
}
}
${CLOSURE_FRAGMENT}
${LINKED_PULL_REQUEST_FRAGMENT}`;
export const OPEN_PULL_REQUESTS_QUERY = `query($owner: String!, $name: String!, $after: String) {
repository(owner: $owner, name: $name) {
pullRequests(states: OPEN, first: ${SWEEP_PAGE_SIZE}, after: $after) {
pageInfo { hasNextPage endCursor }
nodes { ...Linked }
}
}
}
${CLOSURE_FRAGMENT}
${LINKED_PULL_REQUEST_FRAGMENT}`;
const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason });
export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer {
export function closerOf(issue: IssueClosure, repo: string, defaultBranch: string): Closer {
if (issue.state !== "CLOSED") {
return skip("the issue is open again");
return skip(OPEN_AGAIN);
}
const closer = issue.timelineItems.nodes[0]?.closer ?? null;
if (closer === null) {
@ -94,6 +192,9 @@ export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer {
if (closer.__typename === "Commit") {
return skip(`closed by commit ${closer.oid.slice(0, 10)}, not by a pull request`);
}
if (closer.repository.nameWithOwner !== repo) {
return skip(`closed by ${closer.repository.nameWithOwner}#${closer.number}, a pull request in another repository`);
}
if (!closer.merged || closer.mergeCommit === null) {
return skip(`closed by #${closer.number}, which is not merged`);
}
@ -121,8 +222,8 @@ async function tagExists(api: GitHubApi, repo: string, tag: string): Promise<boo
return refs.some((ref) => ref.ref === `refs/tags/${tag}`);
}
async function tagContains(api: GitHubApi, repo: string, tag: string, sha: string): Promise<boolean> {
const comparison = await api.request<Comparison>("GET", `/repos/${repo}/compare/${tag}...${sha}`);
async function refContains(api: GitHubApi, repo: string, ref: string, sha: string): Promise<boolean> {
const comparison = await api.request<Comparison>("GET", `/repos/${repo}/compare/${ref}...${sha}`);
return comparison.status === "behind" || comparison.status === "identical";
}
@ -139,7 +240,7 @@ async function firstReleaseWith(
if (!(await tagExists(api, repo, tag))) {
return { kind: "release", tag, shipped: false };
}
if (await tagContains(api, repo, tag, sha)) {
if (await refContains(api, repo, tag, sha)) {
return { kind: "release", tag, shipped: true };
}
if (bumpsLeft === 0) {
@ -164,21 +265,13 @@ export function fixedBody(pullRequest: number, release: { readonly tag: string;
return `${FIXED_MARKER}\nFixed by #${pullRequest}. ${availability}`;
}
export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Promise<FixedVerdict> {
const [owner, name] = config.repo.split("/");
const response = await api.request<TimelineResponse>("POST", "/graphql", {
query: CLOSER_QUERY,
variables: { owner, name, number: config.issueNumber },
});
const issue = response.data?.repository?.issue ?? null;
if (issue === null) {
return skip("not an issue in this repository");
}
const closer = closerOf(issue, config.defaultBranch);
if (closer.kind === "skip") {
return closer;
}
const issuePath = `/repos/${config.repo}/issues/${config.issueNumber}`;
export async function commentFixedIssue(
api: GitHubApi,
config: FixedConfig,
issueNumber: number,
closer: { readonly number: number; readonly mergeCommit: string },
): Promise<FixedVerdict> {
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
if (comments.some((comment) => comment.body.includes(FIXED_MARKER))) {
return skip("already carries a fixed-in comment");
@ -188,37 +281,259 @@ export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Pr
return release;
}
const body = fixedBody(closer.number, release);
if (!config.dryRun) {
if (!config.commentDryRun) {
await api.request("POST", `${issuePath}/comments`, { body });
}
return { kind: "commented", pullRequest: closer.number, tag: release.tag, body };
}
export function readConfig(env: Readonly<Record<string, string | undefined>>): FixedConfig & { readonly token: string } {
export function fixOf(issue: IssueClosure, repo: string): FixVerdict {
if (issue.state !== "CLOSED") {
return skip("is open again");
}
const closer = issue.timelineItems.nodes[0]?.closer ?? null;
if (closer === null) {
return skip("was closed by hand");
}
if (closer.repository.nameWithOwner !== repo) {
return skip(`was closed from ${closer.repository.nameWithOwner}`);
}
if (closer.__typename === "Commit") {
return { kind: "commit", oid: closer.oid };
}
if (!closer.merged || closer.mergeCommit === null) {
return skip(`was closed by #${closer.number}, which is not merged`);
}
return { kind: "pull_request", number: closer.number, oid: closer.mergeCommit.oid };
}
export function closeVerdict(pullRequest: LinkedPullRequest, config: FixedConfig): CloseVerdict {
if (pullRequest.repository.nameWithOwner !== config.repo) {
return skip(`lives in ${pullRequest.repository.nameWithOwner}`);
}
if (pullRequest.state !== "OPEN") {
return skip(`is ${pullRequest.state.toLowerCase()}`);
}
if (isReleaseLine(pullRequest.baseRefName)) {
return skip(`targets the release line ${pullRequest.baseRefName}`);
}
const { totalCount, nodes: linked } = pullRequest.closingIssuesReferences;
if (linked.length === 0) {
return skip("links no issue");
}
if (totalCount > linked.length) {
return skip(`links ${totalCount} issues, more than the ${MAX_LINKED_ISSUES} this workflow reads`);
}
const foreign = linked.find((issue) => issue.repository.nameWithOwner !== config.repo);
if (foreign !== undefined) {
return skip(`links ${foreign.repository.nameWithOwner}#${foreign.number}`);
}
const stillOpen = linked.find((issue) => issue.state === "OPEN");
if (stillOpen !== undefined) {
return skip(`still linked to open #${stillOpen.number}`);
}
const verdicts = linked.map((issue) => ({ issue: issue.number, fix: fixOf(issue, config.repo) }));
for (const { issue, fix } of verdicts) {
if (fix.kind === "skip") {
return skip(`#${issue} ${fix.reason}`);
}
}
return {
kind: "candidate",
fixes: verdicts.flatMap(({ issue, fix }) => (fix.kind === "skip" ? [] : [{ issue, source: fix }])),
};
}
function describeSource(source: FixSource): string {
return source.kind === "pull_request" ? `#${source.number}` : `commit ${source.oid.slice(0, 10)}`;
}
async function fixOffDefaultBranch(api: GitHubApi, config: FixedConfig, fixes: readonly Fix[]): Promise<Fix | undefined> {
const onBranch = await Promise.all(fixes.map((fix) => refContains(api, config.repo, config.defaultBranch, fix.source.oid)));
return fixes.find((_, index) => !onBranch[index]);
}
export function supersededBody(fixes: readonly Fix[], defaultBranch: string): string {
const pairs = fixes
.map((fix, index) => `#${fix.issue} ${index === 0 ? "was fixed by" : "by"} ${describeSource(fix.source)}`)
.join(" and ");
return `${SUPERSEDED_MARKER}\n${pairs} on ${defaultBranch}, so this pull request is closed. Reopen it if something was missed.`;
}
function reopenedAfter(pullRequest: LinkedPullRequest, comment: Comment): boolean {
const reopen = pullRequest.reopens.nodes[0];
return reopen !== undefined && Date.parse(reopen.createdAt) > Date.parse(comment.created_at);
}
async function closePullRequest(
api: GitHubApi,
config: FixedConfig,
pullRequest: LinkedPullRequest,
pause: () => Promise<void>,
): Promise<CloseOutcome> {
const verdict = closeVerdict(pullRequest, config);
if (verdict.kind === "skip") {
return { kind: "skip", number: pullRequest.number, reason: verdict.reason };
}
const offBranch = await fixOffDefaultBranch(api, config, verdict.fixes);
if (offBranch !== undefined) {
const fix = describeSource(offBranch.source);
return { kind: "skip", number: pullRequest.number, reason: `#${offBranch.issue} was fixed by ${fix}, which is not on ${config.defaultBranch}` };
}
const issuePath = `/repos/${config.repo}/issues/${pullRequest.number}`;
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
const marker = comments.find((comment) => comment.user.login === WORKFLOW_LOGIN && comment.body.includes(SUPERSEDED_MARKER));
if (marker !== undefined && reopenedAfter(pullRequest, marker)) {
return { kind: "skip", number: pullRequest.number, reason: "was closed by this workflow once and reopened" };
}
const body = marker?.body ?? supersededBody(verdict.fixes, config.defaultBranch);
if (config.closeDryRun) {
return { kind: "closed", number: pullRequest.number, body };
}
if (marker === undefined) {
await pause();
await api.request("POST", `${issuePath}/comments`, { body });
}
await pause();
await api.request("PATCH", `/repos/${config.repo}/pulls/${pullRequest.number}`, { state: "closed" });
return { kind: "closed", number: pullRequest.number, body };
}
export function closePullRequests(
api: GitHubApi,
config: FixedConfig,
candidates: readonly LinkedPullRequest[],
pause: () => Promise<void>,
): Promise<readonly CloseOutcome[]> {
return candidates.reduce<Promise<readonly CloseOutcome[]>>(
async (previous, candidate) => [...(await previous), await closePullRequest(api, config, candidate, pause)],
Promise.resolve([]),
);
}
type NextPage = (after: string | null) => Promise<PullRequestsPage>;
async function collectPages(page: PullRequestsPage, nextPage: NextPage): Promise<readonly LinkedPullRequest[]> {
if (!page.pageInfo.hasNextPage) {
return page.nodes;
}
return [...page.nodes, ...(await collectPages(await nextPage(page.pageInfo.endCursor), nextPage))];
}
async function closedIssue(api: GitHubApi, config: FixedConfig, issueNumber: number, after: string | null): Promise<ClosedIssue | null> {
const [owner, name] = config.repo.split("/");
const response = await api.request<TimelineResponse>("POST", "/graphql", {
query: CLOSER_QUERY,
variables: { owner, name, number: issueNumber, after },
});
return response.data?.repository?.issue ?? null;
}
export async function handleFixedIssue(
api: GitHubApi,
config: FixedConfig,
issueNumber: number,
pause: () => Promise<void>,
): Promise<IssueOutcome> {
const issue = await closedIssue(api, config, issueNumber, null);
if (issue === null) {
return { comment: skip("not an issue in this repository"), pullRequests: [] };
}
if (issue.state !== "CLOSED") {
return { comment: skip(OPEN_AGAIN), pullRequests: [] };
}
const closer = closerOf(issue, config.repo, config.defaultBranch);
const comment = closer.kind === "skip" ? closer : await commentFixedIssue(api, config, issueNumber, closer);
const nextPage: NextPage = async (after) => {
const more = await closedIssue(api, config, issueNumber, after);
if (more === null) {
throw new Error(`#${issueNumber} came back without data while reading its linked pull requests after cursor ${after}`);
}
return more.closedByPullRequestsReferences;
};
const linked = await collectPages(issue.closedByPullRequestsReferences, nextPage);
const open = linked.filter((pullRequest) => pullRequest.state === "OPEN");
const pullRequests = await closePullRequests(api, config, open, pause);
return { comment, pullRequests };
}
async function openPullRequests(api: GitHubApi, config: FixedConfig): Promise<readonly LinkedPullRequest[]> {
const [owner, name] = config.repo.split("/");
const nextPage: NextPage = async (after) => {
const response = await api.request<OpenPullRequestsResponse>("POST", "/graphql", {
query: OPEN_PULL_REQUESTS_QUERY,
variables: { owner, name, after },
});
const page = response.data?.repository?.pullRequests;
if (page === undefined) {
throw new Error(`open pull requests after cursor ${after} came back without data: ${JSON.stringify(response)}`);
}
return page;
};
return collectPages(await nextPage(null), nextPage);
}
export async function sweep(api: GitHubApi, config: FixedConfig, pause: () => Promise<void>): Promise<SweepOutcome> {
const open = await openPullRequests(api, config);
const linked = open.filter((pullRequest) => pullRequest.closingIssuesReferences.nodes.length > 0);
return { considered: open.length, pullRequests: await closePullRequests(api, config, linked, pause) };
}
export function readConfig(
env: Readonly<Record<string, string | undefined>>,
): FixedConfig & { readonly token: string; readonly run: Run } {
const token = env.GITHUB_TOKEN;
const repo = env.GITHUB_REPOSITORY;
const defaultBranch = env.DEFAULT_BRANCH;
if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo) || !defaultBranch) {
throw new Error("GITHUB_TOKEN, GITHUB_REPOSITORY (owner/repo) and DEFAULT_BRANCH are required");
}
const config = { token, repo, defaultBranch, commentDryRun: env.DRY_RUN === "true", closeDryRun: env.CLOSE_PRS_DRY_RUN === "true" };
if (env.SWEEP === "true") {
return { ...config, run: { kind: "sweep" } };
}
const issueNumber = Number(env.ISSUE_NUMBER);
if (!Number.isInteger(issueNumber) || issueNumber <= 0) {
throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}"`);
throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}": dispatch with an issue_number or with sweep ticked`);
}
return { token, repo, issueNumber, defaultBranch, dryRun: env.DRY_RUN === "true" };
return { ...config, run: { kind: "issue", number: issueNumber } };
}
function describe(config: FixedConfig, verdict: FixedVerdict): string {
if (verdict.kind === "skip") {
return `#${config.issueNumber}: skipped, ${verdict.reason}`;
const CLOSE_DRY_RUN_HINT = "set the ISSUE_FIXED_CLOSE_PRS_ENABLED repo variable to true to close pull requests";
function describeClose(config: FixedConfig, outcome: CloseOutcome): string {
if (outcome.kind === "skip") {
return `#${outcome.number}: left open, ${outcome.reason}`;
}
if (config.dryRun) {
return `#${config.issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${verdict.body}`;
}
return `#${config.issueNumber}: commented, fixed by #${verdict.pullRequest} in ${verdict.tag}`;
const text = outcome.body.replace(`${SUPERSEDED_MARKER}\n`, "");
return config.closeDryRun ? `#${outcome.number}: DRY RUN, would close with: ${text}` : `#${outcome.number}: closed with: ${text}`;
}
export function describeIssue(config: FixedConfig, issueNumber: number, outcome: IssueOutcome): string {
const comment =
outcome.comment.kind === "skip"
? `#${issueNumber}: skipped, ${outcome.comment.reason}`
: config.commentDryRun
? `#${issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${outcome.comment.body}`
: `#${issueNumber}: commented, fixed by #${outcome.comment.pullRequest} in ${outcome.comment.tag}`;
const hint = config.closeDryRun && outcome.pullRequests.some((pullRequest) => pullRequest.kind === "closed") ? [`Closing is a DRY RUN, ${CLOSE_DRY_RUN_HINT}`] : [];
return [comment, ...hint, ...outcome.pullRequests.map((pullRequest) => describeClose(config, pullRequest))].join("\n");
}
export function describeSweep(config: FixedConfig, outcome: SweepOutcome): string {
const closed = outcome.pullRequests.filter((pullRequest) => pullRequest.kind === "closed");
const verb = config.closeDryRun ? `would be closed, DRY RUN, ${CLOSE_DRY_RUN_HINT}` : "closed";
const header = `Swept ${outcome.considered} open pull requests, ${outcome.pullRequests.length} linked to an issue, ${closed.length} ${verb}`;
return [header, ...closed.map((pullRequest) => describeClose(config, pullRequest))].join("\n");
}
if (import.meta.main) {
const { token, ...config } = readConfig(process.env);
console.log(describe(config, await commentFixedIssue(githubApi(token), config)));
const { token, run, ...config } = readConfig(process.env);
const api = githubApi(token);
const pause = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, CLOSE_PAUSE_MS));
console.log(
run.kind === "sweep"
? describeSweep(config, await sweep(api, config, pause))
: describeIssue(config, run.number, await handleFixedIssue(api, config, run.number, pause)),
);
}

View file

@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
"_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible).
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
]

View file

@ -46,6 +46,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak)
- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/<name>` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=<system>` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager_<system>_ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_<system>.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up <system>` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only)
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`

View file

@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the
### The pull request check
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, `load/`, and `secret_manager/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set. The `secret_manager/` lanes each need a proxy configured against their own secret manager (see Secret manager lanes below)
Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
@ -117,6 +117,31 @@ Fetched values of eight characters or more are masked before use, while shorter
To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
### Secret manager lanes
`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager_<system>_ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally:
```bash
bash tests/e2e/secret_manager/backend.sh up cyberark
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \
LITELLM_MASTER_KEY=sk-1234 DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000)
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \
uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v)
bash tests/e2e/secret_manager/backend.sh down cyberark
```
`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors
A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane
To add a backend, leave the tests and markers alone and add:
1. `secret_manager/secret_store_<system>.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E_<SYSTEM>_*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports
2. its entry in `secret_backends.BACKENDS`
3. `gateway/secret_manager_<system>_ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed
4. an `up_<system>` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env`
5. a CI step that runs `backend.sh up <system>` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env`
### Record and replay
Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop

View file

@ -22,6 +22,7 @@ failures are hard test failures (see `tests/e2e/AGENTS.md`).
| Bedrock | yes (unified only) | yes | yes | yes (unfiltered managed list) | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` on model) |
| Bedrock GovCloud (`us-gov-west-1`) | yes (unified only) | yes | no | no | yes (provider-transformed) | S3 (`s3_bucket_name` + `aws_*` on model, resolved from `AWS_GOVCLOUD_ACCESS_KEY_ID` / `AWS_GOVCLOUD_SECRET_ACCESS_KEY` / `AWS_GOVCLOUD_BATCH_S3_BUCKET` / `AWS_GOVCLOUD_BATCH_ROLE_ARN`) |
| Bedrock split S3 identity | no | no | no | no | yes (file upload, content, delete) | S3 signed with `s3_access_key_id` / `s3_secret_access_key` (`AWS_S3_ONLY_ACCESS_KEY_ID` / `AWS_S3_ONLY_SECRET_ACCESS_KEY`, object rights on `AWS_BATCH_S3_BUCKET` only) while `aws_*` is `AWS_BEDROCK_ONLY_ACCESS_KEY_ID` / `AWS_BEDROCK_ONLY_SECRET_ACCESS_KEY`, an identity with no S3 rights on that bucket |
| Bedrock blank S3 env | yes (unified only, on an owned gateway exporting `AWS_S3_ENCRYPTION_KEY_ID` / `AWS_S3_BUCKET_OWNER` as empty strings) | no | no | no | no | S3 (`s3_bucket_name` + `aws_*` + `AWS_BATCH_ROLE_ARN` in the gateway config); blank env vars must be treated as unset, not serialized |
Bedrock cancel maps to `StopModelInvocationJob` and comes back `cancelling`; the
lifecycle asserts it the same way it does for OpenAI (`_CANCEL_ASSERTED_PROVIDERS`).

View file

@ -0,0 +1,145 @@
"""An owned, source-built proxy whose process env exports AWS_S3_* vars blank.
The shared fixture proxy inherits the harness env, which cannot reproduce a user
shell that exports AWS_S3_ENCRYPTION_KEY_ID / AWS_S3_BUCKET_OWNER as empty
strings. This gateway boots a second proxy with both vars present but blank, so
a batch create through it proves blank means unset, not an empty string.
"""
from __future__ import annotations
import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Final
from e2e_config import unique_marker
from e2e_http import NoBody
from idp import stop_process_group
from proxy_client import ProxyClient, build_proxy_client
from pydantic import TypeAdapter
STARTUP_TIMEOUT_SECONDS: Final = 240
LOG_TAIL_BYTES: Final = 4000
REPO_ROOT: Final = Path(__file__).resolve().parents[3]
_CONFIG_YAML: Final = """model_list:
- model_name: bedrock-blank-s3-batch
litellm_params:
model: bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0
aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID
aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_region_name: os.environ/AWS_REGION
s3_region_name: os.environ/AWS_REGION
s3_bucket_name: os.environ/AWS_BATCH_S3_BUCKET
s3_access_key_id: os.environ/AWS_ACCESS_KEY_ID
s3_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY
aws_batch_role_arn: os.environ/AWS_BATCH_ROLE_ARN
general_settings:
master_key: os.environ/LITELLM_MASTER_KEY
database_url: os.environ/DATABASE_URL
"""
def available_port() -> int:
with socket.socket() as listener:
listener.bind(("127.0.0.1", 0))
return TypeAdapter(tuple[str, int]).validate_python(listener.getsockname())[1]
@dataclass(slots=True)
class BedrockEnvGateway:
base_url: str
master_key: str
proxy: ProxyClient
_environment: Mapping[str, str] = field(repr=False)
_command: tuple[str, ...] = field(repr=False)
_log_path: Path
_child: subprocess.Popen[bytes] | None = field(default=None, init=False, repr=False)
@classmethod
def start(cls) -> BedrockEnvGateway:
assert os.environ.get("DATABASE_URL"), "DATABASE_URL is required for the blank-S3-env gateway"
port: Final = available_port()
base_url: Final = f"http://127.0.0.1:{port}"
master_key: Final = f"sk-e2e-blank-s3-{unique_marker()}"
directory: Final = Path(tempfile.mkdtemp(prefix="litellm-e2e-blank-s3-"))
config: Final = directory / "blank-s3-gateway.yaml"
config.write_text(_CONFIG_YAML)
environment: Final = {
**{key: value for key, value in os.environ.items() if not key.startswith("REDIS_")},
"DATABASE_URL": os.environ["DATABASE_URL"],
"LITELLM_MASTER_KEY": master_key,
"STORE_MODEL_IN_DB": "False",
"PYTHONPATH": str(REPO_ROOT),
"AWS_S3_ENCRYPTION_KEY_ID": "",
"AWS_S3_BUCKET_OWNER": "",
}
gateway: Final = cls(
base_url=base_url,
master_key=master_key,
proxy=build_proxy_client(
base_url=base_url,
control_plane_base_url=base_url,
replica_urls=(base_url,),
master_key=master_key,
),
_environment=environment,
_command=(
sys.executable,
"-m",
"litellm.proxy.proxy_cli",
"--config",
str(config),
"--port",
str(port),
"--host",
"127.0.0.1",
),
_log_path=directory / "blank-s3-gateway.log",
)
with gateway._log_path.open("ab") as log:
gateway._child = subprocess.Popen(
gateway._command,
env=dict(gateway._environment),
stdout=log,
stderr=log,
start_new_session=True,
cwd=REPO_ROOT,
)
deadline: Final = time.monotonic() + STARTUP_TIMEOUT_SECONDS
while time.monotonic() < deadline:
assert gateway._child.poll() is None, (
f"blank-S3-env gateway exited early; log tail:\n{gateway.log_tail()}"
)
result = gateway.proxy.transport.probe("/health/liveliness", params=NoBody())
if result.status_code == 200:
return gateway
time.sleep(0.5)
tail: Final = gateway.log_tail()
gateway.stop()
raise AssertionError(
f"blank-S3-env gateway did not become ready in {STARTUP_TIMEOUT_SECONDS}s; log tail:\n{tail}"
)
def log_tail(self) -> str:
if not self._log_path.exists():
return "<no log written>"
with self._log_path.open("rb") as log:
log.seek(0, 2)
size: Final = log.tell()
log.seek(max(0, size - LOG_TAIL_BYTES))
return log.read().decode("utf-8", errors="replace")
def stop(self) -> None:
if self._child is not None:
stop_process_group(self._child)
shutil.rmtree(self._log_path.parent, ignore_errors=True)

Some files were not shown because too many files have changed in this diff Show more