mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm_bound_auth_cache_publish_wedged_redis
This commit is contained in:
commit
514ad8b2e6
53 changed files with 2340 additions and 312 deletions
10
.github/workflows/test-rust.yml
vendored
10
.github/workflows/test-rust.yml
vendored
|
|
@ -105,6 +105,16 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install Python dependencies for the bridge tests
|
||||
working-directory: .
|
||||
run: |
|
||||
uv sync --frozen --no-install-project
|
||||
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
||||
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
|
||||
|
|
|
|||
21
litellm-rust/Cargo.lock
generated
21
litellm-rust/Cargo.lock
generated
|
|
@ -2946,6 +2946,7 @@ name = "litellm-core-utils"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -3075,6 +3076,7 @@ dependencies = [
|
|||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
|
|
@ -3093,6 +3095,7 @@ dependencies = [
|
|||
"litellm-callbacks-legacy-python",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
@ -3100,6 +3103,7 @@ dependencies = [
|
|||
"litellm-secrets-aws",
|
||||
"litellm-secrets-types",
|
||||
"litellm-token-counter",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"pyo3",
|
||||
"pyo3-async-runtimes",
|
||||
|
|
@ -3172,11 +3176,11 @@ dependencies = [
|
|||
"litellm-auth-aws",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3208,6 +3212,7 @@ dependencies = [
|
|||
"base64 0.22.1",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"moka",
|
||||
"percent-encoding",
|
||||
"reqwest 0.12.28",
|
||||
|
|
@ -3216,7 +3221,6 @@ dependencies = [
|
|||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3331,6 +3335,19 @@ dependencies = [
|
|||
"tiktoken-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"percent-encoding",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ license = "MIT"
|
|||
repository = "https://github.com/BerriAI/litellm"
|
||||
|
||||
[workspace.dependencies]
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
tracing = "0.1"
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-host = { path = "crates/host" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-types.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,109 +1 @@
|
|||
use fancy_regex::Regex;
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn minimum_custom_key_length() -> usize {
|
||||
std::env::var("MINIMUM_CUSTOM_KEY_LENGTH")
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
}
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
Self { pattern }
|
||||
}
|
||||
|
||||
/// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS")
|
||||
.is_ok_and(|value| value.eq_ignore_ascii_case("true"));
|
||||
(!disabled).then(|| Self::new(minimum_custom_key_length()))
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.pattern.replace_all(value, REDACTED).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
pub use litellm_tracing::{REDACTED, SecretRedactor};
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"]
|
|||
tiktoken = ["litellm-token-counter/tiktoken"]
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-host.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use crate::logger::run_async;
|
||||
use litellm_cache_response::PartialHits;
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py};
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_auth_aws::AwsAuthConfig;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization};
|
||||
use litellm_cache_redis::{RedisNode, RedisTopology};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
|
|
|
|||
|
|
@ -470,7 +470,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move {
|
||||
service
|
||||
|
|
@ -495,7 +495,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_lookup(&request, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -550,7 +550,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store(&request, response, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -619,7 +619,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store_batch(entries, now()).await },
|
||||
super::cache_error,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::{collections::VecDeque, time::Duration};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyException, PyRuntimeError},
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ pub(crate) fn call_config(
|
|||
.without_missing_files(&|path: &Path| path.exists());
|
||||
let resolution = Resolution::from(&settings);
|
||||
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
|
||||
PythonSettings::warn(py, &unsupported.to_string())?;
|
||||
crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}"));
|
||||
}
|
||||
Ok(resolution.config)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ mod credentials;
|
|||
mod diagnostics;
|
||||
mod errors;
|
||||
mod http;
|
||||
mod logger;
|
||||
mod marshal;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
|
|
@ -25,6 +26,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::errors::{RustBridgeDeclined, RustUpstreamError};
|
||||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{
|
||||
|
|
@ -87,6 +90,7 @@ mod tests {
|
|||
"chat_completions",
|
||||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"NativeDiagnosticProcessor",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
|
|
|
|||
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
use std::future::Future;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use serde::Serialize;
|
||||
|
||||
pub(crate) fn run_sync<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_async<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
|
||||
pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, Step},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_tracing::Logger;
|
||||
use pyo3::Python;
|
||||
|
||||
pub(crate) struct LoggedMachine<M> {
|
||||
machine: M,
|
||||
logger: OnceLock<Logger>,
|
||||
}
|
||||
|
||||
impl<M> LoggedMachine<M> {
|
||||
pub(crate) fn new(machine: M) -> Self {
|
||||
Self {
|
||||
machine,
|
||||
logger: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<M: Machine> Machine for LoggedMachine<M> {
|
||||
type Route = M::Route;
|
||||
type Complete = M::Complete;
|
||||
|
||||
fn resume(&mut self, result: Option<HostResult<Self::Route>>) -> Step<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result))))
|
||||
}
|
||||
|
||||
fn interrupt(
|
||||
&mut self,
|
||||
failure: HostFailure<<Self::Route as Route>::Error>,
|
||||
) -> Interrupted<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
|
||||
}
|
||||
}
|
||||
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
mod execution;
|
||||
mod machine;
|
||||
|
||||
pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
|
||||
pub(crate) use machine::LoggedMachine;
|
||||
|
||||
use litellm_host_python::Pythonized;
|
||||
use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink};
|
||||
use pyo3::exceptions::PyRuntimeError;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.logger";
|
||||
type NativeDiagnosticOutput = (String, Option<String>, Option<String>, Vec<String>, bool);
|
||||
|
||||
#[pyclass]
|
||||
pub(crate) struct NativeDiagnosticProcessor {
|
||||
inner: Processor,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl NativeDiagnosticProcessor {
|
||||
#[new]
|
||||
fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
inner: Processor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_text(&self, text: &str) -> PyResult<String> {
|
||||
self.inner.redact_text(text).map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_structured_text(key, text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_client_message(&self, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_client_message(text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (message, exception, stack, leaves, policy))]
|
||||
fn process_diagnostic(
|
||||
&self,
|
||||
message: String,
|
||||
exception: Option<String>,
|
||||
stack: Option<String>,
|
||||
leaves: Vec<(Option<String>, String)>,
|
||||
policy: (bool, i64, i64),
|
||||
) -> PyResult<NativeDiagnosticOutput> {
|
||||
let input = DiagnosticInput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
};
|
||||
let policy = Policy {
|
||||
redact: policy.0,
|
||||
base64_limit: policy.1,
|
||||
text_limit: policy.2,
|
||||
};
|
||||
self.inner
|
||||
.process_diagnostic(&input, policy)
|
||||
.map(|output| {
|
||||
(
|
||||
output.message,
|
||||
output.exception,
|
||||
output.stack,
|
||||
output.leaves,
|
||||
output.changed,
|
||||
)
|
||||
})
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn scrub_access_arguments(&self, arguments: Vec<String>) -> PyResult<Vec<String>> {
|
||||
self.inner
|
||||
.scrub_access_arguments(&arguments)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
}
|
||||
|
||||
fn processing_error(_: fancy_regex::Error) -> PyErr {
|
||||
PyRuntimeError::new_err("diagnostic processing failed")
|
||||
}
|
||||
|
||||
struct PythonSink {
|
||||
correlation: (String, String),
|
||||
}
|
||||
|
||||
fn level(level: &Level) -> u8 {
|
||||
match *level {
|
||||
Level::ERROR => 40,
|
||||
Level::WARN => 30,
|
||||
Level::INFO => 20,
|
||||
Level::DEBUG | Level::TRACE => 10,
|
||||
}
|
||||
}
|
||||
|
||||
fn report<T: Default>(py: Python<'_>, result: PyResult<T>) -> T {
|
||||
match result {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
error.write_unraisable(py, None);
|
||||
T::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink for PythonSink {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
|
||||
.and_then(|enabled| enabled.extract()),
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE).and_then(|module| {
|
||||
module
|
||||
.call_method1(
|
||||
"emit",
|
||||
(
|
||||
level(record.metadata.level()),
|
||||
&record.message,
|
||||
record.metadata.file().unwrap_or_default(),
|
||||
record.metadata.line().unwrap_or_default(),
|
||||
record.metadata.target(),
|
||||
Pythonized(&record.fields),
|
||||
(&self.correlation.0, &self.correlation.1),
|
||||
),
|
||||
)
|
||||
.map(|_| ())
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture(py: Python<'_>) -> Logger {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method0("context"))
|
||||
.and_then(|value| value.extract())
|
||||
.map(|correlation| Logger::new(PythonSink { correlation })),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
use std::{process::Command, task::Poll};
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
|
||||
route::Route,
|
||||
};
|
||||
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
struct DiagnosticMachine;
|
||||
|
||||
impl Route for DiagnosticMachine {
|
||||
type Response = ();
|
||||
type Error = String;
|
||||
type Op = ();
|
||||
type OpResult = ();
|
||||
type Chunk = ();
|
||||
type StreamHead = ();
|
||||
}
|
||||
|
||||
impl Machine for DiagnosticMachine {
|
||||
type Route = Self;
|
||||
type Complete = ();
|
||||
|
||||
fn resume(&mut self, _: Option<HostResult<Self>>) -> Step<'_, Self> {
|
||||
litellm_tracing::warn!("machine started");
|
||||
Box::pin(async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("machine warning");
|
||||
Ok(MachineStep::Complete(()))
|
||||
})
|
||||
}
|
||||
|
||||
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
|
||||
Box::pin(async {
|
||||
litellm_tracing::warn!("machine interrupted");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
|
||||
let mut future = Box::pin(async move {
|
||||
machine
|
||||
.resume(None)
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)?;
|
||||
machine
|
||||
.interrupt(HostFailure::Error("stop".into()))
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)
|
||||
});
|
||||
assert!(matches!(
|
||||
litellm_host_python::poll_async_value(py, future.as_mut())?,
|
||||
Poll::Pending
|
||||
));
|
||||
litellm_host_python::run_async_value(py, future)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn warning(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::warn!(attempt = 3, retry = true, "native warning");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn levels(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::trace!("trace");
|
||||
litellm_tracing::debug!("debug");
|
||||
litellm_tracing::info!("info");
|
||||
litellm_tracing::warn!("warn");
|
||||
litellm_tracing::error!("error");
|
||||
litellm_tracing::warn!(target: "unrelated_transport", "private wire data");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
super::run_async_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("async warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("sync warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
litellm_tracing::warn!("failure diagnostic");
|
||||
Err(pyo3::exceptions::PyValueError::new_err("request failed"))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn http_warning(py: Python<'_>) -> PyResult<()> {
|
||||
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
|
||||
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
|
||||
let output = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
std::thread::current().name().unwrap(),
|
||||
"--nocapture",
|
||||
])
|
||||
.env("LITELLM_LOGGER_TEST_PROCESS", "1")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals
|
||||
.set_item(
|
||||
"repo_root",
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"machine_warning",
|
||||
wrap_pyfunction!(machine_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_failure",
|
||||
wrap_pyfunction!(synchronous_failure, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("levels", wrap_pyfunction!(levels, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("warning", wrap_pyfunction!(warning, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"asynchronous_warning",
|
||||
wrap_pyfunction!(asynchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_warning",
|
||||
wrap_pyfunction!(synchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
|
||||
.unwrap();
|
||||
let importable = py
|
||||
.eval(
|
||||
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap()
|
||||
.is_truthy()
|
||||
.unwrap();
|
||||
if !importable {
|
||||
eprintln!("SKIP: litellm package dependencies are not importable in this interpreter");
|
||||
return;
|
||||
}
|
||||
py.run(c"
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
sys.path.insert(0, repo_root)
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, session_id_var, trace_id_var
|
||||
|
||||
class Capture(logging.Handler):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.records = []
|
||||
def emit(self, record):
|
||||
self.records.append(record)
|
||||
warning()
|
||||
|
||||
class Broken(logging.Handler):
|
||||
def emit(self, record):
|
||||
raise ValueError('handler failed')
|
||||
|
||||
capture = Capture()
|
||||
old_handlers = verbose_logger.handlers
|
||||
old_level = verbose_logger.level
|
||||
old_correlation = litellm.request_correlation_in_logs
|
||||
old_curve = litellm.ssl_ecdh_curve
|
||||
old_unraisable = sys.unraisablehook
|
||||
failures = []
|
||||
try:
|
||||
verbose_logger.handlers = [capture]
|
||||
litellm.request_correlation_in_logs = True
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert capture.records == []
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
record = capture.records[0]
|
||||
assert record.getMessage() == 'native warning'
|
||||
assert record.levelno == logging.WARNING
|
||||
assert record.rust_fields == {'attempt': 3, 'retry': True}
|
||||
assert record.pathname.endswith('logger/tests.rs')
|
||||
assert record.lineno > 0
|
||||
assert record.rust_target.endswith('logger::tests')
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
|
||||
async def request(name):
|
||||
session = session_id_var.set(name)
|
||||
trace = trace_id_var.set('trace-' + name)
|
||||
try:
|
||||
await asynchronous_warning()
|
||||
await machine_warning()
|
||||
synchronous_warning()
|
||||
assert session_id_var.get() == name
|
||||
assert trace_id_var.get() == 'trace-' + name
|
||||
finally:
|
||||
trace_id_var.reset(trace)
|
||||
session_id_var.reset(session)
|
||||
|
||||
async def concurrent():
|
||||
await asyncio.gather(request('first'), request('second'))
|
||||
|
||||
asyncio.run(concurrent())
|
||||
assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted(
|
||||
(message, name, 'trace-' + name)
|
||||
for name in ('first', 'second')
|
||||
for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted')
|
||||
)
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
before_levels = len(capture.records)
|
||||
levels()
|
||||
assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [
|
||||
('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO),
|
||||
('warn', logging.WARNING), ('error', logging.ERROR),
|
||||
]
|
||||
|
||||
before = len(capture.records)
|
||||
litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve'
|
||||
http_warning()
|
||||
http_warning()
|
||||
assert len(capture.records) == before + 1
|
||||
assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage()
|
||||
assert capture.records[-1].pathname.endswith('http.rs')
|
||||
|
||||
verbose_logger.handlers = [Broken()]
|
||||
sys.unraisablehook = failures.append
|
||||
warning()
|
||||
assert len(failures) == 1
|
||||
assert str(failures[0].exc_value) == 'handler failed'
|
||||
try:
|
||||
synchronous_failure()
|
||||
except ValueError as error:
|
||||
assert str(error) == 'request failed'
|
||||
else:
|
||||
raise AssertionError('request failure was lost')
|
||||
assert len(failures) == 2
|
||||
finally:
|
||||
sys.unraisablehook = old_unraisable
|
||||
verbose_logger.handlers = old_handlers
|
||||
verbose_logger.setLevel(old_level)
|
||||
litellm.request_correlation_in_logs = old_correlation
|
||||
litellm.ssl_ecdh_curve = old_curve
|
||||
", Some(&locals), Some(&locals)).unwrap();
|
||||
});
|
||||
}
|
||||
|
|
@ -44,11 +44,6 @@ impl PythonSettings {
|
|||
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
|
||||
Snapshot { group: self, value }
|
||||
}
|
||||
|
||||
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
|
||||
py.import(MODULE)?.getattr("warn")?.call1((message,))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::audio_transcription::{
|
||||
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::chat_completions::{
|
||||
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
|
||||
types::ChatCompletionsRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use litellm_types::utils::ChatCompletionsResponse;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ fn run_messages(
|
|||
py,
|
||||
SURFACE,
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
messages_machine(),
|
||||
crate::logger::LoggedMachine::new(messages_machine()),
|
||||
MessagesRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ fn run_ocr(
|
|||
py,
|
||||
if asynchronous { ASYNC_SURFACE } else { SURFACE },
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
ocr_machine(client),
|
||||
crate::logger::LoggedMachine::new(ocr_machine(client)),
|
||||
OcrRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection {
|
|||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let headers = marshal_headers(headers)?;
|
||||
let timeout = optional_timeout(timeout_seconds);
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
|
||||
.await
|
||||
.map_err(responses_error_to_pyerr)?;
|
||||
|
|
@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner
|
||||
.send_text(text)
|
||||
.await
|
||||
|
|
@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.recv_text().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
||||
fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.close().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_host_python::enter_native;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true
|
|||
litellm-core-utils.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
veil.workspace = true
|
||||
aws-sdk-kms = "1.120.0"
|
||||
aws-sdk-secretsmanager = "1.117.0"
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 {
|
|||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("secret created but replication failed");
|
||||
litellm_tracing::warn!("secret created but replication failed");
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ reqwest.workspace = true
|
|||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
veil.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
percent-encoding = "2.3"
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ impl CyberArkSecretManager {
|
|||
.unwrap_or(true);
|
||||
let mut builder = reqwest::Client::builder();
|
||||
if !verify {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
|
||||
);
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
|
|
@ -259,11 +259,13 @@ impl CyberArkSecretManager {
|
|||
.endpoint
|
||||
.join(&format!("policies/{}/policy/root", self.account));
|
||||
let Ok(policy_url) = policy_url else {
|
||||
tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
litellm_tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
return;
|
||||
};
|
||||
let Ok(authorization) = self.authorization_header(context).await else {
|
||||
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
|
||||
litellm_tracing::warn!(
|
||||
"Could not authenticate while ensuring CyberArk variable exists"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let body = format!(
|
||||
|
|
@ -288,19 +290,19 @@ impl CyberArkSecretManager {
|
|||
reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
|
||||
) =>
|
||||
{
|
||||
tracing::debug!(
|
||||
litellm_tracing::debug!(
|
||||
"CyberArk variable policy already exists or conflicts: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Ok(response) => {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"Could not ensure CyberArk variable exists: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -324,7 +326,7 @@ impl CyberArkSecretManager {
|
|||
_recovery_window_in_days: Option<u32>,
|
||||
_context: &SecretOperationContext,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
|
||||
);
|
||||
self.secrets.invalidate(name).await;
|
||||
|
|
|
|||
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
26
litellm-rust/crates/tracing/README.md
Normal file
26
litellm-rust/crates/tracing/README.md
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Native diagnostic tracing
|
||||
|
||||
`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber
|
||||
|
||||
Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work
|
||||
|
||||
Bindings implement `litellm_tracing::Sink` to connect events to their host runtime:
|
||||
|
||||
```rust
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
```
|
||||
|
||||
Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value
|
||||
|
||||
Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion
|
||||
|
||||
The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger
|
||||
|
||||
Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events
|
||||
|
||||
A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue
|
||||
|
||||
This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate
|
||||
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
use std::{
|
||||
cell::Cell,
|
||||
fmt,
|
||||
future::{Future, poll_fn},
|
||||
pin::pin,
|
||||
};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use tracing::{
|
||||
Dispatch, Event, Subscriber,
|
||||
field::{Field, Visit},
|
||||
subscriber::Interest,
|
||||
};
|
||||
use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
|
||||
|
||||
mod processing;
|
||||
mod redaction;
|
||||
|
||||
pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor};
|
||||
pub use redaction::{REDACTED, SecretRedactor};
|
||||
pub use tracing::{Level, Metadata, debug, error, info, trace, warn};
|
||||
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Record {
|
||||
pub metadata: &'static Metadata<'static>,
|
||||
pub message: String,
|
||||
pub fields: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Logger {
|
||||
dispatch: Dispatch,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
pub fn new(sink: impl Sink) -> Self {
|
||||
Self {
|
||||
dispatch: Dispatch::new(Registry::default().with(Output(sink))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
if EMITTING.get() {
|
||||
return operation();
|
||||
}
|
||||
tracing::dispatcher::with_default(&self.dispatch, operation)
|
||||
}
|
||||
|
||||
pub fn instrument<F: Future>(&self, future: F) -> impl Future<Output = F::Output> + use<F> {
|
||||
let logger = self.clone();
|
||||
async move {
|
||||
let mut future = pin!(future);
|
||||
poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static EMITTING: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
struct Emitting;
|
||||
|
||||
impl Emitting {
|
||||
fn enter() -> Option<Self> {
|
||||
EMITTING.with(|active| (!active.replace(true)).then_some(Self))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Emitting {
|
||||
fn drop(&mut self) {
|
||||
EMITTING.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
struct Output<S>(S);
|
||||
|
||||
impl<S: Sink, R: Subscriber> Layer<R> for Output<S> {
|
||||
fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {
|
||||
Interest::sometimes()
|
||||
}
|
||||
|
||||
fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return false;
|
||||
};
|
||||
self.0.enabled(metadata)
|
||||
}
|
||||
|
||||
fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return;
|
||||
};
|
||||
let mut record = Record {
|
||||
metadata: event.metadata(),
|
||||
message: String::new(),
|
||||
fields: Map::new(),
|
||||
};
|
||||
event.record(&mut record);
|
||||
self.0.emit(&record);
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
fn field(&mut self, field: &Field, value: Value) {
|
||||
if field.name() == "message" {
|
||||
self.message = match value {
|
||||
Value::String(message) => message,
|
||||
value => value.to_string(),
|
||||
};
|
||||
} else {
|
||||
self.fields.insert(field.name().to_owned(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit for Record {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
self.field(field, format!("{value:?}").into());
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_bool(&mut self, field: &Field, value: bool) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_i64(&mut self, field: &Field, value: i64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_u64(&mut self, field: &Field, value: u64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_f64(&mut self, field: &Field, value: f64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
}
|
||||
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
use fancy_regex::Result;
|
||||
use percent_encoding::percent_decode_str;
|
||||
|
||||
use crate::{REDACTED, SecretRedactor};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Policy {
|
||||
pub redact: bool,
|
||||
pub base64_limit: i64,
|
||||
pub text_limit: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DiagnosticInput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<(Option<String>, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct DiagnosticOutput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<String>,
|
||||
pub changed: bool,
|
||||
}
|
||||
|
||||
pub struct Processor {
|
||||
redactor: SecretRedactor,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
redactor: SecretRedactor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact_text(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact(text)
|
||||
}
|
||||
|
||||
pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_structured(key, text)
|
||||
}
|
||||
|
||||
pub fn redact_client_message(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_internal(text)
|
||||
}
|
||||
|
||||
pub fn process_diagnostic(
|
||||
&self,
|
||||
input: &DiagnosticInput,
|
||||
policy: Policy,
|
||||
) -> Result<DiagnosticOutput> {
|
||||
let message = self.process_text(&input.message, policy)?;
|
||||
let exception = input
|
||||
.exception
|
||||
.as_deref()
|
||||
.map(|text| self.process_text(text, policy))
|
||||
.transpose()?;
|
||||
let stack = input
|
||||
.stack
|
||||
.as_deref()
|
||||
.map(|text| {
|
||||
if policy.redact {
|
||||
self.redact_text(text)
|
||||
} else {
|
||||
Ok(text.to_owned())
|
||||
}
|
||||
})
|
||||
.transpose()?;
|
||||
let leaves = input
|
||||
.leaves
|
||||
.iter()
|
||||
.map(|(key, text)| {
|
||||
if policy.redact {
|
||||
self.redact_structured_text(key.as_deref(), text)
|
||||
} else {
|
||||
Ok(text.clone())
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let changed = message != input.message
|
||||
|| exception != input.exception
|
||||
|| stack != input.stack
|
||||
|| leaves
|
||||
.iter()
|
||||
.zip(&input.leaves)
|
||||
.any(|(processed, (_, original))| processed != original);
|
||||
Ok(DiagnosticOutput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
changed,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result<Vec<String>> {
|
||||
arguments
|
||||
.iter()
|
||||
.map(|argument| self.scrub_access_arg(argument))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn process_text(&self, text: &str, policy: Policy) -> Result<String> {
|
||||
let collapsed = if policy.base64_limit > 0 {
|
||||
collapse_base64(text, policy.base64_limit as usize)
|
||||
} else {
|
||||
text.to_owned()
|
||||
};
|
||||
let redacted = if policy.redact {
|
||||
self.redact_text(&collapsed)?
|
||||
} else {
|
||||
collapsed
|
||||
};
|
||||
Ok(
|
||||
if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize {
|
||||
truncate_text(&redacted, policy.text_limit as usize)
|
||||
} else {
|
||||
redacted
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn scrub_access_arg(&self, value: &str) -> Result<String> {
|
||||
let length = value.chars().count();
|
||||
let scanned = if length <= 512 {
|
||||
value
|
||||
} else {
|
||||
let head = &value[..char_offset(value, 512)];
|
||||
if head.contains('?') {
|
||||
&head[..head.rfind(['?', '&']).unwrap_or(0)]
|
||||
} else {
|
||||
head
|
||||
}
|
||||
};
|
||||
let scrubbed = self.redact_text(scanned)?;
|
||||
let (path, query) = scrubbed
|
||||
.split_once('?')
|
||||
.map_or((scrubbed.as_str(), None), |(path, query)| {
|
||||
(path, Some(query))
|
||||
});
|
||||
let safe = if self.hides_encoded_credential(path)? {
|
||||
REDACTED.to_owned()
|
||||
} else if query.is_some() && self.hides_encoded_credential(&scrubbed)? {
|
||||
format!("{path}?{REDACTED}")
|
||||
} else {
|
||||
scrubbed
|
||||
};
|
||||
Ok(if length > 512 {
|
||||
format!(
|
||||
"{safe}... ({} more chars truncated) ...",
|
||||
length - scanned.chars().count()
|
||||
)
|
||||
} else {
|
||||
safe
|
||||
})
|
||||
}
|
||||
|
||||
fn hides_encoded_credential(&self, value: &str) -> Result<bool> {
|
||||
if !value.as_bytes().contains(&b'%') {
|
||||
return Ok(false);
|
||||
}
|
||||
let decoded = percent_decode_str(value).decode_utf8_lossy();
|
||||
Ok(self.redact_text(&decoded)? != decoded)
|
||||
}
|
||||
}
|
||||
|
||||
fn char_offset(text: &str, count: usize) -> usize {
|
||||
text.char_indices()
|
||||
.nth(count)
|
||||
.map_or(text.len(), |(index, _)| index)
|
||||
}
|
||||
|
||||
fn marker(skipped_chars: usize) -> String {
|
||||
format!(
|
||||
"... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..."
|
||||
)
|
||||
}
|
||||
|
||||
fn truncate_text(text: &str, limit: usize) -> String {
|
||||
let length = text.chars().count();
|
||||
let kept = limit.saturating_sub(marker(length).len());
|
||||
if kept == 0 {
|
||||
return text[..char_offset(text, limit)].to_owned();
|
||||
}
|
||||
let head = kept / 2;
|
||||
let tail = kept - head;
|
||||
format!(
|
||||
"{}{}{}",
|
||||
&text[..char_offset(text, head)],
|
||||
marker(length - kept),
|
||||
&text[char_offset(text, length - tail)..]
|
||||
)
|
||||
}
|
||||
|
||||
fn base64_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/'
|
||||
}
|
||||
|
||||
fn looks_like_base64(run: &str) -> bool {
|
||||
let unpadded = run.trim_end_matches('=');
|
||||
let lower_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
|
||||
let upper_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
|
||||
let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]);
|
||||
(!lower_hex && !upper_hex) || repeated
|
||||
}
|
||||
|
||||
fn base64_size(chars: usize) -> String {
|
||||
let bytes = chars as f64 * 3.0 / 4.0;
|
||||
if bytes >= 1024.0 * 1024.0 {
|
||||
return format!("{:.2}MB", bytes / (1024.0 * 1024.0));
|
||||
}
|
||||
if bytes >= 1024.0 {
|
||||
return format!("{:.1}KB", bytes / 1024.0);
|
||||
}
|
||||
format!("{}B", bytes as usize)
|
||||
}
|
||||
|
||||
fn collapse_base64(text: &str, limit: usize) -> String {
|
||||
let bytes = text.as_bytes();
|
||||
let mut position = 0;
|
||||
let mut previous = 0;
|
||||
let mut output = String::new();
|
||||
while position < bytes.len() {
|
||||
if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) {
|
||||
position += 1;
|
||||
continue;
|
||||
}
|
||||
let start = position;
|
||||
while position < bytes.len() && base64_byte(bytes[position]) {
|
||||
position += 1;
|
||||
}
|
||||
let run_end = position;
|
||||
while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' {
|
||||
position += 1;
|
||||
}
|
||||
let run = &text[start..position];
|
||||
if run_end - start > limit && looks_like_base64(run) {
|
||||
output.push_str(&text[previous..start]);
|
||||
output.push_str(&format!(
|
||||
"[base64_data truncated: {}]",
|
||||
base64_size(run.len())
|
||||
));
|
||||
previous = position;
|
||||
}
|
||||
}
|
||||
output.push_str(&text[previous..]);
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() {
|
||||
let processor = Processor::new(16);
|
||||
let secret = format!("sk-{}", "q".repeat(48));
|
||||
let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000));
|
||||
let input = DiagnosticInput {
|
||||
message: text,
|
||||
exception: None,
|
||||
stack: None,
|
||||
leaves: vec![],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 0,
|
||||
text_limit: 500,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.chars().count() <= 500);
|
||||
assert!(!output.message.contains("sk-qq"));
|
||||
assert!(output.changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() {
|
||||
let processor = Processor::new(16);
|
||||
let input = DiagnosticInput {
|
||||
message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)),
|
||||
exception: Some(format!("upload failed: {}", "Q".repeat(100))),
|
||||
stack: Some("api_key=secret123".to_owned()),
|
||||
leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 20,
|
||||
text_limit: 0,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.contains("[base64_data truncated: 75B]"));
|
||||
assert!(output.message.contains(&"a1".repeat(50)));
|
||||
assert!(
|
||||
output
|
||||
.exception
|
||||
.unwrap()
|
||||
.contains("[base64_data truncated: 75B]")
|
||||
);
|
||||
assert_eq!(output.stack.as_deref(), Some(REDACTED));
|
||||
assert_eq!(output.leaves, vec![REDACTED]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() {
|
||||
let processor = Processor::new(16);
|
||||
let arguments = vec![
|
||||
"/v1/models?filter=gpt%2D4o&page=2".to_owned(),
|
||||
"/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(),
|
||||
];
|
||||
assert_eq!(
|
||||
processor.scrub_access_arguments(&arguments).unwrap(),
|
||||
vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()]
|
||||
);
|
||||
}
|
||||
}
|
||||
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
use fancy_regex::{NoExpand, Regex};
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
#[cfg(test)]
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
internal_pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
let internal_pattern = Regex::new(concat!(
|
||||
r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#,
|
||||
r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#,
|
||||
r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|",
|
||||
r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|",
|
||||
r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b",
|
||||
))
|
||||
.expect("internal detail patterns compile");
|
||||
Self {
|
||||
pattern,
|
||||
internal_pattern,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.try_redact(value)
|
||||
.unwrap_or_else(|_| REDACTED.to_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
self.pattern
|
||||
.try_replacen(value, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact_structured(
|
||||
&self,
|
||||
key: Option<&str>,
|
||||
value: &str,
|
||||
) -> fancy_regex::Result<String> {
|
||||
let scrubbed = self.try_redact(value)?;
|
||||
if scrubbed != value || key.is_none() {
|
||||
return Ok(scrubbed);
|
||||
}
|
||||
let rendered = format!("'{}': '{value}'", key.unwrap_or_default());
|
||||
Ok(if self.try_redact(&rendered)? != rendered {
|
||||
REDACTED.to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
let without_traceback = value
|
||||
.split_once("Traceback (most recent call last):")
|
||||
.map_or(value, |(prefix, _)| prefix.trim_end());
|
||||
self.internal_pattern
|
||||
.try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc,
|
||||
};
|
||||
|
||||
use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Output {
|
||||
enabled: Arc<AtomicBool>,
|
||||
sender: mpsc::Sender<(String, Value, Level, &'static str, Option<u32>)>,
|
||||
}
|
||||
|
||||
impl Sink for Output {
|
||||
fn enabled(&self, _: &Metadata<'_>) -> bool {
|
||||
self.enabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
self.sender
|
||||
.send((
|
||||
record.message.clone(),
|
||||
Value::Object(record.fields.clone()),
|
||||
*record.metadata.level(),
|
||||
record.metadata.target(),
|
||||
record.metadata.line(),
|
||||
))
|
||||
.unwrap();
|
||||
Logger::default().scope(|| warn!("a sink must not recursively emit"));
|
||||
}
|
||||
}
|
||||
|
||||
fn emit() {
|
||||
warn!(
|
||||
attempt = 3_u64,
|
||||
elapsed = 1.5,
|
||||
retry = true,
|
||||
reason = "timeout",
|
||||
"retry {}",
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let enabled = Arc::new(AtomicBool::new(false));
|
||||
let logger = Logger::new(Output {
|
||||
enabled: enabled.clone(),
|
||||
sender,
|
||||
});
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
enabled.store(true, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
let (message, fields, level, target, line) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "retry 3");
|
||||
assert_eq!(
|
||||
fields,
|
||||
json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"})
|
||||
);
|
||||
assert_eq!(level, Level::WARN);
|
||||
assert_eq!(target, module_path!());
|
||||
assert!(line.is_some());
|
||||
enabled.store(false, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() {
|
||||
let tasks = (0..2)
|
||||
.map(|id| {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let logger = Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
});
|
||||
let task = tokio::spawn(logger.instrument(async move {
|
||||
tokio::task::yield_now().await;
|
||||
info!(id, "worker");
|
||||
}));
|
||||
(id, task, receiver)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (id, task, receiver) in tasks {
|
||||
task.await.unwrap();
|
||||
let (message, fields, level, _, _) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "worker");
|
||||
assert_eq!(fields, json!({"id": id}));
|
||||
assert_eq!(level, Level::INFO);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_scopes_restore_the_previous_sink() {
|
||||
let (outer_sender, outer) = mpsc::channel();
|
||||
let (inner_sender, inner) = mpsc::channel();
|
||||
let logger = |sender| {
|
||||
Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
})
|
||||
};
|
||||
let outside = logger(outer_sender);
|
||||
let inside = logger(inner_sender);
|
||||
outside.scope(|| {
|
||||
info!("before");
|
||||
inside.scope(|| info!("inside"));
|
||||
info!("after");
|
||||
});
|
||||
assert_eq!(
|
||||
outer.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["before", "after"]
|
||||
);
|
||||
assert_eq!(
|
||||
inner.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["inside"]
|
||||
);
|
||||
}
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
import ast
|
||||
import contextvars
|
||||
import functools
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from typing import Any, Final, TextIO
|
||||
|
|
@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
_python_redact_string,
|
||||
_python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_string,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
set_verbose = False
|
||||
|
||||
|
|
@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str:
|
|||
pass through credential redaction.
|
||||
"""
|
||||
stripped: Final = "".join(ch for ch in value if ch.isprintable())
|
||||
return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
|
||||
return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
|
||||
|
||||
|
||||
def set_session_id(session_id: str) -> "contextvars.Token[str]":
|
||||
|
|
@ -74,12 +77,6 @@ def _redact_string(value: str) -> str:
|
|||
return redact_string(value)
|
||||
|
||||
|
||||
def _redact_structured_value(key: str | None, value: str) -> str:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return value
|
||||
return redact_structured_value(key, value)
|
||||
|
||||
|
||||
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
|
||||
_REDACTED_STAMP: Final = object()
|
||||
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
|
||||
|
|
@ -103,14 +100,6 @@ def _plain_text(value: object) -> str:
|
|||
return UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _redact_extra_value(key: str, value: object) -> object:
|
||||
try:
|
||||
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
|
||||
except Exception:
|
||||
return _redact_string(_plain_text(value))
|
||||
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
|
||||
|
||||
|
||||
def redact_secrets(value: str) -> str:
|
||||
"""Public API: redact known secret/credential patterns from an arbitrary string.
|
||||
|
||||
|
|
@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
|||
return None
|
||||
try:
|
||||
return color_message % record.args
|
||||
except TypeError:
|
||||
except Exception:
|
||||
return color_message
|
||||
|
||||
|
||||
|
|
@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter):
|
|||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
|
||||
return True
|
||||
|
||||
# Runs before args are cleared, and before the extra-field loop below
|
||||
# that redacts the substituted result.
|
||||
substituted_color_message: Final = _substituted_color_message(record)
|
||||
if substituted_color_message is not None:
|
||||
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
try:
|
||||
record.msg = _redact_string(record.getMessage())
|
||||
record.args = None
|
||||
except Exception:
|
||||
if isinstance(record.msg, str):
|
||||
record.msg = _redact_string(record.msg)
|
||||
|
||||
# Redact exception tracebacks
|
||||
if record.exc_info and record.exc_info[1] is not None:
|
||||
try:
|
||||
record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(record.stack_info, str):
|
||||
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
# Redact extra fields passed via logger.debug("msg", extra={...})
|
||||
record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items())
|
||||
for key, value in record_items:
|
||||
if key in _STANDARD_RECORD_ATTRS:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
setattr(record, key, _redact_structured_value(key, value))
|
||||
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
|
||||
setattr(record, key, _redact_extra_value(key, value))
|
||||
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
return _process_record(record, base64_limit=0, text_limit=0, redact=True)
|
||||
|
||||
|
||||
_secret_filter: Final = SecretRedactionFilter()
|
||||
|
|
@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
|
|||
def _hides_a_credential(value: str) -> bool:
|
||||
"""Whether *value* only looks clean until it is percent-decoded."""
|
||||
decoded: Final = unquote(value)
|
||||
return _redact_string(decoded) != decoded
|
||||
return _python_redact_string(decoded) != decoded
|
||||
|
||||
|
||||
def _drop_encoded_credential(scrubbed: str) -> str:
|
||||
|
|
@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str:
|
|||
pattern and would then be logged raw.
|
||||
"""
|
||||
if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
|
||||
return _drop_encoded_credential(_redact_string(value))
|
||||
return _drop_encoded_credential(_python_redact_string(value))
|
||||
head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
|
||||
kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
|
||||
scrubbed: Final = _drop_encoded_credential(_redact_string(kept))
|
||||
scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
|
||||
return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
|
||||
|
||||
|
||||
|
|
@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter):
|
|||
if not _ENABLE_SECRET_REDACTION:
|
||||
return True
|
||||
if isinstance(record.args, tuple) and record.args:
|
||||
strings: Final = tuple(arg for arg in record.args if isinstance(arg, str))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.scrub_access_arguments(strings),
|
||||
lambda: tuple(_scrub_access_arg(arg) for arg in strings),
|
||||
)
|
||||
scrubbed: Final = (
|
||||
candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings)
|
||||
)
|
||||
values: Final = iter(scrubbed)
|
||||
record.args = tuple( # rebind-ok: a Filter scrubs records in place
|
||||
_scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args
|
||||
next(values) if isinstance(arg, str) else arg for arg in record.args
|
||||
)
|
||||
return True
|
||||
# No positional args means everything is in msg, where collapsing is correct.
|
||||
|
|
@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str:
|
|||
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
|
||||
|
||||
|
||||
def _extra_structure(key: str, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return safe_json_structure(value, key=key)
|
||||
except Exception:
|
||||
return _plain_text(value)
|
||||
|
||||
|
||||
def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]:
|
||||
if isinstance(value, str):
|
||||
yield key, value
|
||||
elif isinstance(value, dict):
|
||||
yield from itertools.chain.from_iterable(
|
||||
_string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str)
|
||||
)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value)
|
||||
|
||||
|
||||
def _replace_string_leaves(value: object, values: Iterator[str]) -> object:
|
||||
if isinstance(value, str):
|
||||
return next(values)
|
||||
if isinstance(value, dict):
|
||||
return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers
|
||||
key: _replace_string_leaves(child, values) for key, child in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers
|
||||
_replace_string_leaves(child, values) for child in value
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_replace_string_leaves(child, values) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def _sort_processed_sets(original: object, processed: object) -> object:
|
||||
if isinstance(original, set) and isinstance(processed, list):
|
||||
return sorted(processed)
|
||||
if isinstance(original, dict) and isinstance(processed, dict):
|
||||
return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict
|
||||
key: _sort_processed_sets(original.get(key), value) for key, value in processed.items()
|
||||
}
|
||||
if isinstance(original, list) and isinstance(processed, list):
|
||||
return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list
|
||||
_sort_processed_sets(before, after) for before, after in zip(original, processed)
|
||||
]
|
||||
if isinstance(original, tuple) and isinstance(processed, tuple):
|
||||
return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed))
|
||||
return processed
|
||||
|
||||
|
||||
def _python_process_diagnostic(
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
redact: bool,
|
||||
base64_limit: int,
|
||||
text_limit: int,
|
||||
) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
|
||||
def process_text(text: str) -> str:
|
||||
collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
|
||||
scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
|
||||
return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
|
||||
|
||||
processed_message: Final = process_text(message)
|
||||
processed_exception: Final = process_text(exception) if exception is not None else None
|
||||
processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
|
||||
processed_leaves: Final = tuple(
|
||||
_python_redact_structured_value(key, text) if redact else text for key, text in leaves
|
||||
)
|
||||
changed: Final = (
|
||||
processed_message != message
|
||||
or processed_exception != exception
|
||||
or processed_stack != stack
|
||||
or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves))
|
||||
)
|
||||
return processed_message, processed_exception, processed_stack, processed_leaves, changed
|
||||
|
||||
|
||||
def _render_message(record: logging.LogRecord) -> str:
|
||||
try:
|
||||
return record.getMessage()
|
||||
except Exception:
|
||||
return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _render_exception(record: logging.LogRecord) -> str | None:
|
||||
if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None:
|
||||
return None
|
||||
try:
|
||||
return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info)
|
||||
except Exception:
|
||||
return "REDACTED"
|
||||
|
||||
|
||||
def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool:
|
||||
if _is_redacted(record):
|
||||
return True
|
||||
message: Final = _render_message(record)
|
||||
exception: Final = _render_exception(record)
|
||||
stack: Final = record.stack_info if isinstance(record.stack_info, str) else None
|
||||
substituted_color: Final = _substituted_color_message(record)
|
||||
extras: Final = (
|
||||
tuple(
|
||||
(
|
||||
key,
|
||||
value,
|
||||
_extra_structure(
|
||||
key, substituted_color if key == "color_message" and substituted_color is not None else value
|
||||
),
|
||||
)
|
||||
for key, value in record.__dict__.items()
|
||||
if key not in _STANDARD_RECORD_ATTRS
|
||||
and key != _REDACTED_RECORD_ATTR
|
||||
and not isinstance(value, _UNREDACTED_SCALAR_TYPES)
|
||||
)
|
||||
if redact
|
||||
else ()
|
||||
)
|
||||
extra_leaves: Final = tuple(
|
||||
itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras)
|
||||
)
|
||||
raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None
|
||||
color_template: Final = record.__dict__.get("color_message")
|
||||
raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None
|
||||
leaves: Final = (
|
||||
extra_leaves
|
||||
+ (((None, raw_template),) if raw_template is not None else ())
|
||||
+ (((None, raw_color),) if raw_color is not None else ())
|
||||
)
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)),
|
||||
lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit),
|
||||
)
|
||||
processed_message, processed_exception, processed_stack, processed_leaves, _ = (
|
||||
candidate
|
||||
if len(candidate[3]) == len(leaves)
|
||||
else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit)
|
||||
)
|
||||
raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template
|
||||
safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message
|
||||
if redact or safe_message != message:
|
||||
record.msg = safe_message # rebind-ok: the Filter interface mutates the record
|
||||
record.args = None # rebind-ok: the rendered message replaces interpolation inputs
|
||||
if processed_exception is not None:
|
||||
record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record
|
||||
if processed_stack is not None:
|
||||
record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record
|
||||
processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
|
||||
for key, original, prepared in extras:
|
||||
replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
|
||||
if not _scrubbing_changed_nothing(replacement, original):
|
||||
setattr(record, key, replacement)
|
||||
raw_color_changed: Final = (
|
||||
raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color
|
||||
)
|
||||
if raw_color_changed and getattr(record, "color_message", None) == substituted_color:
|
||||
setattr(record, "color_message", "REDACTED")
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
|
||||
|
||||
def _redact_json_record(value: object) -> object:
|
||||
prepared: Final = safe_json_structure(value)
|
||||
leaves: Final = tuple(_string_leaves(None, prepared))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
|
||||
lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
|
||||
)
|
||||
replacements: Final = (
|
||||
candidate
|
||||
if len(candidate) == len(leaves)
|
||||
else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
|
||||
)
|
||||
return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
|
||||
|
||||
|
||||
class StdoutLogTruncationFilter(logging.Filter):
|
||||
"""Bounds how much of an oversized log line reaches stdout.
|
||||
|
||||
|
|
@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter):
|
|||
return True
|
||||
|
||||
|
||||
_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
|
||||
class DiagnosticProcessingFilter(StdoutLogTruncationFilter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return _process_record(
|
||||
record,
|
||||
base64_limit=_get_max_base64_length_stdout_log(),
|
||||
text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0,
|
||||
redact=_ENABLE_SECRET_REDACTION,
|
||||
)
|
||||
|
||||
|
||||
_diagnostic_filter: Final = DiagnosticProcessingFilter()
|
||||
|
||||
|
||||
class CorrelationContextFilter(logging.Filter):
|
||||
|
|
@ -633,7 +785,9 @@ class JsonFormatter(Formatter):
|
|||
if record.exc_info:
|
||||
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
|
||||
|
||||
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
|
||||
return safe_dumps(
|
||||
json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record)
|
||||
)
|
||||
|
||||
|
||||
class CorrelationPlainFormatter(logging.Formatter):
|
||||
|
|
@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter):
|
|||
# Create a handler with JSON formatting for exceptions
|
||||
error_handler: Final = logging.StreamHandler()
|
||||
error_handler.setFormatter(formatter)
|
||||
error_handler.addFilter(_stdout_truncation_filter)
|
||||
error_handler.addFilter(_diagnostic_filter)
|
||||
error_handler.addFilter(_secret_filter)
|
||||
error_handler.addFilter(_correlation_filter)
|
||||
|
||||
|
|
@ -734,10 +888,10 @@ verbose_logger.addHandler(handler)
|
|||
|
||||
# Filters attached to the logger, not the handler, survive callers swapping in their own
|
||||
# handlers (JSON mode, uvicorn log config, a host app's root handler).
|
||||
verbose_router_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_router_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
|
||||
verbose_logger.addFilter(_diagnostic_filter)
|
||||
|
||||
|
||||
def _suppress_loggers():
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import re
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
REDACTED: Final = "REDACTED"
|
||||
|
||||
|
|
@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]":
|
|||
_SECRET_RE: Final = _build_secret_patterns()
|
||||
|
||||
|
||||
def _python_redact_string(value: str) -> str:
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
|
||||
|
||||
def redact_string(value: str) -> str:
|
||||
"""Scrub known secret/credential patterns from *value* and return the result."""
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value))
|
||||
|
||||
|
||||
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
|
||||
|
|
@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile(
|
|||
_TRACEBACK_MARKER: Final = "Traceback (most recent call last):"
|
||||
|
||||
|
||||
def redact_internal_details(value: str) -> str:
|
||||
def _python_redact_internal_details(value: str) -> str:
|
||||
"""Drop an embedded traceback and scrub filesystem paths and internal hostnames,
|
||||
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
|
||||
marker_index: Final = value.find(_TRACEBACK_MARKER)
|
||||
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback))
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
def redact_internal_details(value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value)
|
||||
)
|
||||
|
||||
|
||||
def _python_redact_structured_value(key: str | None, value: str) -> str:
|
||||
"""Scrub *value* as it appeared under *key* inside a structured record.
|
||||
|
||||
redact_string() replaces a whole ``key: value`` span with REDACTED, which is
|
||||
|
|
@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str:
|
|||
repr would, so the key-name patterns still fire, but collapses only the value
|
||||
so the caller's structure survives.
|
||||
"""
|
||||
scrubbed: Final = redact_string(value)
|
||||
scrubbed: Final = _python_redact_string(value)
|
||||
if scrubbed != value or key is None:
|
||||
return scrubbed
|
||||
rendered: Final = f"'{key}': '{value}'"
|
||||
return REDACTED if redact_string(rendered) != rendered else value
|
||||
return REDACTED if _python_redact_string(rendered) != rendered else value
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_structured_text(key, value),
|
||||
lambda: _python_redact_structured_value(key, value),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
return self._is_deepseek_r1_model(model=model, base_model=base_model)
|
||||
|
||||
@classmethod
|
||||
def _supports_sampling_params(cls, model: str) -> bool:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if base_model.startswith("anthropic"):
|
||||
return True
|
||||
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
|
||||
for candidate in candidates:
|
||||
if (
|
||||
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
|
||||
candidate, "supports_sampling_params"
|
||||
)
|
||||
) is not None:
|
||||
return flag
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
supports_sampling: Final = self._supports_sampling_params(model)
|
||||
supported_params: Final = [
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
"stream_options",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
*(("temperature", "top_p") if supports_sampling else ()),
|
||||
"extra_headers",
|
||||
"response_format",
|
||||
"requestMetadata",
|
||||
|
|
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
|
|||
value = [value]
|
||||
optional_params["stopSequences"] = value
|
||||
if param == "temperature" or param == "top_p":
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
if base_model.startswith("anthropic"):
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
elif not self._supports_sampling_params(model):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}={value}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
else:
|
||||
optional_params["topP" if param == "top_p" else param] = value
|
||||
if param == "tools" and isinstance(value, list):
|
||||
self._apply_tool_call_transformation(
|
||||
tools=cast(list[OpenAIChatCompletionToolParam], value),
|
||||
|
|
|
|||
|
|
@ -55087,7 +55087,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 4e-06,
|
||||
|
|
@ -55116,7 +55117,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
|
|
@ -55145,7 +55147,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -55174,7 +55177,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
|
|
@ -55203,7 +55207,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -55232,7 +55237,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-6-astra": {
|
||||
"input_cost_per_token": 1.1e-05,
|
||||
|
|
|
|||
|
|
@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ...
|
|||
class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
|
||||
class ProcessReservedForForking(RuntimeError): ...
|
||||
|
||||
@final
|
||||
class NativeDiagnosticProcessor:
|
||||
def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: Sequence[tuple[str | None, str]],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, list[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ...
|
||||
|
||||
def ocr(
|
||||
request: LiteLLMOcrRequest,
|
||||
args: tuple[object, ...],
|
||||
|
|
@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ...
|
|||
__all__ = [
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"NativeDiagnosticProcessor",
|
||||
"ProcessReservedForForking",
|
||||
"ResponsesWebSocketConnection",
|
||||
"RustBridgeDeclined",
|
||||
|
|
|
|||
|
|
@ -86,11 +86,25 @@ class SecretManagerRule:
|
|||
return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerContext:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerRule:
|
||||
rollout: Rollout
|
||||
|
||||
def matches(self, context: Context) -> bool:
|
||||
return isinstance(context, LoggerContext)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule
|
||||
Rules: TypeAlias = tuple[Rule, ...]
|
||||
|
||||
RULES: Final[Rules] = (
|
||||
LoggerRule(Rollout.RUST_OPT_IN),
|
||||
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
|
||||
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
|
||||
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
|
||||
|
|
|
|||
59
litellm/rust_bridge/diagnostics.py
Normal file
59
litellm/rust_bridge/diagnostics.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from functools import lru_cache
|
||||
from typing import Final, Protocol, TypeVar, cast
|
||||
|
||||
from litellm.rust_bridge.bindings import NativeBinding
|
||||
|
||||
ResultT: Final = TypeVar("ResultT")
|
||||
|
||||
|
||||
class NativeDiagnosticProcessor(Protocol):
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, Sequence[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ...
|
||||
|
||||
|
||||
class NativeDiagnosticFactory(Protocol):
|
||||
def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
|
||||
|
||||
def _as_factory(value: object) -> NativeDiagnosticFactory | None:
|
||||
if not isinstance(value, type):
|
||||
return None
|
||||
return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type
|
||||
|
||||
|
||||
PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory)
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor:
|
||||
return factory(minimum_custom_key_length)
|
||||
|
||||
|
||||
def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT:
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge.catalog import LoggerContext, decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
||||
selected: Final = decision(LoggerContext())
|
||||
if selected is Decision.PYTHON:
|
||||
return python()
|
||||
factory: Final = PROCESSOR.load()
|
||||
if factory is None:
|
||||
return python()
|
||||
try:
|
||||
return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH))
|
||||
except Exception:
|
||||
return python()
|
||||
63
litellm/rust_bridge/logger.py
Normal file
63
litellm/rust_bridge/logger.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm._logging import (
|
||||
CorrelationContextFilter,
|
||||
DiagnosticProcessingFilter,
|
||||
session_id_var,
|
||||
set_session_id,
|
||||
set_trace_id,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
)
|
||||
|
||||
_REDACTION: Final = DiagnosticProcessingFilter()
|
||||
_CORRELATION: Final = CorrelationContextFilter()
|
||||
|
||||
|
||||
def context() -> tuple[str, str]:
|
||||
return session_id_var.get(), trace_id_var.get()
|
||||
|
||||
|
||||
def enabled(level: int) -> bool:
|
||||
return verbose_logger.isEnabledFor(level)
|
||||
|
||||
|
||||
def emit(
|
||||
level: int,
|
||||
message: str,
|
||||
pathname: str,
|
||||
lineno: int,
|
||||
target: str,
|
||||
fields: Mapping[str, JsonValue],
|
||||
correlation: tuple[str, str],
|
||||
) -> None:
|
||||
if not enabled(level):
|
||||
return
|
||||
session_token: Final = set_session_id(correlation[0])
|
||||
trace_token: Final = set_trace_id(correlation[1])
|
||||
try:
|
||||
record: Final = verbose_logger.makeRecord(
|
||||
verbose_logger.name,
|
||||
level,
|
||||
pathname,
|
||||
lineno,
|
||||
message,
|
||||
(),
|
||||
None,
|
||||
func=target,
|
||||
extra={
|
||||
"rust_target": target,
|
||||
"rust_fields": dict(fields),
|
||||
}, # mutable-ok: LogRecord requires JSON dict extras
|
||||
)
|
||||
_REDACTION.filter(record)
|
||||
_CORRELATION.filter(record)
|
||||
verbose_logger.handle(record)
|
||||
finally:
|
||||
trace_id_var.reset(trace_token)
|
||||
session_id_var.reset(session_token)
|
||||
|
|
@ -58,12 +58,6 @@ class SecretManagerBinding:
|
|||
settings_object: object
|
||||
|
||||
|
||||
def warn(message: str) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.warning("%s", message)
|
||||
|
||||
|
||||
def secret_manager() -> SecretManager:
|
||||
from litellm.secret_managers.main import (
|
||||
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private
|
||||
|
|
|
|||
|
|
@ -55087,7 +55087,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-sol": {
|
||||
"input_cost_per_token": 4e-06,
|
||||
|
|
@ -55116,7 +55117,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2.2e-06,
|
||||
|
|
@ -55145,7 +55147,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-terra": {
|
||||
"input_cost_per_token": 2e-06,
|
||||
|
|
@ -55174,7 +55177,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"us.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2.2e-07,
|
||||
|
|
@ -55203,7 +55207,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"global.openai.gpt-5.6-luna": {
|
||||
"input_cost_per_token": 2e-07,
|
||||
|
|
@ -55232,7 +55237,8 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_vision": true
|
||||
"supports_vision": true,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"bedrock_mantle/openai.gpt-6-astra": {
|
||||
"input_cost_per_token": 1.1e-05,
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [
|
|||
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
|
||||
"_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible).
|
||||
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
|
||||
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import litellm.cost_calculator
|
|||
|
||||
import asyncio
|
||||
import time
|
||||
from typing import Optional
|
||||
from typing import Final, Optional
|
||||
from unittest.mock import MagicMock, patch
|
||||
import pytest
|
||||
|
||||
|
|
@ -21,6 +21,9 @@ import json
|
|||
import httpx
|
||||
from litellm.types.utils import PromptTokensDetails
|
||||
from litellm.litellm_core_utils.litellm_logging import CustomLogger
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
|
||||
convert_to_model_response_object,
|
||||
)
|
||||
|
||||
|
||||
class CustomLoggingHandler(CustomLogger):
|
||||
|
|
@ -328,35 +331,39 @@ def test_whisper_azure():
|
|||
assert round(cost, 5) == round(expected_cost, 5)
|
||||
|
||||
|
||||
def test_dalle_3_azure_cost_tracking():
|
||||
litellm.set_verbose = True
|
||||
# model = "azure/dall-e-3-test"
|
||||
# response = litellm.image_generation(
|
||||
# model=model,
|
||||
# prompt="A cute baby sea otter",
|
||||
# api_version="2023-12-01-preview",
|
||||
# api_base=os.getenv("AZURE_SWEDEN_API_BASE"),
|
||||
# api_key=os.getenv("AZURE_SWEDEN_API_KEY"),
|
||||
# base_model="dall-e-3",
|
||||
# )
|
||||
# print(f"response: {response}")
|
||||
response = litellm.ImageResponse(
|
||||
created=1710265780,
|
||||
data=[
|
||||
{
|
||||
"b64_json": None,
|
||||
"revised_prompt": "A close-up image of an adorable baby sea otter. Its fur is thick and fluffy to provide buoyancy and insulation against the cold water. Its eyes are round, curious and full of life. It's lying on its back, floating effortlessly on the calm sea surface under the warm sun. Surrounding the otter are patches of colorful kelp drifting along the gentle waves, giving the scene a touch of vibrancy. The sea otter has its small paws folded on its chest, and it seems to be taking a break from its play.",
|
||||
"url": "test-azure-blob-url-with-sas-token",
|
||||
}
|
||||
],
|
||||
def test_gpt_image_2_azure_cost_tracking():
|
||||
azure_image_generation_response: Final = {
|
||||
"created": 1758585600,
|
||||
"data": [{"b64_json": "iVBORw0KGgo=", "revised_prompt": None, "url": None}],
|
||||
"output_format": "png",
|
||||
"quality": "low",
|
||||
"size": "1024x1024",
|
||||
"usage": {
|
||||
"input_tokens": 12,
|
||||
"input_tokens_details": {"image_tokens": 0, "text_tokens": 12},
|
||||
"output_tokens": 196,
|
||||
"output_tokens_details": {"image_tokens": 196, "text_tokens": 0},
|
||||
"total_tokens": 208,
|
||||
},
|
||||
}
|
||||
response: Final = convert_to_model_response_object(
|
||||
response_object=azure_image_generation_response,
|
||||
model_response_object=litellm.ImageResponse(),
|
||||
response_type="image_generation",
|
||||
hidden_params={"model": "gpt-image-2", "custom_llm_provider": "azure"},
|
||||
)
|
||||
response.usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
|
||||
response._hidden_params = {"model": "dall-e-3", "model_id": None}
|
||||
print(f"response hidden params: {response._hidden_params}")
|
||||
cost = litellm.completion_cost(
|
||||
completion_response=response, call_type="image_generation"
|
||||
|
||||
cost: Final = litellm.completion_cost(
|
||||
completion_response=response,
|
||||
model="azure/my-gpt-image-2-deployment",
|
||||
custom_llm_provider="azure",
|
||||
base_model="gpt-image-2",
|
||||
call_type="image_generation",
|
||||
)
|
||||
assert cost > 0
|
||||
|
||||
pricing: Final = litellm.model_cost["azure/gpt-image-2"]
|
||||
expected_cost: Final = pricing["input_cost_per_token"] * 12 + pricing["output_cost_per_image_token"] * 196
|
||||
assert round(cost, 8) == round(expected_cost, 8)
|
||||
|
||||
|
||||
def test_replicate_llama3_cost_tracking():
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import litellm
|
||||
from litellm import ModelResponse
|
||||
from litellm.llms.bedrock.chat.converse_transformation import AmazonConverseConfig
|
||||
|
|
@ -7500,3 +7499,92 @@ def test_eager_input_streaming_non_boolean_is_a_bad_request():
|
|||
"us.anthropic.claude-sonnet-4-5-20250929-v1:0",
|
||||
[_eager_openai_tool(eager_input_streaming="true")],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ("anthropic.claude-opus-4-7", "us.anthropic.claude-opus-4-7"))
|
||||
def test_converse_accepts_anthropic_default_temperature(model: str) -> None:
|
||||
result: Final = litellm.utils.get_optional_params(
|
||||
model=model,
|
||||
custom_llm_provider="bedrock",
|
||||
temperature=1,
|
||||
drop_params=False,
|
||||
)
|
||||
|
||||
assert result["temperature"] == 1
|
||||
|
||||
|
||||
def test_get_supported_openai_params_drops_sampling_params_for_gpt5_models():
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"global.openai.gpt-5.6-luna",
|
||||
"global.openai.gpt-5.6-sol",
|
||||
"us.openai.gpt-5.6-terra",
|
||||
"eu.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
"bedrock/openai.gpt-5.6-luna",
|
||||
]:
|
||||
supported = config.get_supported_openai_params(model=model)
|
||||
assert "temperature" not in supported
|
||||
assert "top_p" not in supported
|
||||
|
||||
supported_oss = config.get_supported_openai_params(model="openai.gpt-oss-120b-1:0")
|
||||
assert "temperature" in supported_oss
|
||||
assert "top_p" in supported_oss
|
||||
|
||||
|
||||
def test_map_openai_params_drops_temperature_and_top_p_when_drop_params_true():
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
"eu.openai.gpt-5.6-luna",
|
||||
]:
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"temperature": 1.0, "top_p": 0.9, "max_tokens": 50},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=True,
|
||||
)
|
||||
assert "temperature" not in result
|
||||
assert "topP" not in result
|
||||
assert result.get("maxTokens") == 50
|
||||
|
||||
|
||||
def test_map_openai_params_raises_unsupported_params_when_drop_params_false(monkeypatch):
|
||||
monkeypatch.setattr(litellm, "drop_params", False)
|
||||
config = AmazonConverseConfig()
|
||||
for model in [
|
||||
"bedrock/converse/global.openai.gpt-5.6-luna",
|
||||
"openai.gpt-5.6-luna",
|
||||
]:
|
||||
with pytest.raises(litellm.utils.UnsupportedParamsError) as exc_info:
|
||||
config.map_openai_params(
|
||||
non_default_params={"temperature": 1.0},
|
||||
optional_params={},
|
||||
model=model,
|
||||
drop_params=False,
|
||||
)
|
||||
assert "does not support temperature=1.0" in str(exc_info.value)
|
||||
|
||||
|
||||
def test_map_openai_params_retains_sampling_params_for_supported_models():
|
||||
config = AmazonConverseConfig()
|
||||
result = config.map_openai_params(
|
||||
non_default_params={"temperature": 0.7, "top_p": 0.8},
|
||||
optional_params={},
|
||||
model="openai.gpt-oss-120b-1:0",
|
||||
drop_params=False,
|
||||
)
|
||||
assert result.get("temperature") == 0.7
|
||||
assert result.get("topP") == 0.8
|
||||
|
||||
|
||||
def test_supports_sampling_params_prefixed_and_anthropic_fallback(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setitem(
|
||||
litellm.model_cost,
|
||||
"global.custom-test-reasoning-model",
|
||||
{"supports_sampling_params": False},
|
||||
)
|
||||
assert AmazonConverseConfig._supports_sampling_params("custom-test-reasoning-model") is False
|
||||
assert AmazonConverseConfig._supports_sampling_params("anthropic.claude-custom-unregistered") is True
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ from litellm.rust_bridge.catalog import (
|
|||
CacheRule,
|
||||
Context,
|
||||
Delivery,
|
||||
LoggerContext,
|
||||
Route,
|
||||
RouteContext,
|
||||
RouteRule,
|
||||
|
|
@ -89,6 +90,13 @@ def test_backend_rollouts_stay_on_python_when_global_rust_is_enabled(
|
|||
assert catalog.decision(context) is Decision.PYTHON
|
||||
|
||||
|
||||
def test_logger_rollout_obeys_the_global_switch() -> None:
|
||||
assert catalog.rollout(LoggerContext()) is Rollout.RUST_OPT_IN
|
||||
assert catalog.decision(LoggerContext()) is Decision.PYTHON
|
||||
configuration.rust(True)
|
||||
assert catalog.decision(LoggerContext()) is Decision.RUST_WITH_FALLBACK
|
||||
|
||||
|
||||
def test_response_cache_rules_select_the_whole_backend_runtime() -> None:
|
||||
rules: Final = (
|
||||
CacheRule(Rollout.RUST_REQUIRED, backends=frozenset({"local"})),
|
||||
|
|
|
|||
137
tests/test_litellm/rust_bridge/test_logger.py
Normal file
137
tests/test_litellm/rust_bridge/test_logger.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
import logging
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm._logging import (
|
||||
DiagnosticProcessingFilter,
|
||||
_python_process_diagnostic,
|
||||
redact_secrets,
|
||||
session_id_var,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
)
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
_python_redact_internal_details,
|
||||
_python_redact_string,
|
||||
_python_redact_structured_value,
|
||||
)
|
||||
from litellm.rust_bridge import diagnostics, logger
|
||||
|
||||
|
||||
def test_native_records_preserve_metadata_and_redact_before_custom_handlers(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(verbose_logger, "handlers", [])
|
||||
secret: Final = "sk-" + "a" * 48
|
||||
message: Final = f"Authorization: Bearer {secret}"
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
logger.emit(
|
||||
logging.WARNING, message, "native.rs", 42, "litellm_http", {"retry": True, "api_key": secret}, ("", "")
|
||||
)
|
||||
|
||||
record: Final = caplog.records[0]
|
||||
assert len(caplog.records) == 1
|
||||
assert record.getMessage() == redact_secrets(message)
|
||||
assert secret not in record.getMessage()
|
||||
assert (record.pathname, record.lineno, record.funcName) == ("native.rs", 42, "litellm_http")
|
||||
assert record.__dict__["rust_fields"]["retry"] is True
|
||||
assert secret not in str(record.__dict__["rust_fields"])
|
||||
|
||||
|
||||
def test_native_context_is_scoped_and_respects_correlation_setting(
|
||||
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "request_correlation_in_logs", True)
|
||||
context_before: Final = logger.context()
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
logger.emit(logging.WARNING, "native", "native.rs", 1, "litellm_http", {}, ("session", "trace"))
|
||||
monkeypatch.setattr(litellm, "request_correlation_in_logs", False)
|
||||
logger.emit(logging.WARNING, "disabled", "native.rs", 2, "litellm_http", {}, ("hidden", "hidden"))
|
||||
|
||||
first, second = caplog.records
|
||||
assert (first.__dict__["session_id"], first.__dict__["trace_id"]) == ("session", "trace")
|
||||
assert "session_id" not in second.__dict__
|
||||
assert "trace_id" not in second.__dict__
|
||||
assert (session_id_var.get(), trace_id_var.get()) == context_before
|
||||
|
||||
|
||||
def test_native_logging_observes_level_changes(caplog: pytest.LogCaptureFixture) -> None:
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM"):
|
||||
assert not logger.enabled(logging.WARNING)
|
||||
logger.emit(logging.WARNING, "filtered", "native.rs", 1, "litellm_http", {}, ("", ""))
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
assert logger.enabled(logging.WARNING)
|
||||
logger.emit(logging.WARNING, "visible", "native.rs", 1, "litellm_http", {}, ("", ""))
|
||||
|
||||
assert [record.getMessage() for record in caplog.records] == ["visible"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"text",
|
||||
(
|
||||
"Authorization: Bearer abcdefghijklmnop",
|
||||
"s3_secret_access_key=secret123",
|
||||
"postgres://user:pass@database.internal/name",
|
||||
'{"type":"service_account","private_key":"secret123"}',
|
||||
"GET /v1?key=abcdefghij&page=2",
|
||||
),
|
||||
)
|
||||
def test_native_credential_patterns_match_python(text: str) -> None:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
from litellm.rust_bridge._native import NativeDiagnosticProcessor
|
||||
|
||||
processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
assert processor.redact_text(text) == _python_redact_string(text)
|
||||
assert processor.redact_structured_text("api_key", "secret123") == _python_redact_structured_value(
|
||||
"api_key", "secret123"
|
||||
)
|
||||
|
||||
|
||||
def test_native_client_redaction_matches_python() -> None:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
from litellm.rust_bridge._native import NativeDiagnosticProcessor
|
||||
|
||||
text: Final = "error at /etc/secrets/config on db.internal\nTraceback (most recent call last):\nsecret"
|
||||
processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
assert processor.redact_client_message(text) == _python_redact_internal_details(text)
|
||||
|
||||
|
||||
def test_native_diagnostic_batch_matches_python() -> None:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
from litellm.rust_bridge._native import NativeDiagnosticProcessor
|
||||
|
||||
message: Final = "é" * 110 + "sk-" + "q" * 48 + "界" * 1000
|
||||
exception: Final = "document=" + "Q" * 200
|
||||
stack: Final = "api_key=secret123"
|
||||
leaves: Final = (("api_key", "secret123"), (None, "safe"))
|
||||
processor: Final = NativeDiagnosticProcessor(MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
rust: Final = processor.process_diagnostic(message, exception, stack, leaves, (True, 20, 500))
|
||||
python: Final = _python_process_diagnostic(message, exception, stack, leaves, True, 20, 500)
|
||||
|
||||
assert rust[:3] == python[:3]
|
||||
assert tuple(rust[3]) == python[3]
|
||||
assert rust[4] == python[4]
|
||||
assert "sk-qq" not in rust[0]
|
||||
assert len(rust[0]) <= 500
|
||||
assert rust[3] == ["REDACTED", "safe"]
|
||||
|
||||
|
||||
def test_missing_native_diagnostic_processor_falls_back_before_record_mutation(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST", "1")
|
||||
diagnostics.PROCESSOR.override(None)
|
||||
try:
|
||||
record: Final = logging.makeLogRecord({"name": "LiteLLM", "levelno": logging.INFO, "msg": "api_key=secret123"})
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
assert record.getMessage() == "REDACTED"
|
||||
finally:
|
||||
diagnostics.PROCESSOR.reset()
|
||||
|
||||
|
||||
def test_unsupported_unicode_uses_safe_python_redaction(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("LITELLM_RUST", "1")
|
||||
assert redact_secrets("broken\ud800 api_key=secret123") == "broken\ud800 REDACTED"
|
||||
|
|
@ -1,4 +1,3 @@
|
|||
import logging
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
|
@ -11,6 +10,7 @@ from litellm.rust_bridge import settings
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem
|
||||
|
||||
|
||||
def test_url_policy_reads_the_litellm_globals(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "user_url_validation", False)
|
||||
monkeypatch.setattr(litellm, "user_url_allowed_hosts", ["docs.internal:8443"])
|
||||
|
|
@ -57,13 +57,6 @@ def test_http_settings_ignores_environment_overrides(monkeypatch: pytest.MonkeyP
|
|||
assert result.ssl_verify is True
|
||||
|
||||
|
||||
def test_warn_reaches_the_litellm_logger(caplog: pytest.LogCaptureFixture) -> None:
|
||||
with caplog.at_level(logging.WARNING, logger="LiteLLM"):
|
||||
settings.warn("ssl_ecdh_curve 'secp521r1' is not supported")
|
||||
|
||||
assert [record.getMessage() for record in caplog.records] == ["ssl_ecdh_curve 'secp521r1' is not supported"]
|
||||
|
||||
|
||||
class _VaultSecrets(CustomSecretManager):
|
||||
def __init__(self, secrets: dict[str, str]) -> None:
|
||||
super().__init__(secret_manager_name="rust_bridge_settings_test")
|
||||
|
|
|
|||
|
|
@ -19,6 +19,16 @@ from litellm._logging import (
|
|||
_COLOR_LOG_FORMAT,
|
||||
_MAX_SCRUBBED_ACCESS_ARG,
|
||||
_PLAIN_LOG_FORMAT,
|
||||
ALL_LOGGERS,
|
||||
AccessLogPathFilter,
|
||||
AccessLogRedactionFilter,
|
||||
CorrelationContextFilter,
|
||||
CorrelationPlainFormatter,
|
||||
DiagnosticProcessingFilter,
|
||||
JsonFormatter,
|
||||
LevelRoutingStreamHandler,
|
||||
SecretRedactionFilter,
|
||||
StdoutLogTruncationFilter,
|
||||
_get_uvicorn_json_log_config,
|
||||
_initialize_loggers_with_handler,
|
||||
_parse_json_logs_env,
|
||||
|
|
@ -33,15 +43,6 @@ from litellm._logging import (
|
|||
verbose_logger,
|
||||
verbose_proxy_logger,
|
||||
verbose_router_logger,
|
||||
ALL_LOGGERS,
|
||||
AccessLogPathFilter,
|
||||
AccessLogRedactionFilter,
|
||||
CorrelationContextFilter,
|
||||
CorrelationPlainFormatter,
|
||||
JsonFormatter,
|
||||
LevelRoutingStreamHandler,
|
||||
SecretRedactionFilter,
|
||||
StdoutLogTruncationFilter,
|
||||
)
|
||||
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
|
|
@ -824,6 +825,75 @@ def test_secret_filter_keeps_truncated_traceback(monkeypatch):
|
|||
assert "sk-1234567890abcdefghij" not in record.exc_text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_diagnostic_redaction_precedes_a_credential_cut(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setenv("MAX_STRING_LENGTH_STDOUT_LOG", "500")
|
||||
monkeypatch.setenv("MAX_BASE64_LENGTH_STDOUT_LOG", "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
secret = "sk-" + "q" * 48
|
||||
record = _make_record(logging.INFO, "%s", ("é" * 110 + secret + "界" * 1000,))
|
||||
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
|
||||
assert len(record.getMessage()) <= 500
|
||||
assert "sk-qq" not in record.getMessage()
|
||||
|
||||
|
||||
def test_correlation_id_redacts_before_its_length_bound(monkeypatch):
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
secret = "sk-" + "q" * 48
|
||||
token = set_trace_id("x" * 250 + secret)
|
||||
try:
|
||||
assert "sk-qq" not in trace_id_var.get()
|
||||
assert len(trace_id_var.get()) <= 256
|
||||
finally:
|
||||
trace_id_var.reset(token)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_malformed_interpolation_still_scrubs_a_record(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "bad % api_key=secret123", ("value",))
|
||||
record.color_message = "bad % api_key=secret123"
|
||||
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == "REDACTED"
|
||||
assert record.color_message == "REDACTED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_key_pattern_template_keeps_the_rendered_redacted_line(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.INFO, "password=%s ok", ("hunter2",))
|
||||
record.color_message = "password=%s ok"
|
||||
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
|
||||
assert record.getMessage() == "REDACTED ok"
|
||||
assert record.color_message == "REDACTED ok"
|
||||
|
||||
|
||||
def test_disabled_diagnostic_call_does_not_render_arguments(caplog):
|
||||
class Unrenderable:
|
||||
def __str__(self):
|
||||
raise AssertionError("disabled call rendered its argument")
|
||||
|
||||
with caplog.at_level(logging.ERROR, logger="LiteLLM"):
|
||||
verbose_logger.debug("hidden %s", Unrenderable())
|
||||
|
||||
assert not caplog.records
|
||||
|
||||
|
||||
def test_truncation_filter_survives_json_reconfiguration():
|
||||
"""The cap lives on the loggers, so swapping handlers (JSON mode) can't drop it."""
|
||||
_turn_on_json()
|
||||
|
|
@ -983,10 +1053,10 @@ _REQUEST_DUMP = "{'model': 'gpt-4', 'messages': [{'role': 'user', 'content': 'he
|
|||
(CorrelationPlainFormatter(_PLAIN_LOG_FORMAT), JsonFormatter()),
|
||||
ids=("plain", "json"),
|
||||
)
|
||||
def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter):
|
||||
"""Every pass of the secret regex over a multi-megabyte debug line costs seconds of
|
||||
event-loop time, so a formatter must not rescan what SecretRedactionFilter scrubbed."""
|
||||
def test_scrubbed_record_scans_the_large_rendered_value_once(monkeypatch, formatter):
|
||||
"""The raw format template gets its own check, while the large rendered value gets one scan."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setenv("LITELLM_RUST", "0")
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,))
|
||||
|
|
@ -997,14 +1067,15 @@ def test_scrubbed_record_is_scanned_for_secrets_once(monkeypatch, formatter):
|
|||
|
||||
assert _REQUEST_DUMP in rendered
|
||||
assert "litellm_redacted" not in rendered
|
||||
assert counting.calls == 1
|
||||
assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}")
|
||||
assert counting.calls == 2
|
||||
assert counting.scanned_chars == len(f"receiving data: {_REQUEST_DUMP}") + len("receiving data: %s")
|
||||
|
||||
|
||||
def test_stamped_record_is_not_scanned_again(monkeypatch):
|
||||
"""JSON mode puts the filter on a third-party logger and again on the root handler its
|
||||
records propagate to, so the second filter must trust the stamp instead of rescanning."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setenv("LITELLM_RUST", "0")
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "receiving data: %s", (_REQUEST_DUMP,))
|
||||
|
|
@ -1012,13 +1083,14 @@ def test_stamped_record_is_not_scanned_again(monkeypatch):
|
|||
assert SecretRedactionFilter().filter(record) is True
|
||||
assert SecretRedactionFilter().filter(record) is True
|
||||
|
||||
assert counting.calls == 1
|
||||
assert counting.calls == 2
|
||||
|
||||
|
||||
def test_caller_supplied_stamp_never_skips_the_scrub(monkeypatch):
|
||||
"""The stamp is a private sentinel, so a caller passing extra={"litellm_redacted": True}
|
||||
still gets the full scrub, and only the filter's own stamp lets a later pass skip it."""
|
||||
counting = _CountingPattern(secret_redaction._SECRET_RE)
|
||||
monkeypatch.setenv("LITELLM_RUST", "0")
|
||||
monkeypatch.setattr(secret_redaction, "_SECRET_RE", counting)
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.DEBUG, "api_key=sk-1234567890abcdefghij")
|
||||
|
|
@ -1619,3 +1691,75 @@ def test_access_log_path_filter_keeps_a_record_without_a_string_path_arg(monkeyp
|
|||
exc_info=None,
|
||||
)
|
||||
assert AccessLogPathFilter().filter(record) is True
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_diagnostic_filter_scrubs_exc_stack_and_nested_extras(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
secret = "sk-" + "q" * 48
|
||||
try:
|
||||
raise ValueError(f"upstream rejected {secret}")
|
||||
except ValueError:
|
||||
record = _make_record(logging.ERROR, "call failed", exc_info=sys.exc_info())
|
||||
record.stack_info = f"Stack (most recent call last): {secret}"
|
||||
record.payload = {
|
||||
"api_key": secret,
|
||||
"items": [secret, "ok"],
|
||||
"tags": {secret},
|
||||
"pair": (secret, "ok"),
|
||||
"count": 2,
|
||||
}
|
||||
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
|
||||
assert secret not in (record.exc_text or "")
|
||||
assert secret not in (record.stack_info or "")
|
||||
assert secret not in repr(record.payload)
|
||||
assert record.payload["count"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_diagnostic_filter_stamps_records_so_a_second_pass_is_free(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
record = _make_record(logging.WARNING, "api_key=secret123")
|
||||
diagnostic_filter = DiagnosticProcessingFilter()
|
||||
|
||||
assert diagnostic_filter.filter(record) is True
|
||||
assert diagnostic_filter.filter(record) is True
|
||||
assert record.getMessage() == "REDACTED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_json_formatter_scrubs_unfiltered_extras(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
secret = "sk-" + "q" * 48
|
||||
record = _make_record(logging.INFO, "response complete")
|
||||
record.payload = {"api_key": secret, "nested": {"list": [secret]}}
|
||||
|
||||
rendered = JsonFormatter().format(record)
|
||||
|
||||
assert secret not in rendered
|
||||
assert "REDACTED" in rendered
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_diagnostic_filter_redacts_a_non_string_message_object(monkeypatch, native):
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
monkeypatch.setattr("litellm._logging._ENABLE_SECRET_REDACTION", True)
|
||||
secret = "sk-" + "q" * 48
|
||||
record = _make_record(logging.ERROR, {"api_key": secret})
|
||||
|
||||
assert DiagnosticProcessingFilter().filter(record) is True
|
||||
|
||||
assert secret not in record.getMessage()
|
||||
|
|
|
|||
|
|
@ -19,7 +19,11 @@ from litellm._logging import (
|
|||
verbose_proxy_logger,
|
||||
verbose_router_logger,
|
||||
)
|
||||
from litellm.litellm_core_utils.secret_redaction import redact_internal_details, redact_string
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
redact_internal_details,
|
||||
redact_string,
|
||||
redact_structured_value,
|
||||
)
|
||||
|
||||
SECRET = "sk-proj-abc123def456ghi789jklmnopqrst"
|
||||
|
||||
|
|
@ -71,6 +75,17 @@ def test_redact_string_catches_secret_patterns():
|
|||
assert redact_string(normal) == normal
|
||||
|
||||
|
||||
@pytest.mark.parametrize("native", (False, True), ids=("python", "rust"))
|
||||
def test_diagnostic_redaction_policy_matches_across_backends(monkeypatch: pytest.MonkeyPatch, native: bool) -> None:
|
||||
if native:
|
||||
pytest.importorskip("litellm.rust_bridge._native")
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if native else "0")
|
||||
|
||||
assert redact_string("GET /v1?api_key=abcdefgh12345&page=2") == "GET /v1?REDACTED&page=2"
|
||||
assert redact_structured_value("db_url", "postgresql://reader@example.org/database") == "REDACTED"
|
||||
assert redact_internal_details("failed at /etc/service/keys on db.internal") == "failed at REDACTED on REDACTED"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"connection_string",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import logging
|
|||
import os
|
||||
import queue
|
||||
import threading
|
||||
from collections.abc import Callable, Iterator
|
||||
from collections.abc import Callable, Iterator, Mapping
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import PurePath
|
||||
|
|
@ -5283,23 +5283,31 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
|
|||
) -> None:
|
||||
"""Regression: an error raised after the deployment call already succeeded (e.g. inside
|
||||
async_post_call_success_deployment_hook or post_call_processing) is not a deployment
|
||||
attempt failure and must not reach async_post_call_failure_deployment_hook."""
|
||||
attempt failure and must not reach async_post_call_failure_deployment_hook. The raising
|
||||
callback is a guardrail because a plain logger's success hook error is isolated and
|
||||
logged instead of propagating out of the call."""
|
||||
|
||||
class ExplodingSuccessLogger(CustomLogger):
|
||||
class ExplodingSuccessGuardrail(CustomGuardrail):
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.failure_calls: list[Exception] = []
|
||||
super().__init__(guardrail_name="exploding")
|
||||
self.failure_calls: tuple[Exception, ...] = ()
|
||||
|
||||
async def async_post_call_success_deployment_hook(self, request_data, response, call_type):
|
||||
async def async_post_call_success_deployment_hook(
|
||||
self, request_data: Mapping[str, object], response: LLMResponseTypes, call_type: CallTypes | None
|
||||
) -> LLMResponseTypes | None:
|
||||
raise RuntimeError("boom in success hook, model call itself succeeded")
|
||||
|
||||
async def async_post_call_failure_deployment_hook(
|
||||
self, request_data, exception, call_type, fallback_depth=None
|
||||
):
|
||||
self.failure_calls.append(exception)
|
||||
self,
|
||||
request_data: Mapping[str, object],
|
||||
exception: Exception,
|
||||
call_type: CallTypes | None,
|
||||
fallback_depth: int | None = None,
|
||||
) -> None:
|
||||
self.failure_calls = (*self.failure_calls, exception)
|
||||
|
||||
exploding_logger = ExplodingSuccessLogger()
|
||||
monkeypatch.setattr(litellm, "callbacks", [exploding_logger])
|
||||
exploding_guardrail: Final = ExplodingSuccessGuardrail()
|
||||
monkeypatch.setattr(litellm, "callbacks", [exploding_guardrail])
|
||||
|
||||
with pytest.raises(RuntimeError, match="boom in success hook"):
|
||||
await litellm.acompletion(
|
||||
|
|
@ -5308,7 +5316,7 @@ async def test_wrapper_async_does_not_fire_failure_hook_for_post_success_error(
|
|||
mock_response="this call succeeds",
|
||||
)
|
||||
|
||||
assert exploding_logger.failure_calls == []
|
||||
assert exploding_guardrail.failure_calls == ()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue