chore: merge origin/main into litellm_config_vector_stores_read_only

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
ryan 2026-09-23 03:55:15 +00:00
commit 74fe1dad49
126 changed files with 7630 additions and 1009 deletions

View file

@ -0,0 +1,33 @@
name: Compat Matrix Image
on:
pull_request:
paths:
- tests/e2e/claude_code/cron_vm/**
- .github/workflows/compat-matrix-image.yml
workflow_dispatch:
permissions: {}
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
compat-matrix-image:
name: compat-matrix-image
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Build the Render cron image
run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
- name: Run the pinned binaries as the cron user
run: |
docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version'

View file

@ -105,6 +105,16 @@ jobs:
with:
python-version: "3.12"
- uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install Python dependencies for the bridge tests
working-directory: .
run: |
uv sync --frozen --no-install-project
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
- run: rustup toolchain install --no-self-update
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

View file

@ -1,10 +1,10 @@
# syntax=docker/dockerfile:1.7
# Base image for building
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
# Runtime image
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43

View file

@ -1,8 +1,8 @@
# syntax=docker/dockerfile:1.7
# Base images
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG PROXY_EXTRAS_SOURCE=published
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Pinned by digest like the other base images; bump explicitly on Node upgrades.

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.69"
version = "0.1.70"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.69"
version = "0.1.70"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-enterprise==",

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
ARG PGBOUNCER_VERSION=1.25.2

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.100"
version = "0.4.101"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.100"
version = "0.4.101"
version_files = [
"pyproject.toml:^version",
"../pyproject.toml:litellm-proxy-extras==",

View file

@ -2946,6 +2946,7 @@ name = "litellm-core-utils"
version = "0.1.0"
dependencies = [
"fancy-regex 0.19.2",
"litellm-tracing",
"litellm-types",
"rstest",
"serde",
@ -3075,6 +3076,7 @@ dependencies = [
"aws-sdk-secretsmanager",
"bytes",
"criterion",
"fancy-regex 0.19.2",
"futures-util",
"litellm-auth",
"litellm-auth-aws",
@ -3093,6 +3095,7 @@ dependencies = [
"litellm-callbacks-legacy-python",
"litellm-core",
"litellm-core-utils",
"litellm-host",
"litellm-host-python",
"litellm-http",
"litellm-llms",
@ -3100,6 +3103,7 @@ dependencies = [
"litellm-secrets-aws",
"litellm-secrets-types",
"litellm-token-counter",
"litellm-tracing",
"litellm-types",
"pyo3",
"pyo3-async-runtimes",
@ -3172,11 +3176,11 @@ dependencies = [
"litellm-auth-aws",
"litellm-core-utils",
"litellm-secrets-types",
"litellm-tracing",
"rstest",
"serde_json",
"thiserror 2.0.19",
"tokio",
"tracing",
"veil",
"wiremock",
]
@ -3208,6 +3212,7 @@ dependencies = [
"base64 0.22.1",
"litellm-core-utils",
"litellm-secrets-types",
"litellm-tracing",
"moka",
"percent-encoding",
"reqwest 0.12.28",
@ -3216,7 +3221,6 @@ dependencies = [
"serde_json",
"thiserror 2.0.19",
"tokio",
"tracing",
"veil",
"wiremock",
]
@ -3331,6 +3335,19 @@ dependencies = [
"tiktoken-rs",
]
[[package]]
name = "litellm-tracing"
version = "0.1.0"
dependencies = [
"fancy-regex 0.19.2",
"percent-encoding",
"rstest",
"serde_json",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "litellm-types"
version = "0.1.0"

View file

@ -9,6 +9,8 @@ license = "MIT"
repository = "https://github.com/BerriAI/litellm"
[workspace.dependencies]
litellm-tracing = { path = "crates/tracing" }
tracing = "0.1"
litellm-core = { path = "crates/core" }
litellm-host = { path = "crates/host" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }

View file

@ -7,6 +7,7 @@ repository.workspace = true
[dependencies]
fancy-regex.workspace = true
litellm-tracing.workspace = true
litellm-types.workspace = true
serde.workspace = true
serde_json.workspace = true

View file

@ -1,109 +1 @@
use fancy_regex::Regex;
pub const REDACTED: &str = "REDACTED";
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
fn minimum_custom_key_length() -> usize {
std::env::var("MINIMUM_CUSTOM_KEY_LENGTH")
.ok()
.and_then(|value| value.trim().parse().ok())
.unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH)
}
fn secret_patterns(minimum_custom_key_length: usize) -> String {
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
[
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
r"\bya29\.[A-Za-z0-9_.~+/-]+",
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r"x-ak-[A-Za-z0-9\-_]{20,}",
r"AIza[0-9A-Za-z\-_]{35}",
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
r"dapi[0-9a-f]{32}",
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
concat!(
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
r"aws_secret_access_key|aws_session_token|aws_access_key_id|",
r"signing_key|encryption_key|",
r"auth_token|access_token|refresh_token|",
r"slack_webhook_url|webhook_url|",
r"database_connection_string|",
r"huggingface_token|jwt_secret)",
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
),
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
]
.join("|")
}
/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration.
#[derive(Clone, Debug)]
pub struct SecretRedactor {
pattern: Regex,
}
impl SecretRedactor {
pub fn new(minimum_custom_key_length: usize) -> Self {
let pattern = Regex::new(&format!(
"(?i){}",
secret_patterns(minimum_custom_key_length)
))
.expect("secret redaction patterns compile");
Self { pattern }
}
/// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off.
pub fn from_env() -> Option<Self> {
let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS")
.is_ok_and(|value| value.eq_ignore_ascii_case("true"));
(!disabled).then(|| Self::new(minimum_custom_key_length()))
}
pub fn redact(&self, value: &str) -> String {
self.pattern.replace_all(value, REDACTED).into_owned()
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
expected
);
}
#[test]
fn sk_threshold_follows_the_minimum_custom_key_length() {
let redactor = SecretRedactor::new(8);
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
}
}
pub use litellm_tracing::{REDACTED, SecretRedactor};

View file

@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"]
tiktoken = ["litellm-token-counter/tiktoken"]
[dependencies]
fancy-regex.workspace = true
litellm-tracing.workspace = true
litellm-host.workspace = true
bytes.workspace = true
futures-util.workspace = true
litellm-cache.workspace = true

View file

@ -1,6 +1,7 @@
use crate::logger::run_sync_value;
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use litellm_cache_redis_semantic::RedisSemanticConfig;
use litellm_host_python::{release_gil, run_sync_value};
use litellm_host_python::release_gil;
use litellm_http::ClientVariant;
use pyo3::prelude::*;

View file

@ -1,5 +1,6 @@
use crate::logger::run_async;
use litellm_cache_response::PartialHits;
use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py};
use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyRuntimeError, PyValueError},

View file

@ -1,10 +1,11 @@
use crate::logger::run_sync_value;
use litellm_auth_aws::AwsAuthConfig;
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization};
use litellm_cache_redis::{RedisNode, RedisTopology};
use litellm_cache_redis_semantic::RedisSemanticConfig;
use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
use litellm_host_python::{release_gil, run_sync_value};
use litellm_host_python::release_gil;
use litellm_http::ClientVariant;
use pyo3::{
PyTraverseError, PyVisit,

View file

@ -470,7 +470,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move {
service
@ -495,7 +495,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_lookup(&request, now()).await },
super::cache_error,
@ -550,7 +550,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_store(&request, response, now()).await },
super::cache_error,
@ -619,7 +619,7 @@ impl NativeResponseCache {
match self {
Self::Exact(_) | Self::QdrantSemantic(_) => {
let service = self.clone();
litellm_host_python::run_async(
crate::logger::run_async(
py,
async move { service.async_store_batch(entries, now()).await },
super::cache_error,

View file

@ -1,7 +1,8 @@
use crate::logger::run_async;
use std::{collections::VecDeque, time::Duration};
use litellm_cache::Error;
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
use pyo3::{
PyTraverseError, PyVisit,
exceptions::{PyException, PyRuntimeError},

View file

@ -102,7 +102,7 @@ pub(crate) fn call_config(
.without_missing_files(&|path: &Path| path.exists());
let resolution = Resolution::from(&settings);
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
PythonSettings::warn(py, &unsupported.to_string())?;
crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}"));
}
Ok(resolution.config)
}

View file

@ -4,6 +4,7 @@ mod credentials;
mod diagnostics;
mod errors;
mod http;
mod logger;
mod marshal;
mod python_settings;
mod routes;
@ -25,6 +26,8 @@ mod _native {
#[pymodule_export]
use crate::errors::{RustBridgeDeclined, RustUpstreamError};
#[pymodule_export]
use crate::logger::NativeDiagnosticProcessor;
#[pymodule_export]
use crate::routes::audio_transcription::{atranscription, transcription};
#[pymodule_export]
use crate::routes::chat_completions::{
@ -87,6 +90,7 @@ mod tests {
"chat_completions",
"achat_completions",
"ResponsesWebSocketConnection",
"NativeDiagnosticProcessor",
"TokenCounter",
"Tokenizer",
"gil_stats",

View file

@ -0,0 +1,46 @@
use std::future::Future;
use pyo3::prelude::*;
use serde::Serialize;
pub(crate) fn run_sync<T, E, F>(
py: Python<'_>,
future: F,
map_error: fn(E) -> PyErr,
) -> PyResult<Py<PyAny>>
where
T: Serialize + Send + 'static,
E: Send + 'static,
F: Future<Output = Result<T, E>> + Send + 'static,
{
litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
}
pub(crate) fn run_async<T, E, F>(
py: Python<'_>,
future: F,
map_error: fn(E) -> PyErr,
) -> PyResult<Bound<'_, PyAny>>
where
T: Serialize + Send + 'static,
E: Send + 'static,
F: Future<Output = Result<T, E>> + Send + 'static,
{
litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
}
pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
where
T: Send + 'static,
F: Future<Output = PyResult<T>> + Send + 'static,
{
litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
}
pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
where
T: for<'py> IntoPyObject<'py> + Send + 'static,
F: Future<Output = PyResult<T>> + Send + 'static,
{
litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
}

View file

@ -0,0 +1,41 @@
use std::sync::OnceLock;
use litellm_host::{
host::HostResult,
machine::{HostFailure, Interrupted, Machine, Step},
route::Route,
};
use litellm_tracing::Logger;
use pyo3::Python;
pub(crate) struct LoggedMachine<M> {
machine: M,
logger: OnceLock<Logger>,
}
impl<M> LoggedMachine<M> {
pub(crate) fn new(machine: M) -> Self {
Self {
machine,
logger: OnceLock::new(),
}
}
}
impl<M: Machine> Machine for LoggedMachine<M> {
type Route = M::Route;
type Complete = M::Complete;
fn resume(&mut self, result: Option<HostResult<Self::Route>>) -> Step<'_, Self> {
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result))))
}
fn interrupt(
&mut self,
failure: HostFailure<<Self::Route as Route>::Error>,
) -> Interrupted<'_, Self> {
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
}
}

View file

@ -0,0 +1,166 @@
mod execution;
mod machine;
pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
pub(crate) use machine::LoggedMachine;
use litellm_host_python::Pythonized;
use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink};
use pyo3::exceptions::PyRuntimeError;
use pyo3::prelude::*;
const MODULE: &str = "litellm.rust_bridge.logger";
type NativeDiagnosticOutput = (String, Option<String>, Option<String>, Vec<String>, bool);
#[pyclass]
pub(crate) struct NativeDiagnosticProcessor {
inner: Processor,
}
#[pymethods]
impl NativeDiagnosticProcessor {
#[new]
fn new(minimum_custom_key_length: usize) -> Self {
Self {
inner: Processor::new(minimum_custom_key_length),
}
}
fn redact_text(&self, text: &str) -> PyResult<String> {
self.inner.redact_text(text).map_err(processing_error)
}
fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult<String> {
self.inner
.redact_structured_text(key, text)
.map_err(processing_error)
}
fn redact_client_message(&self, text: &str) -> PyResult<String> {
self.inner
.redact_client_message(text)
.map_err(processing_error)
}
#[pyo3(signature = (message, exception, stack, leaves, policy))]
fn process_diagnostic(
&self,
message: String,
exception: Option<String>,
stack: Option<String>,
leaves: Vec<(Option<String>, String)>,
policy: (bool, i64, i64),
) -> PyResult<NativeDiagnosticOutput> {
let input = DiagnosticInput {
message,
exception,
stack,
leaves,
};
let policy = Policy {
redact: policy.0,
base64_limit: policy.1,
text_limit: policy.2,
};
self.inner
.process_diagnostic(&input, policy)
.map(|output| {
(
output.message,
output.exception,
output.stack,
output.leaves,
output.changed,
)
})
.map_err(processing_error)
}
fn scrub_access_arguments(&self, arguments: Vec<String>) -> PyResult<Vec<String>> {
self.inner
.scrub_access_arguments(&arguments)
.map_err(processing_error)
}
}
fn processing_error(_: fancy_regex::Error) -> PyErr {
PyRuntimeError::new_err("diagnostic processing failed")
}
struct PythonSink {
correlation: (String, String),
}
fn level(level: &Level) -> u8 {
match *level {
Level::ERROR => 40,
Level::WARN => 30,
Level::INFO => 20,
Level::DEBUG | Level::TRACE => 10,
}
}
fn report<T: Default>(py: Python<'_>, result: PyResult<T>) -> T {
match result {
Ok(value) => value,
Err(error) => {
error.write_unraisable(py, None);
T::default()
}
}
}
impl Sink for PythonSink {
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::")
{
return false;
}
Python::try_attach(|py| {
report(
py,
py.import(MODULE)
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
.and_then(|enabled| enabled.extract()),
)
})
.unwrap_or(false)
}
fn emit(&self, record: &Record) {
Python::try_attach(|py| {
report(
py,
py.import(MODULE).and_then(|module| {
module
.call_method1(
"emit",
(
level(record.metadata.level()),
&record.message,
record.metadata.file().unwrap_or_default(),
record.metadata.line().unwrap_or_default(),
record.metadata.target(),
Pythonized(&record.fields),
(&self.correlation.0, &self.correlation.1),
),
)
.map(|_| ())
}),
);
});
}
}
pub(crate) fn capture(py: Python<'_>) -> Logger {
report(
py,
py.import(MODULE)
.and_then(|module| module.call_method0("context"))
.and_then(|value| value.extract())
.map(|correlation| Logger::new(PythonSink { correlation })),
)
}
#[cfg(test)]
mod tests;

View file

@ -0,0 +1,295 @@
use std::{process::Command, task::Poll};
use litellm_host::{
host::HostResult,
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
route::Route,
};
use pyo3::{prelude::*, types::PyDict};
struct DiagnosticMachine;
impl Route for DiagnosticMachine {
type Response = ();
type Error = String;
type Op = ();
type OpResult = ();
type Chunk = ();
type StreamHead = ();
}
impl Machine for DiagnosticMachine {
type Route = Self;
type Complete = ();
fn resume(&mut self, _: Option<HostResult<Self>>) -> Step<'_, Self> {
litellm_tracing::warn!("machine started");
Box::pin(async {
tokio::task::yield_now().await;
litellm_tracing::warn!("machine warning");
Ok(MachineStep::Complete(()))
})
}
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
Box::pin(async {
litellm_tracing::warn!("machine interrupted");
Ok(())
})
}
}
#[pyfunction]
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
let mut future = Box::pin(async move {
machine
.resume(None)
.await
.map_err(pyo3::exceptions::PyValueError::new_err)?;
machine
.interrupt(HostFailure::Error("stop".into()))
.await
.map_err(pyo3::exceptions::PyValueError::new_err)
});
assert!(matches!(
litellm_host_python::poll_async_value(py, future.as_mut())?,
Poll::Pending
));
litellm_host_python::run_async_value(py, future)
}
#[pyfunction]
fn warning(py: Python<'_>) {
super::capture(py).scope(|| {
litellm_tracing::warn!(attempt = 3, retry = true, "native warning");
});
}
#[pyfunction]
fn levels(py: Python<'_>) {
super::capture(py).scope(|| {
litellm_tracing::trace!("trace");
litellm_tracing::debug!("debug");
litellm_tracing::info!("info");
litellm_tracing::warn!("warn");
litellm_tracing::error!("error");
litellm_tracing::warn!(target: "unrelated_transport", "private wire data");
});
}
#[pyfunction]
fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
super::run_async_value(py, async {
tokio::task::yield_now().await;
litellm_tracing::warn!("async warning");
Ok(())
})
}
#[pyfunction]
fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
super::run_sync_value(py, async {
tokio::task::yield_now().await;
litellm_tracing::warn!("sync warning");
Ok(())
})
}
#[pyfunction]
fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
super::run_sync_value(py, async {
litellm_tracing::warn!("failure diagnostic");
Err(pyo3::exceptions::PyValueError::new_err("request failed"))
})
}
#[pyfunction]
fn http_warning(py: Python<'_>) -> PyResult<()> {
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
}
#[test]
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
let output = Command::new(std::env::current_exe().unwrap())
.args([
"--exact",
std::thread::current().name().unwrap(),
"--nocapture",
])
.env("LITELLM_LOGGER_TEST_PROCESS", "1")
.output()
.unwrap();
assert!(
output.status.success(),
"{}\n{}",
String::from_utf8_lossy(&output.stdout),
String::from_utf8_lossy(&output.stderr)
);
return;
}
Python::initialize();
Python::attach(|py| {
let locals = PyDict::new(py);
locals
.set_item(
"repo_root",
concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."),
)
.unwrap();
locals
.set_item(
"machine_warning",
wrap_pyfunction!(machine_warning, py).unwrap(),
)
.unwrap();
locals
.set_item(
"synchronous_failure",
wrap_pyfunction!(synchronous_failure, py).unwrap(),
)
.unwrap();
locals
.set_item("levels", wrap_pyfunction!(levels, py).unwrap())
.unwrap();
locals
.set_item("warning", wrap_pyfunction!(warning, py).unwrap())
.unwrap();
locals
.set_item(
"asynchronous_warning",
wrap_pyfunction!(asynchronous_warning, py).unwrap(),
)
.unwrap();
locals
.set_item(
"synchronous_warning",
wrap_pyfunction!(synchronous_warning, py).unwrap(),
)
.unwrap();
locals
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
.unwrap();
let importable = py
.eval(
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
Some(&locals),
Some(&locals),
)
.unwrap()
.is_truthy()
.unwrap();
if !importable {
eprintln!("SKIP: litellm package dependencies are not importable in this interpreter");
return;
}
py.run(c"
import asyncio
import logging
import sys
sys.path.insert(0, repo_root)
import litellm
from litellm._logging import verbose_logger, session_id_var, trace_id_var
class Capture(logging.Handler):
def __init__(self):
super().__init__()
self.records = []
def emit(self, record):
self.records.append(record)
warning()
class Broken(logging.Handler):
def emit(self, record):
raise ValueError('handler failed')
capture = Capture()
old_handlers = verbose_logger.handlers
old_level = verbose_logger.level
old_correlation = litellm.request_correlation_in_logs
old_curve = litellm.ssl_ecdh_curve
old_unraisable = sys.unraisablehook
failures = []
try:
verbose_logger.handlers = [capture]
litellm.request_correlation_in_logs = True
verbose_logger.setLevel(logging.ERROR)
warning()
assert capture.records == []
verbose_logger.setLevel(logging.WARNING)
warning()
assert len(capture.records) == 1
record = capture.records[0]
assert record.getMessage() == 'native warning'
assert record.levelno == logging.WARNING
assert record.rust_fields == {'attempt': 3, 'retry': True}
assert record.pathname.endswith('logger/tests.rs')
assert record.lineno > 0
assert record.rust_target.endswith('logger::tests')
verbose_logger.setLevel(logging.ERROR)
warning()
assert len(capture.records) == 1
verbose_logger.setLevel(logging.WARNING)
async def request(name):
session = session_id_var.set(name)
trace = trace_id_var.set('trace-' + name)
try:
await asynchronous_warning()
await machine_warning()
synchronous_warning()
assert session_id_var.get() == name
assert trace_id_var.get() == 'trace-' + name
finally:
trace_id_var.reset(trace)
session_id_var.reset(session)
async def concurrent():
await asyncio.gather(request('first'), request('second'))
asyncio.run(concurrent())
assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted(
(message, name, 'trace-' + name)
for name in ('first', 'second')
for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted')
)
verbose_logger.setLevel(logging.DEBUG)
before_levels = len(capture.records)
levels()
assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [
('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO),
('warn', logging.WARNING), ('error', logging.ERROR),
]
before = len(capture.records)
litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve'
http_warning()
http_warning()
assert len(capture.records) == before + 1
assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage()
assert capture.records[-1].pathname.endswith('http.rs')
verbose_logger.handlers = [Broken()]
sys.unraisablehook = failures.append
warning()
assert len(failures) == 1
assert str(failures[0].exc_value) == 'handler failed'
try:
synchronous_failure()
except ValueError as error:
assert str(error) == 'request failed'
else:
raise AssertionError('request failure was lost')
assert len(failures) == 2
finally:
sys.unraisablehook = old_unraisable
verbose_logger.handlers = old_handlers
verbose_logger.setLevel(old_level)
litellm.request_correlation_in_logs = old_correlation
litellm.ssl_ecdh_curve = old_curve
", Some(&locals), Some(&locals)).unwrap();
});
}

View file

@ -44,11 +44,6 @@ impl PythonSettings {
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
Snapshot { group: self, value }
}
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
py.import(MODULE)?.getattr("warn")?.call1((message,))?;
Ok(())
}
}
#[cfg(test)]

View file

@ -1,7 +1,8 @@
use crate::logger::{run_async, run_sync};
use litellm_core::audio_transcription::{
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use litellm_host_python::from_py_argument;
use pyo3::prelude::*;
use serde_json::{Map, Value};

View file

@ -1,8 +1,9 @@
use crate::logger::{run_async, run_sync};
use litellm_core::chat_completions::{
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
types::ChatCompletionsRequest,
};
use litellm_host_python::{from_py_argument, run_async, run_sync};
use litellm_host_python::from_py_argument;
use litellm_types::utils::ChatCompletionsResponse;
use pyo3::prelude::*;
use serde_json::{Map, Value};

View file

@ -43,7 +43,7 @@ fn run_messages(
py,
SURFACE,
PublicCall::capture(&request, &args, &kwargs)?,
messages_machine(),
crate::logger::LoggedMachine::new(messages_machine()),
MessagesRouteHost::new(request.unbind()),
asynchronous,
)

View file

@ -73,7 +73,7 @@ fn run_ocr(
py,
if asynchronous { ASYNC_SURFACE } else { SURFACE },
PublicCall::capture(&request, &args, &kwargs)?,
ocr_machine(client),
crate::logger::LoggedMachine::new(ocr_machine(client)),
OcrRouteHost::new(request.unbind()),
asynchronous,
)

View file

@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection {
) -> PyResult<Bound<'py, PyAny>> {
let headers = marshal_headers(headers)?;
let timeout = optional_timeout(timeout_seconds);
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
.await
.map_err(responses_error_to_pyerr)?;
@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection {
fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner
.send_text(text)
.await
@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection {
fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner.recv_text().await.map_err(responses_error_to_pyerr)
})
}
fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
let inner = self.inner.clone();
litellm_host_python::run_async_value(py, async move {
crate::logger::run_async_value(py, async move {
inner.close().await.map_err(responses_error_to_pyerr)
})
}

View file

@ -1,7 +1,8 @@
use crate::logger::run_async;
use std::sync::Arc;
use std::{num::NonZero, thread::available_parallelism};
use litellm_host_python::{enter_native, run_async};
use litellm_host_python::enter_native;
use litellm_token_counter::{
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
};

View file

@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true
litellm-core-utils.workspace = true
serde_json.workspace = true
thiserror.workspace = true
tracing = "0.1"
litellm-tracing.workspace = true
veil.workspace = true
aws-sdk-kms = "1.120.0"
aws-sdk-secretsmanager = "1.117.0"

View file

@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 {
.await
.is_err()
{
tracing::warn!("secret created but replication failed");
litellm_tracing::warn!("secret created but replication failed");
}
Ok(response)
}

View file

@ -14,7 +14,7 @@ reqwest.workspace = true
serde_json.workspace = true
thiserror.workspace = true
veil.workspace = true
tracing = "0.1"
litellm-tracing.workspace = true
percent-encoding = "2.3"
tokio = { workspace = true, features = ["sync"] }

View file

@ -92,7 +92,7 @@ impl CyberArkSecretManager {
.unwrap_or(true);
let mut builder = reqwest::Client::builder();
if !verify {
tracing::warn!(
litellm_tracing::warn!(
"CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
);
builder = builder.danger_accept_invalid_certs(true);
@ -259,11 +259,13 @@ impl CyberArkSecretManager {
.endpoint
.join(&format!("policies/{}/policy/root", self.account));
let Ok(policy_url) = policy_url else {
tracing::warn!("Could not build CyberArk policy endpoint");
litellm_tracing::warn!("Could not build CyberArk policy endpoint");
return;
};
let Ok(authorization) = self.authorization_header(context).await else {
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
litellm_tracing::warn!(
"Could not authenticate while ensuring CyberArk variable exists"
);
return;
};
let body = format!(
@ -288,19 +290,19 @@ impl CyberArkSecretManager {
reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
) =>
{
tracing::debug!(
litellm_tracing::debug!(
"CyberArk variable policy already exists or conflicts: {}",
response.status()
);
}
Ok(response) => {
tracing::warn!(
litellm_tracing::warn!(
"Could not ensure CyberArk variable exists: {}",
response.status()
);
}
Err(error) => {
tracing::warn!("Error ensuring CyberArk variable exists: {error}");
litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}");
}
}
}
@ -324,7 +326,7 @@ impl CyberArkSecretManager {
_recovery_window_in_days: Option<u32>,
_context: &SecretOperationContext,
) -> Result<DeleteOutcome, Error> {
tracing::warn!(
litellm_tracing::warn!(
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
);
self.secrets.invalidate(name).await;

View file

@ -0,0 +1,17 @@
[package]
name = "litellm-tracing"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
fancy-regex.workspace = true
percent-encoding.workspace = true
serde_json.workspace = true
tracing.workspace = true
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
[dev-dependencies]
rstest.workspace = true
tokio.workspace = true

View file

@ -0,0 +1,26 @@
# Native diagnostic tracing
`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber
Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work
Bindings implement `litellm_tracing::Sink` to connect events to their host runtime:
```rust
pub trait Sink: Send + Sync + 'static {
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
fn emit(&self, record: &Record);
}
```
Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value
Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion
The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger
Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events
A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue
This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate

View file

@ -0,0 +1,146 @@
use std::{
cell::Cell,
fmt,
future::{Future, poll_fn},
pin::pin,
};
use serde_json::{Map, Value};
use tracing::{
Dispatch, Event, Subscriber,
field::{Field, Visit},
subscriber::Interest,
};
use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
mod processing;
mod redaction;
pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor};
pub use redaction::{REDACTED, SecretRedactor};
pub use tracing::{Level, Metadata, debug, error, info, trace, warn};
pub trait Sink: Send + Sync + 'static {
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
fn emit(&self, record: &Record);
}
#[derive(Debug)]
pub struct Record {
pub metadata: &'static Metadata<'static>,
pub message: String,
pub fields: Map<String, Value>,
}
#[derive(Clone, Default)]
pub struct Logger {
dispatch: Dispatch,
}
impl Logger {
pub fn new(sink: impl Sink) -> Self {
Self {
dispatch: Dispatch::new(Registry::default().with(Output(sink))),
}
}
pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
if EMITTING.get() {
return operation();
}
tracing::dispatcher::with_default(&self.dispatch, operation)
}
pub fn instrument<F: Future>(&self, future: F) -> impl Future<Output = F::Output> + use<F> {
let logger = self.clone();
async move {
let mut future = pin!(future);
poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await
}
}
}
thread_local! {
static EMITTING: Cell<bool> = const { Cell::new(false) };
}
struct Emitting;
impl Emitting {
fn enter() -> Option<Self> {
EMITTING.with(|active| (!active.replace(true)).then_some(Self))
}
}
impl Drop for Emitting {
fn drop(&mut self) {
EMITTING.set(false);
}
}
struct Output<S>(S);
impl<S: Sink, R: Subscriber> Layer<R> for Output<S> {
fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {
Interest::sometimes()
}
fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool {
let Some(_guard) = Emitting::enter() else {
return false;
};
self.0.enabled(metadata)
}
fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) {
let Some(_guard) = Emitting::enter() else {
return;
};
let mut record = Record {
metadata: event.metadata(),
message: String::new(),
fields: Map::new(),
};
event.record(&mut record);
self.0.emit(&record);
}
}
impl Record {
fn field(&mut self, field: &Field, value: Value) {
if field.name() == "message" {
self.message = match value {
Value::String(message) => message,
value => value.to_string(),
};
} else {
self.fields.insert(field.name().to_owned(), value);
}
}
}
impl Visit for Record {
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
self.field(field, format!("{value:?}").into());
}
fn record_str(&mut self, field: &Field, value: &str) {
self.field(field, value.into());
}
fn record_bool(&mut self, field: &Field, value: bool) {
self.field(field, value.into());
}
fn record_i64(&mut self, field: &Field, value: i64) {
self.field(field, value.into());
}
fn record_u64(&mut self, field: &Field, value: u64) {
self.field(field, value.into());
}
fn record_f64(&mut self, field: &Field, value: f64) {
self.field(field, value.into());
}
}

View file

@ -0,0 +1,333 @@
use fancy_regex::Result;
use percent_encoding::percent_decode_str;
use crate::{REDACTED, SecretRedactor};
#[derive(Clone, Copy, Debug)]
pub struct Policy {
pub redact: bool,
pub base64_limit: i64,
pub text_limit: i64,
}
#[derive(Clone, Debug)]
pub struct DiagnosticInput {
pub message: String,
pub exception: Option<String>,
pub stack: Option<String>,
pub leaves: Vec<(Option<String>, String)>,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct DiagnosticOutput {
pub message: String,
pub exception: Option<String>,
pub stack: Option<String>,
pub leaves: Vec<String>,
pub changed: bool,
}
pub struct Processor {
redactor: SecretRedactor,
}
impl Processor {
pub fn new(minimum_custom_key_length: usize) -> Self {
Self {
redactor: SecretRedactor::new(minimum_custom_key_length),
}
}
pub fn redact_text(&self, text: &str) -> Result<String> {
self.redactor.try_redact(text)
}
pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result<String> {
self.redactor.try_redact_structured(key, text)
}
pub fn redact_client_message(&self, text: &str) -> Result<String> {
self.redactor.try_redact_internal(text)
}
pub fn process_diagnostic(
&self,
input: &DiagnosticInput,
policy: Policy,
) -> Result<DiagnosticOutput> {
let message = self.process_text(&input.message, policy)?;
let exception = input
.exception
.as_deref()
.map(|text| self.process_text(text, policy))
.transpose()?;
let stack = input
.stack
.as_deref()
.map(|text| {
if policy.redact {
self.redact_text(text)
} else {
Ok(text.to_owned())
}
})
.transpose()?;
let leaves = input
.leaves
.iter()
.map(|(key, text)| {
if policy.redact {
self.redact_structured_text(key.as_deref(), text)
} else {
Ok(text.clone())
}
})
.collect::<Result<Vec<_>>>()?;
let changed = message != input.message
|| exception != input.exception
|| stack != input.stack
|| leaves
.iter()
.zip(&input.leaves)
.any(|(processed, (_, original))| processed != original);
Ok(DiagnosticOutput {
message,
exception,
stack,
leaves,
changed,
})
}
pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result<Vec<String>> {
arguments
.iter()
.map(|argument| self.scrub_access_arg(argument))
.collect()
}
fn process_text(&self, text: &str, policy: Policy) -> Result<String> {
let collapsed = if policy.base64_limit > 0 {
collapse_base64(text, policy.base64_limit as usize)
} else {
text.to_owned()
};
let redacted = if policy.redact {
self.redact_text(&collapsed)?
} else {
collapsed
};
Ok(
if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize {
truncate_text(&redacted, policy.text_limit as usize)
} else {
redacted
},
)
}
fn scrub_access_arg(&self, value: &str) -> Result<String> {
let length = value.chars().count();
let scanned = if length <= 512 {
value
} else {
let head = &value[..char_offset(value, 512)];
if head.contains('?') {
&head[..head.rfind(['?', '&']).unwrap_or(0)]
} else {
head
}
};
let scrubbed = self.redact_text(scanned)?;
let (path, query) = scrubbed
.split_once('?')
.map_or((scrubbed.as_str(), None), |(path, query)| {
(path, Some(query))
});
let safe = if self.hides_encoded_credential(path)? {
REDACTED.to_owned()
} else if query.is_some() && self.hides_encoded_credential(&scrubbed)? {
format!("{path}?{REDACTED}")
} else {
scrubbed
};
Ok(if length > 512 {
format!(
"{safe}... ({} more chars truncated) ...",
length - scanned.chars().count()
)
} else {
safe
})
}
fn hides_encoded_credential(&self, value: &str) -> Result<bool> {
if !value.as_bytes().contains(&b'%') {
return Ok(false);
}
let decoded = percent_decode_str(value).decode_utf8_lossy();
Ok(self.redact_text(&decoded)? != decoded)
}
}
fn char_offset(text: &str, count: usize) -> usize {
text.char_indices()
.nth(count)
.map_or(text.len(), |(index, _)| index)
}
fn marker(skipped_chars: usize) -> String {
format!(
"... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..."
)
}
fn truncate_text(text: &str, limit: usize) -> String {
let length = text.chars().count();
let kept = limit.saturating_sub(marker(length).len());
if kept == 0 {
return text[..char_offset(text, limit)].to_owned();
}
let head = kept / 2;
let tail = kept - head;
format!(
"{}{}{}",
&text[..char_offset(text, head)],
marker(length - kept),
&text[char_offset(text, length - tail)..]
)
}
fn base64_byte(byte: u8) -> bool {
byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/'
}
fn looks_like_base64(run: &str) -> bool {
let unpadded = run.trim_end_matches('=');
let lower_hex = unpadded
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
let upper_hex = unpadded
.bytes()
.all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]);
(!lower_hex && !upper_hex) || repeated
}
fn base64_size(chars: usize) -> String {
let bytes = chars as f64 * 3.0 / 4.0;
if bytes >= 1024.0 * 1024.0 {
return format!("{:.2}MB", bytes / (1024.0 * 1024.0));
}
if bytes >= 1024.0 {
return format!("{:.1}KB", bytes / 1024.0);
}
format!("{}B", bytes as usize)
}
fn collapse_base64(text: &str, limit: usize) -> String {
let bytes = text.as_bytes();
let mut position = 0;
let mut previous = 0;
let mut output = String::new();
while position < bytes.len() {
if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) {
position += 1;
continue;
}
let start = position;
while position < bytes.len() && base64_byte(bytes[position]) {
position += 1;
}
let run_end = position;
while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' {
position += 1;
}
let run = &text[start..position];
if run_end - start > limit && looks_like_base64(run) {
output.push_str(&text[previous..start]);
output.push_str(&format!(
"[base64_data truncated: {}]",
base64_size(run.len())
));
previous = position;
}
}
output.push_str(&text[previous..]);
output
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() {
let processor = Processor::new(16);
let secret = format!("sk-{}", "q".repeat(48));
let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000));
let input = DiagnosticInput {
message: text,
exception: None,
stack: None,
leaves: vec![],
};
let output = processor
.process_diagnostic(
&input,
Policy {
redact: true,
base64_limit: 0,
text_limit: 500,
},
)
.unwrap();
assert!(output.message.chars().count() <= 500);
assert!(!output.message.contains("sk-qq"));
assert!(output.changed);
}
#[test]
fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() {
let processor = Processor::new(16);
let input = DiagnosticInput {
message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)),
exception: Some(format!("upload failed: {}", "Q".repeat(100))),
stack: Some("api_key=secret123".to_owned()),
leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())],
};
let output = processor
.process_diagnostic(
&input,
Policy {
redact: true,
base64_limit: 20,
text_limit: 0,
},
)
.unwrap();
assert!(output.message.contains("[base64_data truncated: 75B]"));
assert!(output.message.contains(&"a1".repeat(50)));
assert!(
output
.exception
.unwrap()
.contains("[base64_data truncated: 75B]")
);
assert_eq!(output.stack.as_deref(), Some(REDACTED));
assert_eq!(output.leaves, vec![REDACTED]);
}
#[test]
fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() {
let processor = Processor::new(16);
let arguments = vec![
"/v1/models?filter=gpt%2D4o&page=2".to_owned(),
"/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(),
];
assert_eq!(
processor.scrub_access_arguments(&arguments).unwrap(),
vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()]
);
}
}

View file

@ -0,0 +1,140 @@
use fancy_regex::{NoExpand, Regex};
pub const REDACTED: &str = "REDACTED";
#[cfg(test)]
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
fn secret_patterns(minimum_custom_key_length: usize) -> String {
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
[
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
r"\bya29\.[A-Za-z0-9_.~+/-]+",
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r"x-ak-[A-Za-z0-9\-_]{20,}",
r"AIza[0-9A-Za-z\-_]{35}",
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
r"dapi[0-9a-f]{32}",
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
concat!(
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|",
r"signing_key|encryption_key|",
r"auth_token|access_token|refresh_token|",
r"slack_webhook_url|webhook_url|",
r"database_connection_string|",
r"huggingface_token|jwt_secret)",
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
),
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
]
.join("|")
}
#[derive(Clone, Debug)]
pub struct SecretRedactor {
pattern: Regex,
internal_pattern: Regex,
}
impl SecretRedactor {
pub fn new(minimum_custom_key_length: usize) -> Self {
let pattern = Regex::new(&format!(
"(?i){}",
secret_patterns(minimum_custom_key_length)
))
.expect("secret redaction patterns compile");
let internal_pattern = Regex::new(concat!(
r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#,
r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#,
r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|",
r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|",
r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b",
))
.expect("internal detail patterns compile");
Self {
pattern,
internal_pattern,
}
}
pub fn redact(&self, value: &str) -> String {
self.try_redact(value)
.unwrap_or_else(|_| REDACTED.to_owned())
}
pub fn try_redact(&self, value: &str) -> fancy_regex::Result<String> {
self.pattern
.try_replacen(value, 0, NoExpand(REDACTED))
.map(|value| value.into_owned())
}
pub fn try_redact_structured(
&self,
key: Option<&str>,
value: &str,
) -> fancy_regex::Result<String> {
let scrubbed = self.try_redact(value)?;
if scrubbed != value || key.is_none() {
return Ok(scrubbed);
}
let rendered = format!("'{}': '{value}'", key.unwrap_or_default());
Ok(if self.try_redact(&rendered)? != rendered {
REDACTED.to_owned()
} else {
value.to_owned()
})
}
pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result<String> {
let without_traceback = value
.split_once("Traceback (most recent call last):")
.map_or(value, |(prefix, _)| prefix.trim_end());
self.internal_pattern
.try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED))
.map(|value| value.into_owned())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
assert_eq!(
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
expected
);
}
#[test]
fn sk_threshold_follows_the_minimum_custom_key_length() {
let redactor = SecretRedactor::new(8);
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
}
}

View file

@ -0,0 +1,122 @@
use std::sync::{
Arc,
atomic::{AtomicBool, Ordering},
mpsc,
};
use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn};
use serde_json::{Value, json};
struct Output {
enabled: Arc<AtomicBool>,
sender: mpsc::Sender<(String, Value, Level, &'static str, Option<u32>)>,
}
impl Sink for Output {
fn enabled(&self, _: &Metadata<'_>) -> bool {
self.enabled.load(Ordering::Relaxed)
}
fn emit(&self, record: &Record) {
self.sender
.send((
record.message.clone(),
Value::Object(record.fields.clone()),
*record.metadata.level(),
record.metadata.target(),
record.metadata.line(),
))
.unwrap();
Logger::default().scope(|| warn!("a sink must not recursively emit"));
}
}
fn emit() {
warn!(
attempt = 3_u64,
elapsed = 1.5,
retry = true,
reason = "timeout",
"retry {}",
3
);
}
#[test]
fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() {
let (sender, receiver) = mpsc::channel();
let enabled = Arc::new(AtomicBool::new(false));
let logger = Logger::new(Output {
enabled: enabled.clone(),
sender,
});
logger.scope(emit);
assert!(receiver.try_recv().is_err());
enabled.store(true, Ordering::Relaxed);
logger.scope(emit);
let (message, fields, level, target, line) = receiver.try_recv().unwrap();
assert_eq!(message, "retry 3");
assert_eq!(
fields,
json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"})
);
assert_eq!(level, Level::WARN);
assert_eq!(target, module_path!());
assert!(line.is_some());
enabled.store(false, Ordering::Relaxed);
logger.scope(emit);
assert!(receiver.try_recv().is_err());
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() {
let tasks = (0..2)
.map(|id| {
let (sender, receiver) = mpsc::channel();
let logger = Logger::new(Output {
enabled: Arc::new(AtomicBool::new(true)),
sender,
});
let task = tokio::spawn(logger.instrument(async move {
tokio::task::yield_now().await;
info!(id, "worker");
}));
(id, task, receiver)
})
.collect::<Vec<_>>();
for (id, task, receiver) in tasks {
task.await.unwrap();
let (message, fields, level, _, _) = receiver.try_recv().unwrap();
assert_eq!(message, "worker");
assert_eq!(fields, json!({"id": id}));
assert_eq!(level, Level::INFO);
assert!(receiver.try_recv().is_err());
}
}
#[test]
fn nested_scopes_restore_the_previous_sink() {
let (outer_sender, outer) = mpsc::channel();
let (inner_sender, inner) = mpsc::channel();
let logger = |sender| {
Logger::new(Output {
enabled: Arc::new(AtomicBool::new(true)),
sender,
})
};
let outside = logger(outer_sender);
let inside = logger(inner_sender);
outside.scope(|| {
info!("before");
inside.scope(|| info!("inside"));
info!("after");
});
assert_eq!(
outer.try_iter().map(|event| event.0).collect::<Vec<_>>(),
["before", "after"]
);
assert_eq!(
inner.try_iter().map(|event| event.0).collect::<Vec<_>>(),
["inside"]
);
}

View file

@ -1,11 +1,12 @@
import ast
import contextvars
import functools
import itertools
import logging
import os
import re
import sys
from collections.abc import Sequence
from collections.abc import Iterator
from datetime import datetime
from logging import Formatter
from typing import Any, Final, TextIO
@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.litellm_core_utils.secret_redaction import (
_python_redact_string,
_python_redact_structured_value,
redact_internal_details,
redact_string,
redact_structured_value,
)
from litellm.rust_bridge import diagnostics
set_verbose = False
@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str:
pass through credential redaction.
"""
stripped: Final = "".join(ch for ch in value if ch.isprintable())
return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
def set_session_id(session_id: str) -> "contextvars.Token[str]":
@ -74,12 +77,6 @@ def _redact_string(value: str) -> str:
return redact_string(value)
def _redact_structured_value(key: str | None, value: str) -> str:
if not _ENABLE_SECRET_REDACTION:
return value
return redact_structured_value(key, value)
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
_REDACTED_STAMP: Final = object()
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
@ -103,14 +100,6 @@ def _plain_text(value: object) -> str:
return UNSERIALIZABLE_OBJECT
def _redact_extra_value(key: str, value: object) -> object:
try:
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
except Exception:
return _redact_string(_plain_text(value))
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
def redact_secrets(value: str) -> str:
"""Public API: redact known secret/credential patterns from an arbitrary string.
@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None:
return None
try:
return color_message % record.args
except TypeError:
except Exception:
return color_message
@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
return True
# Runs before args are cleared, and before the extra-field loop below
# that redacts the substituted result.
substituted_color_message: Final = _substituted_color_message(record)
if substituted_color_message is not None:
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
try:
record.msg = _redact_string(record.getMessage())
record.args = None
except Exception:
if isinstance(record.msg, str):
record.msg = _redact_string(record.msg)
# Redact exception tracebacks
if record.exc_info and record.exc_info[1] is not None:
try:
record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
except Exception:
pass
if isinstance(record.stack_info, str):
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
# Redact extra fields passed via logger.debug("msg", extra={...})
record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items())
for key, value in record_items:
if key in _STANDARD_RECORD_ATTRS:
continue
if isinstance(value, str):
setattr(record, key, _redact_structured_value(key, value))
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
setattr(record, key, _redact_extra_value(key, value))
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
return True
return _process_record(record, base64_limit=0, text_limit=0, redact=True)
_secret_filter: Final = SecretRedactionFilter()
@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
def _hides_a_credential(value: str) -> bool:
"""Whether *value* only looks clean until it is percent-decoded."""
decoded: Final = unquote(value)
return _redact_string(decoded) != decoded
return _python_redact_string(decoded) != decoded
def _drop_encoded_credential(scrubbed: str) -> str:
@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str:
pattern and would then be logged raw.
"""
if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
return _drop_encoded_credential(_redact_string(value))
return _drop_encoded_credential(_python_redact_string(value))
head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
scrubbed: Final = _drop_encoded_credential(_redact_string(kept))
scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter):
if not _ENABLE_SECRET_REDACTION:
return True
if isinstance(record.args, tuple) and record.args:
strings: Final = tuple(arg for arg in record.args if isinstance(arg, str))
candidate: Final = diagnostics.run(
lambda native: native.scrub_access_arguments(strings),
lambda: tuple(_scrub_access_arg(arg) for arg in strings),
)
scrubbed: Final = (
candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings)
)
values: Final = iter(scrubbed)
record.args = tuple( # rebind-ok: a Filter scrubs records in place
_scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args
next(values) if isinstance(arg, str) else arg for arg in record.args
)
return True
# No positional args means everything is in msg, where collapsing is correct.
@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str:
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
def _extra_structure(key: str, value: object) -> object:
if isinstance(value, str):
return value
try:
return safe_json_structure(value, key=key)
except Exception:
return _plain_text(value)
def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]:
if isinstance(value, str):
yield key, value
elif isinstance(value, dict):
yield from itertools.chain.from_iterable(
_string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str)
)
elif isinstance(value, (list, tuple)):
yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value)
def _replace_string_leaves(value: object, values: Iterator[str]) -> object:
if isinstance(value, str):
return next(values)
if isinstance(value, dict):
return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers
key: _replace_string_leaves(child, values) for key, child in value.items()
}
if isinstance(value, list):
return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers
_replace_string_leaves(child, values) for child in value
]
if isinstance(value, tuple):
return tuple(_replace_string_leaves(child, values) for child in value)
return value
def _sort_processed_sets(original: object, processed: object) -> object:
if isinstance(original, set) and isinstance(processed, list):
return sorted(processed)
if isinstance(original, dict) and isinstance(processed, dict):
return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict
key: _sort_processed_sets(original.get(key), value) for key, value in processed.items()
}
if isinstance(original, list) and isinstance(processed, list):
return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list
_sort_processed_sets(before, after) for before, after in zip(original, processed)
]
if isinstance(original, tuple) and isinstance(processed, tuple):
return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed))
return processed
def _python_process_diagnostic(
message: str,
exception: str | None,
stack: str | None,
leaves: tuple[tuple[str | None, str], ...],
redact: bool,
base64_limit: int,
text_limit: int,
) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
def process_text(text: str) -> str:
collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
processed_message: Final = process_text(message)
processed_exception: Final = process_text(exception) if exception is not None else None
processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
processed_leaves: Final = tuple(
_python_redact_structured_value(key, text) if redact else text for key, text in leaves
)
changed: Final = (
processed_message != message
or processed_exception != exception
or processed_stack != stack
or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves))
)
return processed_message, processed_exception, processed_stack, processed_leaves, changed
def _render_message(record: logging.LogRecord) -> str:
try:
return record.getMessage()
except Exception:
return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT
def _render_exception(record: logging.LogRecord) -> str | None:
if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None:
return None
try:
return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info)
except Exception:
return "REDACTED"
def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool:
if _is_redacted(record):
return True
message: Final = _render_message(record)
exception: Final = _render_exception(record)
stack: Final = record.stack_info if isinstance(record.stack_info, str) else None
substituted_color: Final = _substituted_color_message(record)
extras: Final = (
tuple(
(
key,
value,
_extra_structure(
key, substituted_color if key == "color_message" and substituted_color is not None else value
),
)
for key, value in record.__dict__.items()
if key not in _STANDARD_RECORD_ATTRS
and key != _REDACTED_RECORD_ATTR
and not isinstance(value, _UNREDACTED_SCALAR_TYPES)
)
if redact
else ()
)
extra_leaves: Final = tuple(
itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras)
)
raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None
color_template: Final = record.__dict__.get("color_message")
raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None
leaves: Final = (
extra_leaves
+ (((None, raw_template),) if raw_template is not None else ())
+ (((None, raw_color),) if raw_color is not None else ())
)
candidate: Final = diagnostics.run(
lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)),
lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit),
)
processed_message, processed_exception, processed_stack, processed_leaves, _ = (
candidate
if len(candidate[3]) == len(leaves)
else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit)
)
raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template
safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message
if redact or safe_message != message:
record.msg = safe_message # rebind-ok: the Filter interface mutates the record
record.args = None # rebind-ok: the rendered message replaces interpolation inputs
if processed_exception is not None:
record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record
if processed_stack is not None:
record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record
processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
for key, original, prepared in extras:
replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
if not _scrubbing_changed_nothing(replacement, original):
setattr(record, key, replacement)
raw_color_changed: Final = (
raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color
)
if raw_color_changed and getattr(record, "color_message", None) == substituted_color:
setattr(record, "color_message", "REDACTED")
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
return True
def _redact_json_record(value: object) -> object:
prepared: Final = safe_json_structure(value)
leaves: Final = tuple(_string_leaves(None, prepared))
candidate: Final = diagnostics.run(
lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
)
replacements: Final = (
candidate
if len(candidate) == len(leaves)
else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
)
return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
class StdoutLogTruncationFilter(logging.Filter):
"""Bounds how much of an oversized log line reaches stdout.
@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter):
return True
_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
class DiagnosticProcessingFilter(StdoutLogTruncationFilter):
def filter(self, record: logging.LogRecord) -> bool:
return _process_record(
record,
base64_limit=_get_max_base64_length_stdout_log(),
text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0,
redact=_ENABLE_SECRET_REDACTION,
)
_diagnostic_filter: Final = DiagnosticProcessingFilter()
class CorrelationContextFilter(logging.Filter):
@ -633,7 +785,9 @@ class JsonFormatter(Formatter):
if record.exc_info:
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
return safe_dumps(
json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record)
)
class CorrelationPlainFormatter(logging.Formatter):
@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter):
# Create a handler with JSON formatting for exceptions
error_handler: Final = logging.StreamHandler()
error_handler.setFormatter(formatter)
error_handler.addFilter(_stdout_truncation_filter)
error_handler.addFilter(_diagnostic_filter)
error_handler.addFilter(_secret_filter)
error_handler.addFilter(_correlation_filter)
@ -734,10 +888,10 @@ verbose_logger.addHandler(handler)
# Filters attached to the logger, not the handler, survive callers swapping in their own
# handlers (JSON mode, uvicorn log config, a host app's root handler).
verbose_router_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
verbose_logger.addFilter(_stdout_truncation_filter)
verbose_router_logger.addFilter(_diagnostic_filter)
verbose_proxy_logger.addFilter(_diagnostic_filter)
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
verbose_logger.addFilter(_diagnostic_filter)
def _suppress_loggers():

View file

@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
provider_first_model_name_with_quality: Final = (
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
if quality and custom_llm_provider
else None
)
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
@ -2386,6 +2391,7 @@ def default_image_cost_calculator(
models_to_check: Final = (
model_name_with_quality,
provider_first_model_name_with_quality,
base_model_name,
model_name_with_v2_quality,
model_with_quality_without_provider,

View file

@ -10,6 +10,7 @@ import re
from typing import Final
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
from litellm.rust_bridge import diagnostics
REDACTED: Final = "REDACTED"
@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]":
_SECRET_RE: Final = _build_secret_patterns()
def _python_redact_string(value: str) -> str:
return _SECRET_RE.sub(REDACTED, value)
def redact_string(value: str) -> str:
"""Scrub known secret/credential patterns from *value* and return the result."""
return _SECRET_RE.sub(REDACTED, value)
return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value))
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile(
_TRACEBACK_MARKER: Final = "Traceback (most recent call last):"
def redact_internal_details(value: str) -> str:
def _python_redact_internal_details(value: str) -> str:
"""Drop an embedded traceback and scrub filesystem paths and internal hostnames,
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
marker_index: Final = value.find(_TRACEBACK_MARKER)
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback))
def redact_structured_value(key: str | None, value: str) -> str:
def redact_internal_details(value: str) -> str:
return diagnostics.run(
lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value)
)
def _python_redact_structured_value(key: str | None, value: str) -> str:
"""Scrub *value* as it appeared under *key* inside a structured record.
redact_string() replaces a whole ``key: value`` span with REDACTED, which is
@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str:
repr would, so the key-name patterns still fire, but collapses only the value
so the caller's structure survives.
"""
scrubbed: Final = redact_string(value)
scrubbed: Final = _python_redact_string(value)
if scrubbed != value or key is None:
return scrubbed
rendered: Final = f"'{key}': '{value}'"
return REDACTED if redact_string(rendered) != rendered else value
return REDACTED if _python_redact_string(rendered) != rendered else value
def redact_structured_value(key: str | None, value: str) -> str:
return diagnostics.run(
lambda native: native.redact_structured_text(key, value),
lambda: _python_redact_structured_value(key, value),
)

View file

@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
"""
return self._is_deepseek_r1_model(model=model, base_model=base_model)
@classmethod
def _supports_sampling_params(cls, model: str) -> bool:
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
base_model: Final = BedrockModelInfo.get_base_model(model)
if base_model.startswith("anthropic"):
return True
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
for candidate in candidates:
if (
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
candidate, "supports_sampling_params"
)
) is not None:
return flag
return True
def get_supported_openai_params(self, model: str) -> list[str]:
from litellm.utils import supports_function_calling
supports_sampling: Final = self._supports_sampling_params(model)
supported_params: Final = [
"max_tokens",
"max_completion_tokens",
"stream",
"stream_options",
"stop",
"temperature",
"top_p",
*(("temperature", "top_p") if supports_sampling else ()),
"extra_headers",
"response_format",
"requestMetadata",
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
value = [value]
optional_params["stopSequences"] = value
if param == "temperature" or param == "top_p":
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key="topP" if param == "top_p" else param,
)
if base_model.startswith("anthropic"):
AnthropicConfig._apply_sampling_param(
optional_params=optional_params,
model=model,
param=param,
value=value,
drop_params=drop_params,
output_key="topP" if param == "top_p" else param,
)
elif not self._supports_sampling_params(model):
if not (litellm.drop_params or drop_params):
raise litellm.utils.UnsupportedParamsError(
message=(
f"{model} does not support {param}={value}. "
"To drop unsupported params, set `litellm.drop_params = True`."
),
status_code=400,
)
else:
optional_params["topP" if param == "top_p" else param] = value
if param == "tools" and isinstance(value, list):
self._apply_tool_call_transformation(
tools=cast(list[OpenAIChatCompletionToolParam], value),

View file

@ -231,6 +231,12 @@ async def make_call(
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=False,
)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
@ -329,6 +335,12 @@ def make_sync_call(
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
elif bedrock_invoke_provider == "moonshot":
decoder = AmazonOpenAICompatibleStreamDecoder(
model=model,
sync_stream=True,
)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
else:
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
@ -795,6 +807,24 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
def __init__(
self,
model: str,
sync_stream: bool,
) -> None:
super().__init__(model=model)
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
streaming_response=None,
sync_stream=sync_stream,
)
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
class MockResponseIterator: # for returning ai21 streaming responses
def __init__(self, model_response, json_mode: bool | None = False):
self.model_response = model_response

View file

@ -770,9 +770,7 @@ class AmazonAnthropicClaudeMessagesConfig(
aws_decoder: Final = AmazonAnthropicClaudeMessagesStreamDecoder(
model=model,
)
completion_stream: Final = aws_decoder.aiter_bytes(
httpx_response.aiter_bytes(chunk_size=aws_decoder.DEFAULT_CHUNK_SIZE)
)
completion_stream: Final = aws_decoder.aiter_bytes(httpx_response.aiter_bytes())
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
return self.bedrock_sse_wrapper(
completion_stream=completion_stream,
@ -919,16 +917,6 @@ class AmazonAnthropicClaudeMessagesConfig(
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
def __init__(
self,
model: str,
) -> None:
"""
Iterator to return Bedrock invoke response in anthropic /messages format
"""
super().__init__(model=model)
self.DEFAULT_CHUNK_SIZE = 1024
def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
"""
Parse the chunk data into anthropic /messages format

View file

@ -287,7 +287,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
Check if the model is Gemini 3 or newer.
"""
model_name = model.split("/")[-1].lower()
if not model_name:
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
model.startswith("gemini/") and not model_name.startswith("gemini-")
)
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
return False
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):

File diff suppressed because it is too large Load diff

View file

@ -1004,6 +1004,8 @@ def _stamp_deployment_attribution(
return attribution
metadata.setdefault("model_info", attribution["model_info"])
metadata.setdefault("deployment", attribution["deployment"])
if isinstance(model_group, str):
metadata.setdefault("model_group", model_group)
return attribution

View file

@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ...
class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
class ProcessReservedForForking(RuntimeError): ...
@final
class NativeDiagnosticProcessor:
def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
def redact_text(self, text: str) -> str: ...
def redact_structured_text(self, key: str | None, text: str) -> str: ...
def redact_client_message(self, text: str) -> str: ...
def process_diagnostic(
self,
message: str,
exception: str | None,
stack: str | None,
leaves: Sequence[tuple[str | None, str]],
policy: tuple[bool, int, int],
) -> tuple[str, str | None, str | None, list[str], bool]: ...
def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ...
def ocr(
request: LiteLLMOcrRequest,
args: tuple[object, ...],
@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ...
__all__ = [
"ForkedAfterNativeRuntimeStarted",
"HuggingFaceEncoding",
"NativeDiagnosticProcessor",
"ProcessReservedForForking",
"ResponsesWebSocketConnection",
"RustBridgeDeclined",

View file

@ -86,11 +86,25 @@ class SecretManagerRule:
return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems)
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule
@dataclass(frozen=True, slots=True)
class LoggerContext:
pass
@dataclass(frozen=True, slots=True)
class LoggerRule:
rollout: Rollout
def matches(self, context: Context) -> bool:
return isinstance(context, LoggerContext)
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule
Rules: TypeAlias = tuple[Rule, ...]
RULES: Final[Rules] = (
LoggerRule(Rollout.RUST_OPT_IN),
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),

View file

@ -0,0 +1,59 @@
from __future__ import annotations
from collections.abc import Callable, Sequence
from functools import lru_cache
from typing import Final, Protocol, TypeVar, cast
from litellm.rust_bridge.bindings import NativeBinding
ResultT: Final = TypeVar("ResultT")
class NativeDiagnosticProcessor(Protocol):
def redact_text(self, text: str) -> str: ...
def redact_structured_text(self, key: str | None, text: str) -> str: ...
def redact_client_message(self, text: str) -> str: ...
def process_diagnostic(
self,
message: str,
exception: str | None,
stack: str | None,
leaves: tuple[tuple[str | None, str], ...],
policy: tuple[bool, int, int],
) -> tuple[str, str | None, str | None, Sequence[str], bool]: ...
def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ...
class NativeDiagnosticFactory(Protocol):
def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
def _as_factory(value: object) -> NativeDiagnosticFactory | None:
if not isinstance(value, type):
return None
return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type
PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory)
@lru_cache(maxsize=4)
def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor:
return factory(minimum_custom_key_length)
def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT:
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
from litellm.rust_bridge.catalog import LoggerContext, decision
from litellm.rust_bridge.configuration import Decision
selected: Final = decision(LoggerContext())
if selected is Decision.PYTHON:
return python()
factory: Final = PROCESSOR.load()
if factory is None:
return python()
try:
return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH))
except Exception:
return python()

View file

@ -0,0 +1,63 @@
from __future__ import annotations
from collections.abc import Mapping
from typing import Final
from pydantic import JsonValue
from litellm._logging import (
CorrelationContextFilter,
DiagnosticProcessingFilter,
session_id_var,
set_session_id,
set_trace_id,
trace_id_var,
verbose_logger,
)
_REDACTION: Final = DiagnosticProcessingFilter()
_CORRELATION: Final = CorrelationContextFilter()
def context() -> tuple[str, str]:
return session_id_var.get(), trace_id_var.get()
def enabled(level: int) -> bool:
return verbose_logger.isEnabledFor(level)
def emit(
level: int,
message: str,
pathname: str,
lineno: int,
target: str,
fields: Mapping[str, JsonValue],
correlation: tuple[str, str],
) -> None:
if not enabled(level):
return
session_token: Final = set_session_id(correlation[0])
trace_token: Final = set_trace_id(correlation[1])
try:
record: Final = verbose_logger.makeRecord(
verbose_logger.name,
level,
pathname,
lineno,
message,
(),
None,
func=target,
extra={
"rust_target": target,
"rust_fields": dict(fields),
}, # mutable-ok: LogRecord requires JSON dict extras
)
_REDACTION.filter(record)
_CORRELATION.filter(record)
verbose_logger.handle(record)
finally:
trace_id_var.reset(trace_token)
session_id_var.reset(session_token)

View file

@ -58,12 +58,6 @@ class SecretManagerBinding:
settings_object: object
def warn(message: str) -> None:
from litellm._logging import verbose_logger
verbose_logger.warning("%s", message)
def secret_manager() -> SecretManager:
from litellm.secret_managers.main import (
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private

View file

@ -1,5 +1,5 @@
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
FROM $UV_IMAGE AS uvbin

File diff suppressed because it is too large Load diff

View file

@ -1,6 +1,6 @@
[project]
name = "litellm"
version = "1.103.0"
version = "1.104.0"
description = "Library to easily interface with LLM API providers"
readme = "README.md"
requires-python = ">=3.10, <3.15"
@ -75,8 +75,8 @@ proxy = [
"mcp>=2.2.0,<3",
"httpx2>=2.5.0,<3",
"pydantic>=2.12.0,<3",
"litellm-proxy-extras==0.4.100",
"litellm-enterprise==0.1.69",
"litellm-proxy-extras==0.4.101",
"litellm-enterprise==0.1.70",
"RestrictedPython>=8.5,<9.0",
"rich>=13.9.4,<14.0",
"InquirerPy>=0.3.4,<1.0",
@ -355,7 +355,7 @@ litellm-enterprise = { workspace = true }
members = ["enterprise", "litellm-proxy-extras"]
[tool.commitizen]
version = "1.103.0"
version = "1.104.0"
version_files = [
"pyproject.toml:^version",
]

View file

@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
"_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible).
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
]

View file

@ -0,0 +1,41 @@
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
ARG GH_VERSION=2.101.0
ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8
ARG UV_VERSION=0.10.9
ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594
ARG CLAUDE_CODE_VERSION=2.1.228
ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
RUN apt-get update \
&& apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \
&& rm -rf /var/lib/apt/lists/*
RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \
&& echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \
&& tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \
&& rm /tmp/gh.tar.gz
RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \
&& echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \
&& tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \
&& rm /tmp/uv.tar.gz
RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \
&& echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \
&& install -m 0755 /tmp/claude /usr/local/bin/claude \
&& rm /tmp/claude
RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator
ENV HOME=/home/populator \
LITELLM_REPO=/opt/litellm \
DISABLE_AUTOUPDATER=1
COPY --chown=populator:populator . /opt/litellm/tests/e2e/
USER populator
WORKDIR /home/populator
CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"]

View file

@ -1,66 +1,59 @@
# Cron VM setup for the Claude Code compatibility-matrix populator
# Render cron job for the Claude Code compatibility-matrix populator
The populator runs daily on a dedicated GCP VM
(`litellm-compatibility-matrix-populator`) rather than as a GitHub
Action. Trade-offs:
The populator runs daily as the Render cron job `litellm-compat-matrix`
(Docker runtime, built from the `Dockerfile` in this directory) rather
than as a GitHub Action or on a dedicated VM. Trade-offs:
- ✅ Real VM means we can `gh auth login` against an account that's
already a collaborator on `BerriAI/litellm-docs`, instead of
provisioning a GitHub App with `pull-requests: write`.
- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`)
is reused across runs, so each daily run does a fast `git checkout` +
incremental `uv sync` rather than a fresh clone + cold sync.
- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`.
- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers
from short outages, but a multi-day outage means the matrix goes
stale until the VM is back.
- ⚠️ Provider credentials live on the VM filesystem
(`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat
the VM as an environment with comparable blast radius to a CI runner.
This directory used to live at `tests/claude_code/cron_vm/` (paired with
the standalone `tests/claude_code/` suite); it now runs the maintained
`tests/e2e/claude_code/` suite instead. The pytest env interface changed
accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY`
(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure
column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and
`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`.
- ✅ No machine to keep on or patch. Render builds the image from this
directory on every push to `main` that touches `tests/e2e/**` and
runs it on the schedule.
- ✅ Credentials live in Render env vars and secret files, scoped to
this one service, instead of on a VM filesystem.
- ✅ The publish token still uses the `mateo-berri` account, which is a
collaborator on `BerriAI/litellm-docs`, so no GitHub App with
`pull-requests: write` has to be provisioned.
- ⚠️ The disk is ephemeral, so every run starts from a fresh (blobless)
clone of litellm plus a cold `uv sync`. That adds a few minutes on
top of the ~10 minute test run; the job's 12 hour ceiling is nowhere
near.
- ⚠️ The Claude Code CLI version under test is pinned in the
`Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a
PR, see the gotchas below.
## Layout
| File | Purpose |
| --- | --- |
| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). |
| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
| `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. |
| `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. |
| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. |
| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. |
| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. |
| `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. |
## What `run_daily.sh` does
1. **Resolves the latest LiteLLM final release tag** (newest bare
`vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the
GitHub Releases API (`curl | jq`).
2. **Reads the local Claude Code CLI version** via `claude --version`.
The cron does not auto-upgrade the CLI — operators do that
out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`.
3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`:
`git fetch --tags --force`, `git reset --hard`,
`git clean -fdx -e .venv -e .uv-bin`, `git checkout --force <tag>`.
The `.venv` is preserved across runs so `uv sync --frozen` is
incremental. Then **shims the test suite**: `tests/e2e/` in the
worktree is rebuilt from the dev checkout — the `claude_code/` suite
plus the five shared transport helpers it imports (`proxy_client.py`,
`e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the
cron always runs *today's* tests against the latest stable proxy. The
tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`,
whose imports the stable venv doesn't install) is deliberately not
used.
2. **Reads the Claude Code CLI version** via `claude --version`. That
is whatever the `Dockerfile` pins; the job never upgrades it on its
own.
3. **Clones the worktree** at `~/litellm-cron-worktree/` (a
`--filter=blob:none` clone, so only the checked-out tag's blobs are
fetched), `git checkout --force <tag>`, then `uv sync --frozen
--no-install-project` against a uv-managed CPython 3.12 followed by
`uv pip install --no-build litellm==<version>`, so the proxy under
test is the published PyPI wheel (what users install) rather than a
source build: the tag builds a Rust extension through maturin, and
the image ships no C or Rust toolchain. Then **shims the test suite**:
`tests/e2e/` in the worktree is replaced by the image's copy of this
whole tree, so the cron always runs *today's* tests against the
latest stable proxy, and pytest runs with `--confcutdir` pointed at
`claude_code/` so the tree's EKS-harness `conftest.py` (whose imports
the stable venv doesn't install) is never loaded. The tag's own
`tests/e2e/` is deliberately not used.
4. **Boots the proxy** as a `setsid` background process on port `4100`
(so it can't collide with a developer's `:4000`), then polls
`/health/liveliness` until it's up.
bound to loopback, then polls `/health/liveliness` until it's up.
5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL`
pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest
hook writes the per-test results artifact. Test failures become
@ -74,8 +67,9 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
`mateo-berri` token has write access, so this is a same-repo branch,
not a fork), `gh pr create`. A re-run on the same day fast-forwards
the existing branch and `gh pr create` no-ops ("a pull request for
branch ... already exists" is treated as success). These PRs are no
longer gated on a second human review.
branch ... already exists" is treated as success). If the JSON is
byte-identical to what `main` already publishes, the push is skipped
entirely. These PRs are not gated on a second human review.
8. **Gates auto-merge on a regression check**: before enabling
auto-merge, `check_regressions.py` diffs the new matrix against the
one currently on `main`. Auto-merge (`gh pr merge --auto --squash`)
@ -89,107 +83,155 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
human reviews before it lands on the public table. The check fails
*closed*: if it errors, auto-merge is withheld.
9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every
other open `compat-matrix/*` PR on the docs repo is closed (and its
bot-owned branch deleted), so at most one compat-matrix PR is ever
open — the newest.
other open `compat-matrix/*` PR that the publishing account opened
from a branch on the docs repo itself is closed (and its bot-owned
branch deleted), so at most one compat-matrix PR is ever open — the
newest. A contributor's PR under that prefix is never touched.
## One-time VM setup
## The Render service
Run as `mateo` on the cron VM:
Everything below is what the live service is set to; recreate it with
the same values if it ever has to be rebuilt.
| Setting | Value |
| --- | --- |
| Workspace | Litellm (the one that already builds the other litellm services) |
| Type | Cron job, Docker runtime |
| Repo / branch | `BerriAI/litellm` @ `main` |
| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` |
| Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) |
| Build filter | included paths `tests/e2e/**` |
| Schedule | `0 6 * * *` (06:00 UTC daily) |
| Plan / region | `4c-16g` (4 CPU, 16 GB, what the dashboard calls Pro Max; the suite fans out to ~75 concurrent CLI calls) / Oregon |
| Env vars | every key in `litellm-compat-matrix.env.example` |
| Secret files | `github-token` (the publish PAT, one line) and `vertex-service-account.json` (the Vertex service-account key) |
Render mounts secret files at `/etc/secrets/<name>`, which is where
`CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env
example point. Render also passes env vars to `docker build` as build
args, which is why the `Dockerfile` declares no `ARG` that could ever
be given a secret's name.
Creating it through the API looks like this (fill `envVars` and
`secretFiles` from the env example and the two secrets; `ownerId` is
the workspace id from `GET /v1/owners`):
```bash
# 1. Toolchain
sudo apt-get update
sudo apt-get install -y git nodejs npm jq curl
curl -LsSf https://astral.sh/uv/install.sh | sh
sudo apt-get install -y gh # or follow https://cli.github.com/
# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this
# line out-of-band when you want a fresh CLI to be tested)
sudo npm install -g @anthropic-ai/claude-code@latest
# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the
# source of the .service / .timer files. The cron itself runs out
# of the separate worktree at ~/litellm-cron-worktree/.
mkdir -p ~/litellm
git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm
git -C ~/litellm/litellm checkout litellm_internal_staging
# 4. gh auth — must be a collaborator on BerriAI/litellm-docs.
gh auth login # follow prompts; pick HTTPS + token paste flow
# 5. Provider credentials + the publish token.
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \
/etc/litellm-compat-matrix.env
sudoedit /etc/litellm-compat-matrix.env # fill in real values
sudo chmod 0600 /etc/litellm-compat-matrix.env
# The mateo-berri PAT lives in its own file, mapped into the service via
# systemd LoadCredential so it stays out of the test processes' env
# (see the env.example comment for why).
sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token
sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT
# 6. systemd units.
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now litellm-compat-matrix.timer
curl -fsS https://api.render.com/v1/services \
-H "Authorization: Bearer ${RENDER_API_KEY}" \
-H 'Content-Type: application/json' \
-d '{
"type": "cron_job",
"name": "litellm-compat-matrix",
"ownerId": "<workspace id>",
"repo": "https://github.com/BerriAI/litellm",
"branch": "main",
"autoDeploy": "yes",
"buildFilter": {"paths": ["tests/e2e/**"], "ignoredPaths": []},
"envVars": [{"key": "ANTHROPIC_API_KEY", "value": "..."}],
"secretFiles": [{"name": "github-token", "content": "..."},
{"name": "vertex-service-account.json", "content": "..."}],
"serviceDetails": {
"runtime": "docker",
"schedule": "0 6 * * *",
"plan": "4c-16g",
"region": "oregon",
"envSpecificDetails": {
"dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile",
"dockerContext": "tests/e2e"
}
}
}'
```
## Operating it
```bash
# When does it run next?
systemctl list-timers litellm-compat-matrix.timer
# Trigger a real run right now (PRs to litellm-docs). The id is the
# service id (`crn-...`) from the dashboard URL or `GET /v1/services`.
curl -fsS -X POST "https://api.render.com/v1/cron-jobs/${CRON_ID}/runs" \
-H "Authorization: Bearer ${RENDER_API_KEY}"
# Trigger a real run right now (PRs to litellm-docs).
sudo systemctl start litellm-compat-matrix.service
# Follow a run: the Logs tab on the service, or the API.
curl -fsS "https://api.render.com/v1/logs?ownerId=${OWNER_ID}&resource=${CRON_ID}&limit=100" \
-H "Authorization: Bearer ${RENDER_API_KEY}"
# Trigger a run that does NOT open a PR (good for first-time validation).
SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# Rebuild the image after a merge that touches tests/e2e/** (see the
# auto-deploy gotcha below). The deploy is done once its status is
# `live`; a run triggered before that still uses the previous image.
curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \
-H "Authorization: Bearer ${RENDER_API_KEY}" \
-H 'Content-Type: application/json' -d '{"clearCache": "do_not_clear"}'
curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \
-H "Authorization: Bearer ${RENDER_API_KEY}"
# Narrow to one cell while debugging.
SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \
~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# A run that does NOT open a PR (first-time validation, CLI bumps):
# set SKIP_PUBLISH=1 on the service, trigger a run, then remove it.
# The matrix JSON is printed at the end of the run's log (nothing on
# the container's disk outlives the run) and saved to
# ~/compatibility-matrix.json for a local docker run.
# PYTEST_K='basic_messaging_non_streaming and anthropic' narrows the
# run to one cell the same way.
# Watch the most recent run.
journalctl -u litellm-compat-matrix.service -f
# Read older runs.
journalctl -u litellm-compat-matrix.service --since '2 days ago'
# Disable until further notice (e.g. while debugging).
sudo systemctl disable --now litellm-compat-matrix.timer
# Build and run the image locally (docker on Apple silicon needs the
# platform flag; the context is tests/e2e, see the table above).
docker build --platform linux/amd64 \
-f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e
docker run --rm --platform linux/amd64 \
--env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \
-v "$PWD/secrets:/etc/secrets:ro" compat-matrix
```
## Gotchas
- **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The
e2e suite uses PEP 695 `type` aliases, which the VM's system Python
(3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython
e2e suite uses PEP 695 `type` aliases, which the image's Debian
Python can't parse; `run_daily.sh` has uv fetch a managed CPython
into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against
it. The first run after a version bump is a cold venv rebuild.
- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd
into the same VM with their own `:4000` proxy doesn't collide with a
cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env`
if you need to.
it.
- **The proxy port is `4100`, not `4000`.** Kept from the VM days so a
developer running the script locally next to their own `:4000` proxy
doesn't collide. Override with `PROXY_PORT=...`.
- **`uv sync --frozen` requires the resolved tag to be tagged on
GitHub.** If the latest stable release was made but not pushed as a
git tag, the `git checkout` step fails. Push the tag, then rerun.
GitHub, and the wheel install requires it on PyPI.** If the latest
stable release was made but not pushed as a git tag, the `git
checkout` step fails; push the tag, then rerun. PyPI has had every
stable version days before its GitHub release so far (1.102.0 was
uploaded 2026-09-20, released on GitHub 2026-09-22), so the
`--no-build` install failing means the wheel is genuinely missing,
not late.
- **Pushes do not redeploy the service; deploy by hand.** `autoDeploy`
is `yes` on the service, but Render only hears about pushes through
its GitHub app, which is not installed on the `BerriAI` org (an org
admin step), so no push to the branch has ever started a deploy.
After a merge that changes anything under `tests/e2e/**`, run the
deploy command from the operating section (or "Manual Deploy" on the
dashboard) and wait for `live` before triggering a run, otherwise
the next scheduled run still executes the old image.
- **Publish-token rotation is your problem.** The cron does not
refresh the token; if `mateo-berri`'s PAT in
`/etc/litellm-compat-matrix-github-token` expires, the run fails at
the `git push`/`gh pr create` step with a 401 ("Bad credentials" /
"Authentication failed"). Mint a fresh PAT and update that file.
The token needs write access to `BerriAI/litellm-docs` (classic
`repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is
delivered via systemd `LoadCredential`, not the env file, so pytest,
the proxy, and the claude CLI never inherit it; manual runs export
`GITHUB_TOKEN` instead.
- **First run after upgrading the Claude Code CLI is the riskiest one.**
If the new CLI changes its wire format the matrix run can produce
systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI
upgrade before letting the next scheduled fire happen.
- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory
is ~1 GB. Plan for at least 5 GB free on the VM, otherwise
`uv sync` will fail mid-run and leave you with a half-installed venv.
refresh the token; if `mateo-berri`'s PAT in the `github-token`
secret file expires, the run fails at the `git push`/`gh pr create`
step with a 401 ("Bad credentials" / "Authentication failed"). Mint
a fresh PAT and replace the secret file on the service. The token
needs write access to `BerriAI/litellm-docs` (classic `repo` scope,
or fine-grained Contents:RW + Pull requests:RW). It is delivered as
a file, not an env var, so pytest, the proxy, and the claude CLI
never inherit it; manual runs export `GITHUB_TOKEN` instead.
- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION`
in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64`
checksum from
`https://downloads.claude.ai/claude-code-releases/<version>/manifest.json`.
The first run on a new CLI is the riskiest one: if the new CLI
changes its wire format the matrix run can produce systematic
failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled
fire. `gh` and `uv` bump the same way, with the checksum from the
release's `gh_<version>_checksums.txt` and the tarball's `.sha256`
sidecar respectively.
- **A local build on Apple silicon only proves the image assembles.**
Under QEMU the Claude Code binary (a Bun executable) dies with
`CPU lacks AVX support` and `gh` panics in the Go runtime, so
`claude --version` and a full run are verified with a
`SKIP_PUBLISH=1` run on Render, not locally.
- **Nothing persists between runs.** A failed run leaves no
half-installed venv behind, but also no cache: don't expect a rerun
to be faster than the first one.

View file

@ -1,9 +1,8 @@
# Environment file consumed by `litellm-compat-matrix.service`.
# Environment variables of the Render cron job `litellm-compat-matrix`.
#
# Install at `/etc/litellm-compat-matrix.env` and chmod 0600.
# `EnvironmentFile=-` in the unit means the service is allowed to start
# even if this file is missing, but the populator will fail at the
# first provider request without these credentials.
# Set every value here on the Render service (Environment tab, or the
# `envVars` list of the create-service call in README.md). A local run
# passes a filled-in copy with `docker run --env-file`.
# Anthropic
ANTHROPIC_API_KEY=
@ -17,11 +16,12 @@ AWS_BEARER_TOKEN_BEDROCK=
AWS_REGION_NAME=us-east-1
# Vertex AI (vertex_ai + vertex_ai_gpt columns).
# On the GCP VM, the default service-account ADC from the metadata server
# is used -- no JSON key file is needed. If you ever need to run outside
# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json.
# The service-account key JSON is the Render secret file
# `vertex-service-account.json`, mounted at /etc/secrets, and
# GOOGLE_APPLICATION_CREDENTIALS points google-auth at it.
VERTEXAI_PROJECT=
VERTEXAI_LOCATION=global
GOOGLE_APPLICATION_CREDENTIALS=/etc/secrets/vertex-service-account.json
# Azure AI Foundry (azure column — Claude models on Foundry)
AZURE_AI_API_KEY=
@ -35,19 +35,20 @@ AZURE_API_BASE=
AZURE_API_KEY=
# The publish PAT (mateo-berri, write access on BerriAI/litellm-docs)
# deliberately does NOT live in this file. Everything here lands in the
# process environment of pytest, the proxy, and the model-driven claude
# CLI, where any same-UID reader can lift it from /proc/<pid>/environ.
# Instead, install the token at /etc/litellm-compat-matrix-github-token
# (chmod 0600, single line); the service maps it in via systemd
# LoadCredential and run_daily.sh keeps it out of every child process
# env. Used to (a) resolve the latest stable release, (b) push the
# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open
# the same-repo PR, and (d) enable squash auto-merge on it. Scopes:
# deliberately is NOT an env var. Everything here lands in the process
# environment of pytest, the proxy, and the model-driven claude CLI,
# where any same-UID reader can lift it from /proc/<pid>/environ.
# Instead, the token is the Render secret file `github-token` (single
# line), mounted under CREDENTIALS_DIRECTORY, and run_daily.sh reads it
# from there and keeps it out of every child process env. Used to
# (a) resolve the latest stable release, (b) push the daily
# compat-matrix branch directly to BerriAI/litellm-docs, (c) open the
# same-repo PR, and (d) enable squash auto-merge on it. Scopes:
# classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs
# with Contents:RW + Pull requests:RW + Workflows:RW.
# Manual runs export GITHUB_TOKEN instead, or skip publishing entirely
# with SKIP_PUBLISH=1 (only writes the matrix JSON locally).
CREDENTIALS_DIRECTORY=/etc/secrets
# Optional: the bedrock_mantle column is opt-in because the AWS account
# needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the
@ -59,9 +60,9 @@ AZURE_API_KEY=
# usually run them. Skipped cells are recorded as not_tested.
# COMPAT_OPENAI_GPT_CELLS=1
# Optional overrides; defaults are sensible for the cron VM.
# Optional overrides; defaults are sensible for the cron job.
# PROXY_PORT=4100
# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree
# LITELLM_WORKTREE=/home/populator/litellm-cron-worktree
# DOCS_REPO=BerriAI/litellm-docs
# DOCS_BRANCH=main
# DOCS_TARGET_PATH=src/data/compatibility-matrix.json

View file

@ -1,113 +0,0 @@
# systemd service for the Claude Code compatibility-matrix populator.
#
# Triggered by `litellm-compat-matrix.timer`; not started directly. The
# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics
# describe "run once per day" cleanly — there's no long-lived daemon to
# supervise; each invocation runs the populator end-to-end and exits.
#
# Install
# -------
#
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
# sudo systemctl daemon-reload
# sudo systemctl enable --now litellm-compat-matrix.timer
#
# Paths are hard-coded to /home/mateo rather than using systemd's %h
# specifier. Why: in *system* units (this one), %h is expanded at
# parse time against the *manager's* home -- which is /root for PID 1
# -- and *not* against the User= directive. That mismatch makes
# ReadWritePaths point at /root/.cache (which doesn't exist), causing
# the namespace setup to fail with status=226/NAMESPACE before the
# script ever runs. The runtime user (`User=mateo`) must:
#
# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the
# publisher module is importable;
# * have a uv venv at `~/litellm/litellm/.venv` (created by
# `uv sync --frozen` inside that checkout once);
# * have `gh` already authenticated against an account with
# `pull-requests: write` on `BerriAI/litellm-docs`;
# * have provider credentials exported in `/etc/litellm-compat-matrix.env`
# (see `litellm-compat-matrix.env.example` in this directory);
# * have the mateo-berri publish PAT at
# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single
# line), delivered via `LoadCredential=` below.
[Unit]
Description=Claude Code compatibility-matrix populator (oneshot)
Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md
Wants=network-online.target
After=network-online.target
[Service]
Type=oneshot
User=mateo
Group=mateo
# Provider credentials + any gh/PROXY_PORT overrides live here. Format
# is the standard `KEY=value` one line per env var.
EnvironmentFile=-/etc/litellm-compat-matrix.env
# The mateo-berri publish PAT is mapped in via the credential store, NOT
# the EnvironmentFile, so it never lands in the process environment that
# pytest, the proxy, and the model-driven claude CLI inherit (any
# same-UID process can read /proc/<pid>/environ). run_daily.sh reads
# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call.
# Unlike EnvironmentFile= above, this is deliberately NOT optional: a
# missing token file fails the unit at start instead of 30 minutes in.
LoadCredential=github-token:/etc/litellm-compat-matrix-github-token
# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin).
# `uv` and `claude` are installed under the runtime user's `~/.local/bin`
# so we have to prepend it explicitly; otherwise run_daily.sh fails at
# the up-front command-presence check.
Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be
# explicit so anything that reads $HOME (e.g. uv's cache lookup, the
# claude CLI's per-session dir) sees the right value even if a future
# refactor flips DynamicUser= or PrivateUsers= on.
Environment=HOME=/home/mateo
WorkingDirectory=/home/mateo/litellm/litellm
ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new
# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM,
# plus the full feature x provider grid of pytest cells hitting several
# cloud providers.
TimeoutStartSec=90min
# A failed run shouldn't restart automatically — the next timer fire is
# the right retry. Reruns of the same day's matrix are idempotent.
Restart=no
# Security hardening: the populator only reads the litellm checkout and
# the env-file; everything else it writes lives in either the worktree
# (managed) or `/tmp` (cleaned up by tempfile).
#
# ReadWritePaths whitelist:
# * litellm-cron-worktree - the long-lived stable-tag checkout +
# its `.venv` (`uv sync` rewrites every
# run) + `.uv-bin` (pinned `uv` binary
# cache).
# * .cache - uv's wheel cache (~/.cache/uv) so we
# don't redownload pinned deps each run.
# * .claude - `claude` CLI's per-session state under
# `~/.claude/projects/<sha>/`; created
# on every `claude --print` invocation.
# * .config/gh - `gh` CLI host config; technically not
# needed when we pass GH_TOKEN inline,
# but cheap to whitelist and prevents
# future regressions if a code path
# ever falls back to the host config.
# * /tmp - mktemp -d workdir + proxy logs.
NoNewPrivileges=true
ProtectSystem=strict
ProtectHome=read-only
ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp
PrivateTmp=true
[Install]
WantedBy=multi-user.target

View file

@ -1,25 +0,0 @@
# Daily timer for the compatibility-matrix populator.
#
# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so
# operators in US/EU timezones see fresh PRs at the start of their work
# day.
#
# `Persistent=true` causes a missed run (VM was off / suspended) to
# fire the next time the timer is started, which is the property we
# want for a once-a-day job: the matrix should refresh as soon as the
# VM is reachable again, not wait another 24h.
#
# `RandomizedDelaySec=10min` smears load if multiple matrix-style
# pipelines are ever colocated on the same VM in the future.
[Unit]
Description=Run the Claude Code compatibility-matrix populator daily
[Timer]
OnCalendar=*-*-* 06:00:00 UTC
Persistent=true
RandomizedDelaySec=10min
Unit=litellm-compat-matrix.service
[Install]
WantedBy=timers.target

View file

@ -1,8 +1,8 @@
#!/usr/bin/env bash
# Daily Claude Code compatibility-matrix populator.
#
# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the
# systemd timer in this directory. The flow is:
# Runs daily as the Render cron job `litellm-compat-matrix`, built from
# the Dockerfile in this directory (see README.md). The flow is:
#
# 1. Resolve the latest LiteLLM final release tag from the GitHub
# Releases API.
@ -33,12 +33,12 @@
# rather than spawning a new one. If the JSON is byte-identical to the
# docs branch, we skip the push entirely.
#
# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm.
# Required commands on $PATH: git, uv, gh, jq, curl, claude.
# Required state: a litellm checkout at $LITELLM_REPO (this file lives in
# it), $WORKTREE is created on first run, gh is already authenticated.
# it); $WORKTREE is created on first run.
#
# Override any default by setting the matching env var; see the systemd
# unit for the production wiring.
# Override any default by setting the matching env var; see README.md
# for the production wiring.
set -Eeuo pipefail
@ -52,12 +52,12 @@ DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}"
SKIP_PUBLISH="${SKIP_PUBLISH:-0}"
PYTEST_K="${PYTEST_K:-}"
# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python
# >= 3.12 (also what repo CI runs) even when the VM's system python is
# >= 3.12 (also what repo CI runs) even when the host's system python is
# older. uv fetches a managed CPython of this version on first use --
# checksum-verified against the manifest baked into the pinned uv
# binary -- and installs it under ${WORKTREE}/.uv-python (see
# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the
# systemd sandbox lets us write to.
# UV_PYTHON_INSTALL_DIR below) so everything the run writes lives inside
# the worktree.
CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}"
# Merge method for auto-merge. BerriAI/litellm-docs only allows squash
# merges (merge-commit and rebase are disabled at the repo level), so
@ -113,9 +113,9 @@ for cmd in git uv gh jq curl claude; do
done
# Publishing pushes the branch straight to BerriAI/litellm-docs and opens
# the PR as mateo-berri, who has write access on the docs repo. Under
# systemd the PAT arrives as a file via LoadCredential=, NOT via the
# EnvironmentFile: several suite cells let the model-driven claude CLI
# the PR as mateo-berri, who has write access on the docs repo. On
# Render the PAT arrives as a secret file under ${CREDENTIALS_DIRECTORY},
# NOT via an env var: several suite cells let the model-driven claude CLI
# read arbitrary files as this user, and /proc/<pid>/environ of the
# script, pytest, and the proxy would hand an env-borne token to any
# same-UID reader. Kept as an unexported shell variable and passed per
@ -125,13 +125,18 @@ done
# quota.
if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then
GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")"
log "publish token source: systemd credential store"
log "publish token source: ${CREDENTIALS_DIRECTORY}/github-token"
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
log "publish token source: process environment"
fi
if [[ "${SKIP_PUBLISH}" != "1" ]]; then
[[ -n "${GITHUB_TOKEN:-}" ]] \
|| die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|| die "publish token required: the github-token secret file under CREDENTIALS_DIRECTORY, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
# The stale-PR sweep below closes only PRs this account opened, so the
# login is resolved from the token once rather than hardcoded.
PUBLISH_LOGIN="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq .login)" \
|| die "could not resolve the publishing account from the github token"
log "publishing as ${PUBLISH_LOGIN}"
fi
# ---------------------------------------------------------------------------
@ -205,7 +210,7 @@ log "local claude code: ${CLAUDE_CODE_VERSION}"
if [[ ! -d "${WORKTREE}/.git" ]]; then
log "first run: cloning litellm into ${WORKTREE}"
mkdir -p "$(dirname "${WORKTREE}")"
git clone https://github.com/BerriAI/litellm.git "${WORKTREE}"
git clone --filter=blob:none https://github.com/BerriAI/litellm.git "${WORKTREE}"
fi
log "updating worktree to ${LITELLM_VERSION}"
@ -221,40 +226,27 @@ git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python
git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}"
# Always rebuild tests/e2e/ in the worktree from the dev checkout,
# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two
# reasons:
# regardless of what the resolved ${LITELLM_VERSION} tag ships: the
# matrix populator's job is to exercise *today's* tests against the
# latest stable proxy, and the dev checkout carries the most recent
# test fixes that haven't yet rolled into a stable release.
#
# * The matrix populator's job is to exercise *today's* tests against
# the latest stable proxy. The dev checkout carries the most recent
# test fixes that haven't yet rolled into a stable release, and we
# want every cron run to pick those up the moment they land on
# ${LITELLM_REPO}, not whenever the next stable release happens.
# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose
# top-level conftest.py imports modules (e2e_db, lifecycle,
# otel_client, ...) that the stable venv does not install. Copying
# the whole tree would make pytest collection blow up on those
# imports.
#
# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY
# the claude_code suite plus the shared transport helpers it imports.
# The whole tree is copied rather than an allowlist of the helpers the
# suite imports: the helpers import each other (proxy_client ->
# e2e_config -> fixture_mode -> ...), so a new edge in that graph turned
# an allowlist into a ModuleNotFoundError at conftest load. The tree's
# top-level conftest.py pulls in the full EKS harness (e2e_db,
# lifecycle, ...), which the stable venv does not install, so the pytest
# run below points --confcutdir at claude_code/ and never loads it.
# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while
# claude_code/ does), which is what resolves both the `claude_code.*`
# and the bare `proxy_client` / `e2e_http` imports inside the suite.
E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py)
if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then
die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
fi
for helper in "${E2E_HELPER_FILES[@]}"; do
[[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \
|| die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}"
done
log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)"
[[ -d "${LITELLM_REPO}/tests/e2e/claude_code" ]] \
|| die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
log "shimming tests/e2e/ from ${LITELLM_REPO} (always-overwrite)"
rm -rf "${WORKTREE}/tests/e2e"
mkdir -p "${WORKTREE}/tests/e2e"
cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/"
for helper in "${E2E_HELPER_FILES[@]}"; do
cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/"
done
cp -r "${LITELLM_REPO}/tests/e2e/." "${WORKTREE}/tests/e2e/"
# litellm pins an exact uv version in pyproject.toml's [tool.uv]
# `required-version` field, so a system uv that's newer or older
@ -305,10 +297,18 @@ fi
# actually serve. `--group proxy-dev` brings in pytest and the rest of
# what tests/e2e/claude_code/ needs. `--python` pins the venv to
# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates
# the venv from scratch (a one-time cold sync).
# the venv from scratch (a one-time cold sync). `--no-install-project`
# leaves litellm itself out: the tag builds a Rust extension through
# maturin, which needs a C and Rust toolchain the image does not carry,
# so the published PyPI wheel (what users install) goes in right after,
# and every later `uv run` passes `--no-sync` so uv never tries to put
# the source build back.
export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python"
log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}")
log "uv sync --frozen --group proxy-dev --extra proxy --no-install-project --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --no-install-project --python "${CRON_PYTHON_VERSION}")
LITELLM_WHEEL_VERSION="${LITELLM_VERSION#v}"
log "installing the published litellm==${LITELLM_WHEEL_VERSION} wheel from PyPI"
"${WORKTREE_UV}" pip install --python "${WORKTREE}/.venv/bin/python" --no-deps --no-build "litellm==${LITELLM_WHEEL_VERSION}"
PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)"
@ -321,10 +321,10 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
# Bind the proxy to loopback only. The populator proxy is talked to
# exclusively by the pytest run on the same host (the health check and
# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`),
# so there's no reason to expose it on the VM's external interfaces.
# so there's no reason to expose it on the container's external interfaces.
# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with
# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would
# allow anything that can reach :${PROXY_PORT} on the VM to authenticate
# allow anything that can reach :${PROXY_PORT} on the host to authenticate
# and burn upstream provider credentials.
#
# `setsid` puts the proxy in its own session+pgroup so cleanup() can
@ -334,7 +334,7 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c '
echo "$$" > "$0"
cd "$1"
exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4"
exec "$2" run --no-sync litellm --config "$3" --host 127.0.0.1 --port "$4"
' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \
>"${WORKDIR}/proxy.log" 2>&1 &
disown
@ -359,6 +359,7 @@ RESULTS_JSON="${WORKDIR}/compat-results.json"
# the cron skips them if/when they land in the suite.
PYTEST_ARGS=(
tests/e2e/claude_code/
--confcutdir=tests/e2e/claude_code
"--ignore-glob=*_unit_tests*"
)
if [[ -n "${PYTEST_K}" ]]; then
@ -373,7 +374,7 @@ set +e
&& LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \
LITELLM_MASTER_KEY="${PROXY_API_KEY}" \
COMPAT_RESULTS_PATH="${RESULTS_JSON}" \
"${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}"
"${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}"
)
PYTEST_EXIT=$?
set -e
@ -392,7 +393,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json"
log "building ${MATRIX_JSON}"
(
cd "${WORKTREE}" \
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/build_matrix.py" \
--manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \
--results "${RESULTS_JSON}" \
--output "${MATRIX_JSON}" \
@ -405,8 +406,9 @@ log "building ${MATRIX_JSON}"
# ---------------------------------------------------------------------------
if [[ "${SKIP_PUBLISH}" == "1" ]]; then
cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json"
log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json"
cp "${MATRIX_JSON}" "${HOME}/compatibility-matrix.json"
log "SKIP_PUBLISH=1; matrix saved to ${HOME}/compatibility-matrix.json and printed below"
cat "${MATRIX_JSON}"
exit 0
fi
@ -415,7 +417,7 @@ BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}
DOCS_CLONE="${WORKDIR}/litellm-docs"
log "cloning ${DOCS_REPO}@${DOCS_BRANCH}"
gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
GH_TOKEN="${GITHUB_TOKEN}" gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
cd "${DOCS_CLONE}"
git config user.email "litellm-bot@berri.ai"
@ -452,7 +454,7 @@ log "checking for green->red regressions vs the published matrix"
set +e
REGRESSION_REPORT="$(
cd "${WORKTREE}" \
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/check_regressions.py" \
--old "${PUBLISHED_MATRIX}" \
--new "${MATRIX_JSON}"
)"
@ -492,7 +494,7 @@ git commit -m "${COMMIT_MSG}"
#
# Plain --force (not --force-with-lease) is acceptable here: the
# compat-matrix/* branch is bot-owned, only this script ever writes to
# it, and runs are serialized by the systemd timer. --force-with-lease
# it, and runs are serialized by the cron schedule. --force-with-lease
# would require a fetch to populate the remote-tracking ref before each
# push and adds no safety in this single-writer setup.
PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git"
@ -553,7 +555,7 @@ Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without mergi
EOF
)"
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)"
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as ${PUBLISH_LOGIN})"
# GH_TOKEN is mateo-berri's write-scoped token, the same identity used
# for release-listing above. The branch lives on ${DOCS_REPO} itself, so
# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`.
@ -644,15 +646,25 @@ fi
#
# Non-fatal: a sweep failure (rate limit, transient API error) leaves
# stale PRs for the next run to retry; it must not fail the pipeline.
#
# The docs repo carries a few hundred open PRs, so the list has to page
# past gh's default 30 (and the earlier 100, which never reached a
# week-old compat-matrix PR and left it open for good).
#
# `compat-matrix/` is only a naming convention, so the prefix alone does
# not make a PR this job's: a contributor can open a fork PR under that
# name. Only PRs the publishing account itself opened from a branch on
# the docs repo qualify; anything else stays untouched.
log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})"
set +e
STALE_PRS="$(
GH_TOKEN="${GITHUB_TOKEN}" gh pr list \
--repo "${DOCS_REPO}" \
--state open \
--limit 100 \
--json number,headRefName \
--jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"'
--author "${PUBLISH_LOGIN}" \
--limit 1000 \
--json number,headRefName,isCrossRepository \
--jq '.[] | select((.headRefName | startswith("compat-matrix/")) and (.isCrossRepository | not)) | "\(.number)\t\(.headRefName)"'
)"
while IFS=$'\t' read -r stale_pr stale_head; do
[[ -z "${stale_pr}" ]] && continue
@ -660,7 +672,7 @@ while IFS=$'\t' read -r stale_pr stale_head; do
GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \
--repo "${DOCS_REPO}" \
--delete-branch \
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /'
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open" 2>&1 | sed 's/^/ /'
if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
log "closed stale compat-matrix PR #${stale_pr} (${stale_head})"
else

View file

@ -345,6 +345,50 @@ def request_with_retry[T: RetryableResponse](
return issue()
PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError"
PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0
def tolerate_provider_rate_limit[R: BaseModel](
issue: Callable[[], Result[R]],
*,
attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS,
sleep: Callable[[float], None] = time.sleep,
) -> Result[R]:
"""Retry a call up to `attempts` times while the proxy relays the provider's own 429;
any other outcome, the proxy's own 429 included, comes back at once."""
for attempt in range(1, attempts):
match issue():
case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body:
delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1))
print(
f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s",
flush=True,
)
sleep(delay)
case result:
return result
return issue()
class ProxyErrorDetail(BaseModel):
message: str
type: str
code: str
class _ProxyErrorBody(BaseModel):
error: ProxyErrorDetail
def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None:
"""The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own."""
if PROVIDER_RATE_LIMIT_MARKER not in outcome.body:
return None
return _ProxyErrorBody.model_validate_json(outcome.body).error
class ClassifiableResponse(Protocol):
"""What classifying an outcome reads off a response. requests.Response satisfies
it, and so does a fake, so the classification rules are testable on their own."""
@ -931,7 +975,10 @@ class PreparedForward:
def prepare_forward(
method: str, url: str, headers: dict[str, str], body: bytes | None,
method: str,
url: str,
headers: dict[str, str],
body: bytes | None,
) -> PreparedForward | NetworkError:
try:
with requests.Session() as session:
@ -950,7 +997,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream
except requests.RequestException as exc:
return NetworkError(message=str(exc))
return StreamHead(
resp.status_code, {name.lower(): value for name, value in resp.headers.items()},
resp.status_code,
{name.lower(): value for name, value in resp.headers.items()},
primed_steps(_stream_steps(resp)),
)

View file

@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
call, so a never-seen prefix must come back cached on its very first call
(Gemini's implicit caching cannot hit a cold prefix), the cached count must
cover the marked block, and the spend row must be billed below the uncached
price of the prompt.
price of the cached tokens. Vertex's cache create is nondeterministic:
identical bodies come back 200 or with the minimum-token 400 ("The cached
content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to
eight never-seen prefixes are tried with a pause after each rejection. The
billing check prices the cached tokens rather than prompt_tokens, which
Vertex reports inclusive of the cached prefix on some calls and exclusive
of it on others.
- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over
the OpenAI-compatible route; the second call must report cache-read tokens > 0.
- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the
@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
OPENAI_MODEL = "openai/gpt-5.6"
VERTEX_CACHE_TTL: Final = "300s"
VERTEX_COLD_CALL_ATTEMPTS: Final = 3
VERTEX_COLD_CALL_ATTEMPTS: Final = 8
VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0
VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024
CACHED_SHARE_OF_PROMPT: Final = 0.9
VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching"
@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons
return unwrap(result)
def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None:
for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1):
candidate = _cold_cache_call(send)
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS:
return candidate
if attempt < VERTEX_COLD_CALL_ATTEMPTS:
print(
f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; "
f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix",
flush=True,
)
time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS)
return None
def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse:
completion: Final = next(
(
candidate
for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS))
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS
),
None,
)
completion: Final = _first_engaged_cold_call(send)
assert completion is not None, (
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either "
f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} "
"cached tokens on their first call; explicit context caching did not engage"
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over "
f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by "
f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on "
"their first call; explicit context caching did not engage"
)
assert completion.choices, f"{model}: cached call returned no choices: {completion}"
usage: Final = completion.usage
cached: Final = _cached_read_tokens(usage)
assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the "
"cache; the cache_control block was not cached whole"
assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}"
assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the "
"cache_control block was not cached whole"
)
return completion
@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str,
assert row.prompt_tokens == usage.prompt_tokens, (
f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}"
)
uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model)
assert row.spend is not None and row.spend < uncached_prompt_cost, (
f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for "
f"{usage.prompt_tokens} tokens); cache-read pricing was not applied"
cached: Final = _cached_read_tokens(usage)
uncached_read_cost: Final = cached * _input_rate(client, model)
assert row.spend is not None and row.spend < uncached_read_cost, (
f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache "
f"({uncached_read_cost}); cache-read pricing was not applied"
)

View file

@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]:
def _assert_describes_cat(response: ChatResponse) -> None:
assert response.choices, f"vision returned no choices: {response}"
message = response.choices[0].message
content = (message.content if message else None) or ""
choice = response.choices[0]
content = (choice.message.content if choice.message else None) or ""
assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), (
f"vision response did not describe the image: {content[:200]}"
f"vision response did not describe the image: {content[:200]!r} "
f"(finish_reason={choice.finish_reason!r}, usage={response.usage})"
)
@ -421,7 +422,11 @@ class TestVertexChatCompletions:
model = self._register(client, resources, "e2e-vertex-vision")
key = resources.key()
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
response = unwrap(
client.proxy.chat(
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none")
)
)
_assert_describes_cat(response)
@pytest.mark.covers(

View file

@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
fails when no proxy answers and also fails once a request reaches it: the proxy
fetches each provider's referenced secrets, so a
missing credential surfaces as a live provider error rather than silent green.
The provider keys are shared with other pipelines, so a provider's rate limit can
hold across the bounded retries. The case then accepts the gateway's faithful relay
of that 429 (throttling_error, code 429) as its second expected outcome; any other
non-success still fails at once.
"""
from __future__ import annotations
from dataclasses import dataclass
from typing import Protocol
from typing import Final, Protocol
import pytest
from e2e_config import unique_marker
from e2e_http import assert_client_error, unwrap
from e2e_http import (
PROVIDER_RATE_LIMIT_ATTEMPTS,
RateLimitedError,
Success,
assert_client_error,
relayed_provider_rate_limit,
tolerate_provider_rate_limit,
)
from lifecycle import ResourceManager
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
from proxy_client import ProxyClient
@ -146,24 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None:
assert response.pages[0].markdown is not None, "first page has no markdown"
def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None:
detail: Final = relayed_provider_rate_limit(outcome)
assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}"
assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}"
print(
f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; "
f"the gateway relayed it as {detail.type} {detail.code}",
flush=True,
)
class TestRustOcrGateway:
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
def test_rust_ocr_response(
self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase
) -> None:
def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None:
model = f"rust-ocr-{case.suffix}-{unique_marker()}"
model_id = proxy.create_model(model, case.provider.litellm_params())
resources.defer(lambda: proxy.delete_model(model_id))
key = resources.key()
response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document)))
_assert_ocr_document(response)
match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))):
case Success(data=response):
_assert_ocr_document(response)
case RateLimitedError() as outcome:
_assert_provider_rate_limit_relayed(model, outcome)
case outcome:
pytest.fail(f"{model}: {outcome!r}")
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")
@pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works")
def test_missing_document_returns_error(
self, proxy: ProxyClient, resources: ResourceManager
) -> None:
def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
model = f"rust-ocr-val-{unique_marker()}"
model_id = proxy.create_model(model, MistralOcr().litellm_params())
resources.defer(lambda: proxy.delete_model(model_id))
@ -174,4 +198,3 @@ class TestRustOcrGateway:
json=_OptionalOcrBody(model=model),
)
assert_client_error(result, "ocr missing document")

View file

@ -15,9 +15,10 @@ reliability behavior.
from __future__ import annotations
from collections.abc import Sequence
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import ValidationError
from pydantic import BaseModel, ValidationError
from proxy_client import ProxyClient
from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker
@ -367,6 +368,26 @@ def model_id_of(resp: StreamingResponse) -> str | None:
return resp.headers.get("x-litellm-model-id")
class _AzurePromptFilterResult(BaseModel):
content_filter_results: Mapping[str, object] | None = None
class _AzureAnnotatedChatBody(BaseModel):
prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None
def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool:
"""True when Azure's 200 recorded no prompt-filter verdict (every `content_filter_results`
empty), so the prompt has to be sent again."""
try:
annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body)
except ValidationError:
return False
if not annotated.prompt_filter_results:
return False
return all(not entry.content_filter_results for entry in annotated.prompt_filter_results)
def _parsed(resp: StreamingResponse) -> ChatResponse | None:
try:
return ChatResponse.model_validate_json(resp.body)

View file

@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and
reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure
OpenAI content filter rejecting a jailbreak prompt, and a control call first
proves the refusal reaches the customer as a 400 when no reroute is configured.
Azure intermittently answers without running its prompt filter at all (the
body's prompt_filter_results carry no verdict), which is not a pass, so both
calls send the prompt again, a bounded number of times, until the filter ran.
"""
from __future__ import annotations
from collections.abc import Callable
from typing import Final
import pytest
from complexity_router_client import ComplexityRouterClient
@ -29,6 +35,7 @@ from lifecycle import ResourceManager
from models import RouterSettingsOverride
from reliability_support import (
CONTENT_POLICY_PROMPT,
azure_prompt_filter_skipped,
chat_override,
completion_tokens_of,
content_of,
@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None:
assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}"
AZURE_FILTER_ATTEMPTS: Final = 3
def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse:
for attempt in range(1, AZURE_FILTER_ATTEMPTS):
resp = send()
if not azure_prompt_filter_skipped(resp):
return resp
print(
"e2e: azure answered without running its prompt filter; sending the jailbreak prompt again "
f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})",
flush=True,
)
return send()
def _filter_verdict(resp: StreamingResponse) -> str:
if azure_prompt_filter_skipped(resp):
return "azure skipped its prompt filter on every attempt"
return "the filter ran and let the prompt through"
class TestReliabilityFallbacks:
@pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback")
def test_5xx_routes_to_fallback(
@ -124,17 +153,21 @@ class TestReliabilityFallbacks:
model_id = create_content_filtered_deployment(client.proxy, primary)
resources.defer(lambda: client.proxy.delete_model(model_id))
refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
refused = _chat_once_azure_runs_its_filter(
lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
)
assert refused.status_code == 400, (
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: "
f"{refused.body[:300]}"
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} "
f"({_filter_verdict(refused)}): {refused.body[:300]}"
)
resp = chat_override(
client.proxy,
scoped_key,
primary,
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
resp = _chat_once_azure_runs_its_filter(
lambda: chat_override(
client.proxy,
scoped_key,
primary,
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
)
)
_assert_served_by_fallback(resp)

View file

@ -290,6 +290,12 @@
"tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [
"quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"
],
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
],
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
],
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
"mgmt.key.update.project_detach_denied_to_restricted_actor"
],
@ -1810,6 +1816,126 @@
"tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": [
"mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers"
],
"tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": [
"providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment"
],
"tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": [
"providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry"
],
"tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": [
"other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body"
],
"tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": [
"other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider"
],
"tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": [
"providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization"
],
"tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": [
"other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"
],
"tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": [
"other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking"
],
"tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": [
"other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level"
],
"tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": [
"other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format"
],
"tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": [
"providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort"
],
"tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": [
"providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support"
],
"tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": [
"other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types"
],
"tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": [
"providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items"
],
"tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": [
"other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long"
],
"tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": [
"providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text"
],
"tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": [
"other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type"
],
"tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": [
"providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned"
],
"tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": [
"other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text"
],
"tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": [
"other.provider_wire.dashscope.reasoning_effort_reaches_provider"
],
"tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": [
"other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log"
],
"tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": [
"other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base"
],
"tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": [
"other.provider_wire.deepseek.vision_image_content_list_reaches_provider"
],
"tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": [
"other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name"
],
"tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": [
"other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name"
],
"tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": [
"providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire"
],
"tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": [
"other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields"
],
"tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": [
"other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length"
],
"tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": [
"other.provider_wire.tencent.thinking_reaches_provider_in_request_body"
],
"tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": [
"other.provider_wire.tencent.thinking_reaches_provider_in_request_body"
],
"tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": [
"other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use"
],
"tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": [
"other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results"
],
"tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": [
"other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters"
],
"tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": [
"other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers"
],
"tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": [
"other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload"
],
"tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": [
"other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream"
],
"tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": [
"other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost"
],
"tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": [
"other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend"
],
"tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": [
"other.streaming.responses_bridge.empty_choices_chunks_complete_stream"
],
"tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": [
"streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails"
],
"tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": [
"streaming.keepalive.sse_pings_fill_silent_time_to_first_token"
],
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
],

View file

@ -0,0 +1,113 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
_ADVISOR_KEY: Final = "synthetic-advisor-key"
_QUESTION: Final = "which index should this query use"
_ADVICE: Final = "use the composite index on (tenant_id, created_at)"
_FINAL_ANSWER: Final = "done, the composite index is the right one"
_ADVISOR_CALL_MESSAGE: Final = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "advisor-call",
"type": "function",
"function": {"name": "advisor", "arguments": json.dumps({"question": _QUESTION})},
}
],
}
_FINAL_MESSAGE: Final = {"role": "assistant", "content": _FINAL_ANSWER}
def _chat_completion(identity: str, message: dict[str, object], finish_reason: str) -> Reply:
return Reply(
body=json.dumps(
{
"id": f"chatcmpl-{identity}",
"object": "chat.completion",
"created": 1,
"model": "llama-3.3-70b-versatile",
"choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],
"usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14},
}
).encode()
)
def _executor_reply(body: dict[str, object], identity: str) -> Reply:
messages: Final = body["messages"]
assert isinstance(messages, list)
if any(message.get("role") == "tool" for message in messages):
assert messages[-1]["content"] == _ADVICE
return _chat_completion(identity, _FINAL_MESSAGE, "stop")
tools: Final = body["tools"]
assert isinstance(tools, list)
assert tools[0]["function"]["name"] == "advisor"
return _chat_completion(identity, _ADVISOR_CALL_MESSAGE, "tool_calls")
@pytest.mark.covers("providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment")
def test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated(
gateway: Gateway,
) -> None:
identity: Final = "advisor-wire-" + uuid.uuid4().hex
def respond(request: Request) -> Reply:
body: Final = json.loads(request.body)
if request.target == "/v1/chat/completions":
assert request.headers["authorization"] == "Bearer integration-provider-key"
return _executor_reply(body, identity)
assert request.target == "/v1/messages"
assert request.headers["x-api-key"] == _ADVISOR_KEY
assert body["model"] == "claude-opus-4-1-20250805"
assert body["messages"] == [
{"role": "user", "content": "please plan the migration"},
{"role": "user", "content": _QUESTION},
]
assert "tools" not in body
return Reply(
body=json.dumps(
{
"id": f"msg-{identity}",
"type": "message",
"role": "assistant",
"model": "claude-opus-4-1-20250805",
"content": [{"type": "text", "text": _ADVICE}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 6},
}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
executor: Final = scenario.model(model="hosted_vllm/llama-3.3-70b", api_base=wire.url + "/v1")
advisor: Final = scenario.model(
model="anthropic/claude-opus-4-1-20250805", api_base=wire.url, api_key=_ADVISOR_KEY
)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": executor,
"max_tokens": 64,
"messages": [{"role": "user", "content": "please plan the migration"}],
"tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor}],
},
)
assert response.status_code == 200, response.text
body: Final = response.json()
assert body["content"] == [{"type": "text", "text": _FINAL_ANSWER}], response.text
assert body["stop_reason"] == "end_turn", response.text
assert [request.target for request in wire.drain()] == [
"/v1/chat/completions",
"/v1/messages",
"/v1/chat/completions",
]

View file

@ -0,0 +1,85 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "kimi-k2-thinking"
_API_KEY: Final = "synthetic-azure-ai-key"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_THINKING_BLOCK: Final[JsonValue] = {
"type": "thinking",
"thinking": "The user wants the sum of 17 and 26.",
"signature": "synthetic-signature",
}
_HISTORY_WITH_ANTHROPIC_FIELDS: Final[JsonValue] = [
{
"role": "system",
"content": "You are a calculator.",
"cache_control": {"type": "ephemeral"},
},
{"role": "user", "content": "What is 17 + 26?"},
{
"role": "assistant",
"content": "43",
"thinking_blocks": [_THINKING_BLOCK],
"provider_specific_fields": {"citations": None},
},
{"role": "user", "content": "And doubled?"},
]
_HISTORY_AS_OPENAI_SPEC: Final[JsonValue] = [
{"role": "system", "content": "You are a calculator."},
{"role": "user", "content": "What is 17 + 26?"},
{"role": "assistant", "content": "43"},
{"role": "user", "content": "And doubled?"},
]
def _completion(identity: str) -> bytes:
return json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": _BACKEND,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "86"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 31, "completion_tokens": 2, "total_tokens": 33},
}
).encode()
@pytest.mark.covers("providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry")
def test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages(gateway: Gateway) -> None:
identity: Final = f"azure-ai-strip-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/chat/completions"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _BACKEND
assert body["messages"] == _HISTORY_AS_OPENAI_SPEC
return Reply(body=_completion(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"azure_ai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": _HISTORY_WITH_ANTHROPIC_FIELDS},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["id"] == identity
assert payload["choices"] == [
{
"finish_reason": "stop",
"index": 0,
"message": {"role": "assistant", "content": "86"},
"provider_specific_fields": {},
}
]
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]

View file

@ -0,0 +1,48 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_FLEX_MODEL: Final = "azure_ai/FLUX.2-flex"
_PROMPT: Final = "a red fox in the snow"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
@pytest.mark.covers("other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body")
def test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview"
assert request.headers["api-key"] == "synthetic-azure-key"
assert _JSON_OBJECT.validate_json(request.body) == {
"model": "FLUX.2-flex",
"prompt": _PROMPT,
"num_images": 2,
"width": 1536,
"height": 1024,
"guidance": 4.5,
"steps": 32,
}
return Reply(body=json.dumps({"data": [{"b64_json": "aW1n"}, {"b64_json": "aW1n"}]}).encode())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=_FLEX_MODEL, api_base=wire.url, api_key="synthetic-azure-key", api_version="preview"
)
response: Final = gateway.request(
"POST",
"/v1/images/generations",
{"model": model, "prompt": _PROMPT, "n": 2, "size": "1536x1024", "guidance": 4.5, "steps": 32},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["data"] == [
{"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None},
{"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None},
]
assert [(request.method, request.target) for request in wire.drain()] == [
("POST", "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview")
]

View file

@ -0,0 +1,46 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "azure_ai/Cohere-rerank-v4.0-fast"
ENTRA_TOKEN: Final = "synthetic-entra-access-token"
QUERY: Final = "which document mentions the gateway"
DOCUMENTS: Final = ("the gateway proxies rerank calls", "unrelated synthetic text")
RESPONSE: Final = json.dumps(
{
"id": "synthetic-rerank-id",
"results": [{"index": 0, "relevance_score": 0.91}, {"index": 1, "relevance_score": 0.03}],
"meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 1}},
}
).encode()
def entra_rerank_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/providers/cohere/v2/rerank"
assert request.headers["authorization"] == f"Bearer {ENTRA_TOKEN}"
assert "api-key" not in request.headers
body: Final = json.loads(request.body)
assert body == {"model": "Cohere-rerank-v4.0-fast", "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2}
return Reply(body=RESPONSE)
@pytest.mark.covers("other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider")
def test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider(gateway: Gateway) -> None:
with wire_server(entra_rerank_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL,
api_key=None,
api_base=f"{wire.url}/providers/cohere/v2",
azure_ad_token=ENTRA_TOKEN,
model_info={"mode": "rerank"},
)
response: Final = gateway.request(
"POST", "/v1/rerank", {"model": model, "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2}
)
assert response.status_code == 200, response.text
body: Final = response.json()
assert [(result["index"], result["relevance_score"]) for result in body["results"]] == [(0, 0.91), (1, 0.03)]
assert len(wire.drain()) == 1, "Expected exactly one provider rerank call"

View file

@ -7,18 +7,22 @@ from typing import Final
import pytest
import yaml
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0"
TOKEN: Final = "synthetic-bedrock-bearer"
RESPONSE: Final = json.dumps({
"output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}},
"stopReason": "end_turn", "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15},
"metrics": {"latencyMs": 1},
}).encode()
ACCESS_KEY: Final = "AKIAINTEGRATION000002"
CLIENT_OAUTH_TOKEN: Final = "Bearer sk-ant-oat01-synthetic-client-subscription-token"
RESPONSE: Final = json.dumps(
{
"output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15},
"metrics": {"latencyMs": 1},
}
).encode()
def bearer_peer(request: Request) -> Reply:
@ -34,31 +38,58 @@ def bearer_peer(request: Request) -> Reply:
@pytest.mark.covers("other.provider_wire.bedrock.bearer_sdk_skips_credential_chain")
async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
import litellm
empty: Final = tmp_path / "empty-aws-config"
empty.write_text("")
for name in tuple(name for name in os.environ if name.startswith("AWS_")):
monkeypatch.delenv(name, raising=False)
for name, value in {"AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"}.items():
for name, value in {
"AWS_CONFIG_FILE": str(empty),
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
"AWS_EC2_METADATA_DISABLED": "true",
"LITELLM_RUST": "false",
}.items():
monkeypatch.setenv(name, value)
with wire_server(bearer_peer) as wire:
with pytest.raises(litellm.APIConnectionError, match=r"config profile .* could not be found"):
await asyncio.to_thread(litellm.completion, model=MODEL, aws_profile_name="integration-profile-must-not-be-read", aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url, messages=[{"role": "user", "content": "synthetic credential control"}], timeout=5, num_retries=0)
await asyncio.to_thread(
litellm.completion,
model=MODEL,
aws_profile_name="integration-profile-must-not-be-read",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
messages=[{"role": "user", "content": "synthetic credential control"}],
timeout=5,
num_retries=0,
)
assert wire.drain() == ()
for source in ("argument", "environment"):
if source == "environment":
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", TOKEN)
parameters: Final = {
"model": MODEL, "api_key": TOKEN if source == "argument" else None,
"aws_region_name": "us-east-1", "aws_profile_name": "integration-profile-must-not-be-read",
"aws_bedrock_runtime_endpoint": wire.url, "timeout": 5, "num_retries": 0,
"messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}],
"model": MODEL,
"api_key": TOKEN if source == "argument" else None,
"aws_region_name": "us-east-1",
"aws_profile_name": "integration-profile-must-not-be-read",
"aws_bedrock_runtime_endpoint": wire.url,
"timeout": 5,
"num_retries": 0,
"messages": [
{"role": "system", "content": "synthetic system"},
{"role": "user", "content": "synthetic bearer request"},
],
"max_tokens": 16,
}
for asynchronous in (False, True):
result: Final = await litellm.acompletion(**parameters) if asynchronous else await asyncio.to_thread(litellm.completion, **parameters)
result: Final = (
await litellm.acompletion(**parameters)
if asynchronous
else await asyncio.to_thread(litellm.completion, **parameters)
)
assert result.choices[0].message.content == "bedrock wire control"
assert result.choices[0].finish_reason == "stop"
assert result.usage.prompt_tokens == 11 and result.usage.completion_tokens == 4
@ -66,28 +97,57 @@ async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credential
@pytest.mark.covers("other.provider_wire.bedrock.bearer_db_yaml_survives_reload")
def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload(gateway: Gateway, tmp_path: Path) -> None:
def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload(
gateway: Gateway, tmp_path: Path
) -> None:
empty: Final = tmp_path / "empty-aws-config"
empty.write_text("")
with wire_server(bearer_peer) as wire:
parameters: Final = {
"model": MODEL, "api_key": "os.environ/INTEGRATION_BEARER_TOKEN", "aws_region_name": "us-east-1",
"aws_profile_name": "integration-profile-must-not-be-read", "aws_bedrock_runtime_endpoint": wire.url,
"model": MODEL,
"api_key": "os.environ/INTEGRATION_BEARER_TOKEN",
"aws_region_name": "us-east-1",
"aws_profile_name": "integration-profile-must-not-be-read",
"aws_bedrock_runtime_endpoint": wire.url,
}
alias: Final = f"integration-yaml-{uuid.uuid4().hex}"
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
configuration["model_list"] = [{"model_name": alias, "litellm_params": parameters, "model_info": {"id": alias}}]
path: Final = tmp_path / "bedrock.yaml"
path.write_text(yaml.safe_dump(configuration))
overrides: Final = {"INTEGRATION_BEARER_TOKEN": TOKEN, "AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"}
with owned_proxy(gateway, tmp_path, overrides, config=path, remove_environment=tuple(name for name in os.environ if name.startswith("AWS_"))) as candidate, candidate.scenario() as scenario:
overrides: Final = {
"INTEGRATION_BEARER_TOKEN": TOKEN,
"AWS_CONFIG_FILE": str(empty),
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
"AWS_EC2_METADATA_DISABLED": "true",
"LITELLM_RUST": "false",
}
with (
owned_proxy(
gateway,
tmp_path,
overrides,
config=path,
remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")),
) as candidate,
candidate.scenario() as scenario,
):
database_model: Final = scenario.model(**parameters)
for generation in range(2):
for model in (alias, database_model):
response: Final = candidate.request("POST", "/v1/chat/completions", {
"model": model, "messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}],
"max_tokens": 16, "cache": {"no-cache": True},
})
response: Final = candidate.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [
{"role": "system", "content": "synthetic system"},
{"role": "user", "content": "synthetic bearer request"},
],
"max_tokens": 16,
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control"
assert response.json()["usage"]["total_tokens"] == 15
@ -95,5 +155,61 @@ def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload
if generation == 0:
entries: Final = candidate.get("/model/info")["data"]
target: Final = next(entry for entry in entries if entry["model_name"] == database_model)
response: Final = candidate.request("PATCH", f"/model/{target['model_info']['id']}/update", {"model_info": {"description": "bearer reload"}})
response: Final = candidate.request(
"PATCH",
f"/model/{target['model_info']['id']}/update",
{"model_info": {"description": "bearer reload"}},
)
assert response.status_code == 200, response.text
INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-3-haiku-20240307-v1:0"
INVOKE_RESPONSE: Final = json.dumps(
{
"id": "msg_synthetic",
"type": "message",
"role": "assistant",
"model": "anthropic.claude-3-haiku-20240307-v1:0",
"content": [{"type": "text", "text": "bedrock invoke wire control"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 11, "output_tokens": 4},
}
).encode()
def sigv4_invoke_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1:0/invoke"
assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/"), dict(
request.headers
)
assert CLIENT_OAUTH_TOKEN not in request.headers.values(), dict(request.headers)
assert json.loads(request.body)["messages"] == [{"role": "user", "content": "synthetic oauth isolation request"}]
return Reply(body=INVOKE_RESPONSE)
@pytest.mark.covers("providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization")
def test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature(gateway: Gateway) -> None:
with wire_server(sigv4_invoke_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=INVOKE_MODEL,
api_key=None,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key="synthetic-secret-key-for-testing",
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
api_base=wire.url,
)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"messages": [{"role": "user", "content": "synthetic oauth isolation request"}],
"max_tokens": 16,
},
headers={"Authorization": CLIENT_OAUTH_TOKEN, "x-litellm-api-key": f"Bearer {gateway.key}"},
)
assert response.status_code == 200, response.text
assert response.json()["content"] == [{"type": "text", "text": "bedrock invoke wire control"}], response.text
assert len(wire.drain()) == 1, response.text

View file

@ -0,0 +1,78 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/anthropic.claude-3-haiku-20240307-v1:0"
BUCKET: Final = "integration-batch-bucket"
PROMPT: Final = "synthetic completions prompt"
RESPONSES_INPUT: Final = "synthetic responses input"
INPUT_LINES: Final = (
{
"custom_id": "completions-record",
"method": "POST",
"url": "/v1/completions",
"body": {"model": MODEL, "prompt": PROMPT, "max_tokens": 64},
},
{
"custom_id": "responses-record",
"method": "POST",
"url": "/v1/responses",
"body": {"model": MODEL, "input": RESPONSES_INPUT, "max_output_tokens": 16},
},
)
EXPECTED_S3_OBJECT: Final = (
{
"recordId": "completions-record",
"modelInput": {
"messages": [{"role": "user", "content": [{"type": "text", "text": PROMPT}]}],
"max_tokens": 64,
"anthropic_version": "bedrock-2023-05-31",
},
},
{
"recordId": "responses-record",
"modelInput": {
"messages": [{"role": "user", "content": [{"type": "text", "text": RESPONSES_INPUT}]}],
"max_tokens": 16,
"anthropic_version": "bedrock-2023-05-31",
},
},
)
def s3_peer(request: Request) -> Reply:
assert request.method == "PUT" and request.target.startswith(f"/{BUCKET}/"), request.target
assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 ")
return Reply(body=b"")
@pytest.mark.covers(
"other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"
)
def test_completions_and_responses_batch_records_upload_as_anthropic_user_messages(gateway: Gateway) -> None:
with wire_server(s3_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL,
api_key=None,
api_base=None,
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
aws_region_name="us-east-1",
s3_bucket_name=BUCKET,
s3_endpoint_url=wire.url,
)
jsonl: Final = "\n".join(json.dumps(line, separators=(",", ":")) for line in INPUT_LINES) + "\n"
response: Final = gateway.request_multipart(
"/v1/files",
{"purpose": "batch", "model": model},
{"file": ("in.jsonl", jsonl.encode(), "application/jsonl")},
)
assert response.status_code == 200, response.text
assert response.json()["object"] == "file" and response.json()["purpose"] == "batch", response.text
uploads: Final = wire.drain()
assert len(uploads) == 1, f"Expected exactly one S3 PUT, saw {[upload.target for upload in uploads]}"
stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip())
assert stored == EXPECTED_S3_OBJECT

View file

@ -0,0 +1,60 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/invoke/us.anthropic.claude-opus-4-8"
TOKEN: Final = "synthetic-bedrock-bearer"
RESPONSE: Final = json.dumps(
{
"id": "msg_adaptive_control",
"type": "message",
"role": "assistant",
"model": "us.anthropic.claude-opus-4-8",
"content": [{"type": "text", "text": "adaptive thinking control"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 5},
}
).encode()
def adaptive_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/us.anthropic.claude-opus-4-8/invoke"
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "synthetic effort request"}]}]
assert body["thinking"]["type"] == "adaptive", body
assert body["output_config"] == {"effort": "high"}, body
assert "budget_tokens" not in json.dumps(body), body
return Reply(body=RESPONSE)
@pytest.mark.covers("other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking")
def test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens(
gateway: Gateway,
) -> None:
with wire_server(adaptive_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL,
api_key=TOKEN,
aws_region_name="us-east-1",
api_base=wire.url,
aws_bedrock_runtime_endpoint=wire.url,
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": "synthetic effort request"}],
"max_tokens": 4096,
"reasoning_effort": "high",
},
)
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == "adaptive thinking control"
assert response.json()["usage"]["prompt_tokens"] == 12 and response.json()["usage"]["completion_tokens"] == 5
assert len(wire.drain()) == 1

View file

@ -0,0 +1,46 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from integration.providers.test_bedrock_auth_wire import MODEL, RESPONSE, TOKEN
GUARDRAIL: Final = {"guardrailIdentifier": "integration-guardrail", "guardrailVersion": "DRAFT", "trace": "enabled"}
PERFORMANCE: Final = {"latency": "optimized"}
def converse_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse"
body: Final = json.loads(request.body)
assert body["inferenceConfig"] == {"maxTokens": 16, "temperature": 0.2}, body
assert body["guardrailConfig"] == GUARDRAIL, body
assert body["performanceConfig"] == PERFORMANCE, body
return Reply(body=RESPONSE)
@pytest.mark.covers("other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level")
def test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config(gateway: Gateway) -> None:
with wire_server(converse_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL,
api_key=TOKEN,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
guardrailConfig=GUARDRAIL,
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": "synthetic guardrail request"}],
"max_tokens": 16,
"temperature": 0.2,
"performanceConfig": PERFORMANCE,
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control"
assert len(wire.drain()) == 1, response.text

View file

@ -0,0 +1,58 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/cohere.embed-english-v3"
TOKEN: Final = "synthetic-bedrock-bearer"
INPUT: Final = "hello world"
VECTOR: Final = [0.1, 0.2, 0.3]
RESPONSE: Final = json.dumps(
{
"embeddings": {"float": [VECTOR]},
"id": "synthetic-cohere-embed",
"response_type": "embeddings_by_type",
"texts": [INPUT],
}
).encode()
def cohere_english_v3_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/cohere.embed-english-v3/invoke"
assert request.headers["authorization"] == f"Bearer {TOKEN}"
assert json.loads(request.body) == {
"texts": [INPUT],
"input_type": "search_document",
"embedding_types": ["float"],
"output_dimension": 512,
}
return Reply(body=RESPONSE)
@pytest.mark.covers("other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format")
def test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions(gateway: Gateway) -> None:
with wire_server(cohere_english_v3_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL,
api_key=TOKEN,
api_base=wire.url,
aws_region_name="us-east-1",
)
for encoding_format in ("float", "base64"):
response: Final = gateway.request(
"POST",
"/v1/embeddings",
{
"model": model,
"input": INPUT,
"encoding_format": encoding_format,
"dimensions": 512,
},
)
assert response.status_code == 200, f"encoding_format={encoding_format}: {response.text}"
assert response.json()["data"] == [
{"object": "embedding", "index": 0, "embedding": VECTOR, "type": "float"},
], response.text
assert len(wire.drain()) == 1, f"encoding_format={encoding_format} never reached Bedrock"

View file

@ -0,0 +1,50 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/converse/us.openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-bedrock-bearer"
RESPONSE: Final = json.dumps(
{
"output": {"message": {"role": "assistant", "content": [{"text": "gpt-5 reasoning wire control"}]}},
"stopReason": "end_turn",
"usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14},
"metrics": {"latencyMs": 1},
}
).encode()
def gpt5_converse_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/us.openai.gpt-5.6-sol/converse"
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body["messages"] == [{"role": "user", "content": [{"text": "synthetic reasoning request"}]}]
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
assert body["inferenceConfig"] == {"maxTokens": 16}, body
return Reply(body=RESPONSE)
@pytest.mark.covers("providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort")
def test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort(gateway: Gateway) -> None:
with wire_server(gpt5_converse_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": "synthetic reasoning request"}],
"reasoning_effort": "high",
"max_tokens": 16,
"cache": {"no-cache": True},
},
)
assert response.status_code == 200, response.text
assert response.json()["choices"][0]["message"]["content"] == "gpt-5 reasoning wire control", response.text
assert response.json()["usage"]["total_tokens"] == 14, response.text
assert len(wire.drain()) == 1

View file

@ -0,0 +1,88 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
MODEL_ID: Final = "us.anthropic.claude-sonnet-5"
TOKEN: Final = "synthetic-bedrock-bearer"
TOOL_SEARCH_TOOL: Final = {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}
DEFERRED_TOOL: Final = {
"name": "get_weather",
"description": "Weather lookup",
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
"defer_loading": True,
}
RESPONSE: Final = json.dumps(
{
"id": "msg_tool_search_control",
"type": "message",
"role": "assistant",
"model": MODEL_ID,
"content": [
{
"type": "server_tool_use",
"id": "srvtoolu_control",
"name": "tool_search_tool_regex",
"input": {"pattern": "weather"},
},
{
"type": "tool_search_tool_result",
"tool_use_id": "srvtoolu_control",
"content": {
"type": "tool_search_tool_search_result",
"tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}],
},
},
{"type": "text", "text": "tool search wire control"},
],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 6},
}
).encode()
def tool_search_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == f"/model/{MODEL_ID}/invoke", request.target
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body["anthropic_beta"] == ["tool-search-tool-2025-10-19"], body
assert body["messages"] == [{"role": "user", "content": "find the weather tool"}]
assert body["tools"] == [TOOL_SEARCH_TOOL, DEFERRED_TOOL], body["tools"]
assert body["max_tokens"] == 64
assert "model" not in body
return Reply(body=RESPONSE)
@pytest.mark.covers("providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support")
def test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field(gateway: Gateway) -> None:
with wire_server(tool_search_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/invoke/{MODEL_ID}",
api_key=TOKEN,
aws_region_name="us-east-1",
api_base=wire.url,
)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 64,
"messages": [{"role": "user", "content": "find the weather tool"}],
"tools": [TOOL_SEARCH_TOOL, DEFERRED_TOOL],
},
)
assert response.status_code == 200, response.text
body: Final = response.json()
assert body["content"][2] == {"type": "text", "text": "tool search wire control"}, response.text
assert body["stop_reason"] == "end_turn"
assert body["usage"]["input_tokens"] == 12 and body["usage"]["output_tokens"] == 6
assert len(wire.drain()) == 1
entries: Final = gateway.get("/v1/model/info")["data"]
assert isinstance(entries, list)
info: Final = next(entry for entry in entries if isinstance(entry, dict) and entry["model_name"] == model)
assert isinstance(info["model_info"], dict)
assert info["model_info"]["supports_tool_search"] is True, info["model_info"]

View file

@ -0,0 +1,88 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol"
TOKEN: Final = "synthetic-mantle-bearer"
CIPHERTEXT: Final = "synthetic-compaction-ciphertext"
CALL_ID: Final = "call_synthetic_shell"
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000}
RESPONSE: Final = json.dumps(
{
"id": "resp_synthetic_mantle",
"object": "response",
"created_at": 1789788253,
"status": "completed",
"model": "openai.gpt-5.6-sol",
"output": [
{
"type": "message",
"id": "msg_synthetic_mantle",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}],
}
],
"usage": {"input_tokens": 21, "output_tokens": 4, "total_tokens": 25},
}
).encode()
def user_turn(text: str) -> JsonValue:
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}
def codex_history(marker: str) -> tuple[JsonValue, ...]:
return (
user_turn(f"first turn {marker}"),
{"type": "agent_message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]},
{"type": "context_compaction", "encrypted_content": CIPHERTEXT},
{"type": "local_shell_call", "call_id": CALL_ID, "status": "completed", "action": ACTION},
{"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"},
user_turn(f"next turn {marker}"),
)
def mantle_history(marker: str) -> tuple[JsonValue, ...]:
return (
user_turn(f"first turn {marker}"),
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]},
{"type": "compaction", "encrypted_content": CIPHERTEXT},
{"type": "function_call", "call_id": CALL_ID, "name": "local_shell", "arguments": json.dumps(ACTION)},
{"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"},
user_turn(f"next turn {marker}"),
)
@pytest.mark.covers("other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types")
def test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items(
gateway: Gateway,
) -> None:
marker: Final = uuid.uuid4().hex
expected_input: Final = list(mantle_history(marker))
def mantle_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/openai/v1/responses", request.target
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = JSON_OBJECT.validate_json(request.body)
assert body["model"] == "openai.gpt-5.6-sol", body
assert body["input"] == expected_input, body["input"]
return Reply(body=RESPONSE)
with wire_server(mantle_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=MODEL, api_key=TOKEN, api_base=wire.url, aws_region_name="us-east-2")
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": list(codex_history(marker)), "store": False}
)
assert response.status_code == 200, response.text
assert response.json()["output"][0]["content"][0]["text"] == "mantle wire control", response.text
assert response.json()["usage"]["total_tokens"] == 25, response.text
forwarded: Final = wire.drain()
assert len(forwarded) == 1, forwarded
assert JSON_OBJECT.validate_json(forwarded[0].body)["input"] == expected_input, forwarded[0].body

View file

@ -0,0 +1,106 @@
import json
from collections.abc import Callable
from typing import Final
from uuid import uuid4
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol"
_TOKEN: Final = "synthetic-mantle-bearer"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_SHELL_ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000}
_OUTPUT_MESSAGE: Final[dict[str, JsonValue]] = {
"type": "message",
"id": "msg_mantle",
"status": "completed",
"role": "assistant",
"content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}],
}
_RESPONSE: Final = json.dumps(
{
"id": "resp_mantle",
"object": "response",
"status": "completed",
"created_at": 1700000000,
"model": "gpt-5.6-sol",
"output": [_OUTPUT_MESSAGE],
"usage": {
"input_tokens": 11,
"output_tokens": 4,
"total_tokens": 15,
"input_tokens_details": {"cached_tokens": 0},
"output_tokens_details": {"reasoning_tokens": 0},
},
}
).encode()
def _codex_history(marker: str) -> list[JsonValue]:
return [
{"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"},
{
"type": "agent_message",
"id": "msg_agent",
"content": [{"type": "text", "text": "sub-agent said "}, {"type": "text", "encrypted_content": "hello"}],
},
{"type": "context_compaction", "id": "cmp_1", "encrypted_content": "compacted-history"},
{
"type": "local_shell_call",
"id": "lsc_1",
"call_id": "call_shell",
"status": "completed",
"action": _SHELL_ACTION,
},
{"type": "function_call_output", "call_id": "call_shell", "output": "total 0"},
]
def _mantle_history(marker: str) -> list[JsonValue]:
return [
{"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"},
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent said hello"}]},
{"type": "compaction", "encrypted_content": "compacted-history"},
{
"type": "function_call",
"call_id": "call_shell",
"name": "local_shell",
"arguments": json.dumps(_SHELL_ACTION),
},
{"type": "function_call_output", "call_id": "call_shell", "output": "total 0"},
]
def _mantle_peer(marker: str) -> Callable[[Request], Reply]:
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/openai/v1/responses", request.target
assert request.headers["authorization"] == f"Bearer {_TOKEN}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["input"] == _mantle_history(marker), json.dumps(body["input"])
return Reply(body=_RESPONSE)
return respond
@pytest.mark.covers("providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items")
def test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle(gateway: Gateway) -> None:
marker: Final = uuid4().hex
with wire_server(_mantle_peer(marker)) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=_MODEL, api_base=wire.url, api_key=_TOKEN, aws_region_name="us-east-1")
response: Final = gateway.request(
"POST", "/v1/responses", {"model": model, "input": _codex_history(marker), "stream": False}
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["output"] == [
{
**_OUTPUT_MESSAGE,
"phase": None,
"content": [
{"type": "output_text", "text": "mantle wire control", "annotations": [], "logprobs": None}
],
}
], response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/openai/v1/responses")]

View file

@ -0,0 +1,53 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "openai.gpt-5.6-sol"
_API_KEY: Final = "synthetic-mantle-bearer"
_PROMPT: Final = "synthetic long conversation control"
_PROMPT_TOKENS: Final = 1055489
_MODEL_MAXIMUM: Final = 1050000
_RESPONSES_PATH: Final = "/openai/v1/responses"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_OVERFLOW_BODY: Final = json.dumps(
{
"error": {
"code": "validation_error",
"message": f"prompt tokens ({_PROMPT_TOKENS}) exceed model maximum ({_MODEL_MAXIMUM}) for {_BACKEND}",
"type": "invalid_request_error",
}
}
).encode()
def _overflow_peer(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == _RESPONSES_PATH
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _BACKEND
assert _PROMPT in json.dumps(body["input"]), body
return Reply(status=400, body=_OVERFLOW_BODY)
@pytest.mark.covers("other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long")
def test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long(gateway: Gateway) -> None:
with wire_server(_overflow_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
)
assert response.status_code == 400, response.text
error: Final = _JSON_OBJECT.validate_json(response.content)["error"]
assert isinstance(error, dict), response.text
assert error["code"] == "400", response.text
message: Final = error["message"]
assert isinstance(message, str), response.text
assert f"prompt is too long: {_PROMPT_TOKENS} tokens > {_MODEL_MAXIMUM} maximum" in message, response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)]

View file

@ -0,0 +1,88 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
BEDROCK_MODEL: Final = "us.anthropic.claude-opus-5-v1:0"
TOKEN: Final = "synthetic-bedrock-bearer"
SNIPPET: Final = "synthetic snippet about the integration harness"
INTERCEPTED_TURN: Final = (
{"type": "server_tool_use", "id": "srvtoolu_synthetic", "name": "web_search", "input": {"query": "harness docs"}},
{
"type": "web_search_tool_result",
"tool_use_id": "srvtoolu_synthetic",
"content": [
{
"type": "web_search_result",
"url": "https://example.test/harness",
"title": "Harness",
"page_age": None,
"encrypted_content": "",
"snippet": SNIPPET,
},
],
},
{"type": "text", "text": "The harness is documented at example.test"},
)
FLATTENED_TURN: Final = (
{
"type": "text",
"text": f"Web search results for 'harness docs':\n\nTitle: Harness\nURL: https://example.test/harness\nSnippet: {SNIPPET}",
},
{"type": "text", "text": "The harness is documented at example.test"},
)
REPLY: Final = json.dumps(
{
"id": "msg_synthetic_replay",
"type": "message",
"role": "assistant",
"model": BEDROCK_MODEL,
"content": [{"type": "text", "text": "replay accepted"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 30, "output_tokens": 3},
}
).encode()
def bedrock_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == f"/model/{BEDROCK_MODEL}/invoke"
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body["messages"] == [
{"role": "user", "content": "where is the harness documented"},
{"role": "assistant", "content": list(FLATTENED_TURN)},
{"role": "user", "content": "and what does it say"},
], request.body.decode()
assert "tools" not in body, request.body.decode()
return Reply(body=REPLY)
@pytest.mark.covers("providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text")
def test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers(gateway: Gateway) -> None:
with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=f"bedrock/{BEDROCK_MODEL}",
api_key=TOKEN,
api_base=wire.url,
aws_region_name="us-east-1",
)
response: Final = gateway.request(
"POST",
"/v1/messages",
{
"model": model,
"max_tokens": 64,
"messages": [
{"role": "user", "content": "where is the harness documented"},
{"role": "assistant", "content": list(INTERCEPTED_TURN)},
{"role": "user", "content": "and what does it say"},
],
},
headers={"x-api-key": gateway.key, "anthropic-version": "2023-06-01"},
)
assert response.status_code == 200, response.text
assert response.json()["content"] == [{"type": "text", "text": "replay accepted"}], response.text
assert len(wire.drain()) == 1

View file

@ -0,0 +1,42 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.upstream import _aws_event_frame
from integration._support.wire import Reply, Request, wire_server
_MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0"
_EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
_REQUEST_BODY: Final = {"messages": [{"role": "user", "content": [{"text": "synthetic passthrough stream"}]}]}
_EVENTS: Final = (
("messageStart", {"role": "assistant"}),
("contentBlockDelta", {"delta": {"text": "bedrock stream control"}, "contentBlockIndex": 0}),
("messageStop", {"stopReason": "end_turn"}),
("metadata", {"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}}),
)
_STREAM_BYTES: Final = b"".join(_aws_event_frame(kind, payload, "sc", "u") for kind, payload in _EVENTS)
def event_stream_peer(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == f"/model/{_MODEL_ID}/converse-stream"
assert json.loads(request.body)["messages"] == _REQUEST_BODY["messages"]
return Reply(body=_STREAM_BYTES, content_type=_EVENT_STREAM)
@pytest.mark.covers("other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type")
def test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type(gateway: Gateway) -> None:
with wire_server(event_stream_peer) as wire, gateway.scenario() as scenario:
deployment: Final = scenario.model(
model=f"bedrock/{_MODEL_ID}",
api_base=wire.url,
aws_access_key_id="AKIASCRIPTEDPROVIDER",
aws_secret_access_key="scripted-secret",
aws_region_name="us-east-1",
)
response: Final = gateway.request("POST", f"/bedrock/model/{deployment}/converse-stream", _REQUEST_BODY)
assert response.status_code == 200, response.text
assert len(wire.drain()) == 1, response.text
assert response.headers.get("content-type") == _EVENT_STREAM, dict(response.headers)
assert response.content == _STREAM_BYTES, response.text

View file

@ -0,0 +1,92 @@
import json
import os
from pathlib import Path
from typing import Final
import pytest
import yaml
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
MODEL: Final = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
ACCESS_KEY: Final = "AKIAINTEGRATION000002"
FORWARDED_FOR: Final = "203.0.113.5"
RESPONSE: Final = json.dumps(
{"results": [{"index": 1, "relevanceScore": 0.9}, {"index": 0, "relevanceScore": 0.1}]}
).encode()
def signed_headers(authorization: str) -> tuple[str, ...]:
return tuple(authorization.split("SignedHeaders=")[1].split(",")[0].split(";"))
def rerank_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/rerank"
assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/")
assert signed_headers(request.headers["authorization"]) == ("content-type", "host", "x-amz-date"), request.headers[
"authorization"
]
assert request.headers["x-forwarded-for"] == FORWARDED_FOR
body: Final = json.loads(request.body)
assert body["queries"] == [{"textQuery": {"text": "synthetic rerank query"}, "type": "TEXT"}]
assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["modelConfiguration"] == {
"modelArn": "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
}
assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["numberOfResults"] == 2
return Reply(body=RESPONSE)
@pytest.mark.covers("providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned")
def test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature(
gateway: Gateway, tmp_path: Path
) -> None:
empty: Final = tmp_path / "empty-aws-config"
empty.write_text("")
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
configuration["general_settings"]["forward_client_headers_to_llm_api"] = True
path: Final = tmp_path / "forwarding.yaml"
path.write_text(yaml.safe_dump(configuration))
overrides: Final = {
"AWS_CONFIG_FILE": str(empty),
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
"AWS_EC2_METADATA_DISABLED": "true",
"LITELLM_RUST": "false",
}
with wire_server(rerank_peer) as wire:
with (
owned_proxy(
gateway,
tmp_path,
overrides,
config=path,
remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")),
) as candidate,
candidate.scenario() as scenario,
):
model: Final = scenario.model(
model=MODEL,
api_key=None,
api_base=None,
aws_region_name="us-east-1",
aws_bedrock_runtime_endpoint=wire.url,
aws_access_key_id=ACCESS_KEY,
aws_secret_access_key="synthetic-rerank-secret-key-for-testing",
)
response: Final = candidate.request(
"POST",
"/v1/rerank",
{
"model": model,
"query": "synthetic rerank query",
"documents": ["first synthetic document", "second synthetic document"],
"top_n": 2,
},
headers={"x-forwarded-for": FORWARDED_FOR},
)
assert response.status_code == 200, response.text
assert response.json()["results"] == [
{"index": 1, "relevance_score": 0.9},
{"index": 0, "relevance_score": 0.1},
], response.text
assert len(wire.drain()) == 1

View file

@ -0,0 +1,94 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
MODEL: Final = "bedrock/converse/global.anthropic.claude-opus-4-8"
TOKEN: Final = "synthetic-bedrock-bearer"
PROMPT: Final = "How many prime numbers are less than 30? Think it through, then answer with just the number."
RESPONSES_PROMPT: Final = "How many prime numbers are less than 30? Answer with just the number."
REDACTED_DATA: Final = "RWRhY3RlZC1ieS1CZWRyb2Nr"
INPUT_TOKENS: Final = 31
OUTPUT_TOKENS: Final = 257
RESPONSE: Final = json.dumps(
{
"output": {
"message": {
"role": "assistant",
"content": [{"reasoningContent": {"redactedContent": REDACTED_DATA}}, {"text": "10"}],
}
},
"stopReason": "end_turn",
"usage": {
"inputTokens": INPUT_TOKENS,
"outputTokens": OUTPUT_TOKENS,
"totalTokens": INPUT_TOKENS + OUTPUT_TOKENS,
},
"metrics": {"latencyMs": 1},
}
).encode()
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]])
def redacted_thinking_peer(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/model/global.anthropic.claude-opus-4-8/converse"
assert request.headers["authorization"] == f"Bearer {TOKEN}"
body: Final = json.loads(request.body)
assert body["messages"] in (
[{"role": "user", "content": [{"text": PROMPT}]}],
[{"role": "user", "content": [{"text": RESPONSES_PROMPT}]}],
), body
assert body["additionalModelRequestFields"]["thinking"]["type"] == "adaptive", body
return Reply(body=RESPONSE)
@pytest.mark.covers("other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text")
def test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens(gateway: Gateway) -> None:
with wire_server(redacted_thinking_peer) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url
)
chat: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": PROMPT}],
"max_tokens": 4000,
"reasoning_effort": "max",
},
)
assert chat.status_code == 200, chat.text
chat_body: Final = _JSON_OBJECT.validate_json(chat.content)
message: Final = _JSON_OBJECT.validate_python(_JSON_LIST.validate_python(chat_body["choices"])[0]["message"])
assert message["content"] == "10", chat.text
assert message["thinking_blocks"] == [{"type": "redacted_thinking", "data": REDACTED_DATA}], chat.text
usage: Final = _JSON_OBJECT.validate_python(chat_body["usage"])
assert usage["completion_tokens"] == OUTPUT_TOKENS, chat.text
details: Final = _JSON_OBJECT.validate_python(usage["completion_tokens_details"])
assert details == {}, chat.text
assert len(wire.drain()) == 1
responses: Final = gateway.request(
"POST",
"/v1/responses",
{"model": model, "input": RESPONSES_PROMPT, "max_output_tokens": 4000, "reasoning": {"effort": "max"}},
)
assert responses.status_code == 200, responses.text
responses_body: Final = _JSON_OBJECT.validate_json(responses.content)
output: Final = _JSON_LIST.validate_python(responses_body["output"])
reasoning_items: Final = tuple(item for item in output if item["type"] == "reasoning")
assert len(reasoning_items) == 1, responses.text
assert reasoning_items[0]["encrypted_content"] == json.dumps(
[{"type": "redacted_thinking", "data": REDACTED_DATA}], separators=(",", ":")
), responses.text
responses_usage: Final = _JSON_OBJECT.validate_python(responses_body["usage"])
assert responses_usage["output_tokens"] == OUTPUT_TOKENS, responses.text
assert _JSON_OBJECT.validate_python(responses_usage["output_tokens_details"])["reasoning_tokens"] == 0, (
responses.text
)
assert len(wire.drain()) == 1

View file

@ -0,0 +1,62 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "qwen3.7-plus"
_API_KEY: Final = "synthetic-dashscope-key"
_PROMPT: Final = "What is 3^3?"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _completion(identity: str) -> bytes:
return json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": _BACKEND,
"choices": [{"index": 0, "message": {"role": "assistant", "content": "27"}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 17, "completion_tokens": 5, "total_tokens": 22},
}
).encode()
@pytest.mark.covers("other.provider_wire.dashscope.reasoning_effort_reaches_provider")
def test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider(gateway: Gateway) -> None:
identity: Final = f"dashscope-reasoning-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/chat/completions"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
assert _JSON_OBJECT.validate_json(request.body) == {
"model": _BACKEND,
"messages": [{"role": "user", "content": _PROMPT}],
"reasoning_effort": "none",
}
return Reply(body=_completion(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"dashscope/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], "reasoning_effort": "none"},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["id"] == identity
assert payload["choices"] == [
{
"finish_reason": "stop",
"index": 0,
"message": {"role": "assistant", "content": "27", "provider_specific_fields": {"refusal": None}},
"provider_specific_fields": {},
}
]
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]

View file

@ -0,0 +1,129 @@
import json
import uuid
from collections.abc import Mapping
from typing import Final
import pytest
from integration._support.client import Gateway, eventually
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
_BACKEND: Final = "databricks-glm-5-2"
_API_KEY: Final = "synthetic-databricks-key"
_PROMPT: Final = "Summarise the cached briefing in one sentence."
_PROVIDER_USAGE: Final[Mapping[str, JsonValue]] = {
"prompt_tokens": 12011,
"completion_tokens": 8,
"total_tokens": 12019,
"cache_read_input_tokens": 12002,
"cache_creation_input_tokens": 0,
}
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
class _PromptTokensDetails(BaseModel):
model_config = ConfigDict(extra="ignore")
cached_tokens: int | None = None
class _Usage(BaseModel):
model_config = ConfigDict(extra="ignore")
prompt_tokens: int
completion_tokens: int
total_tokens: int
prompt_tokens_details: _PromptTokensDetails | None = None
class _Delta(BaseModel):
model_config = ConfigDict(extra="ignore")
content: str | None = None
class _Choice(BaseModel):
model_config = ConfigDict(extra="ignore")
delta: _Delta
class _Chunk(BaseModel):
model_config = ConfigDict(extra="ignore")
id: str
choices: tuple[_Choice, ...]
usage: _Usage | None = None
def _frame(identity: str, choices: list[Mapping[str, object]], usage: Mapping[str, JsonValue] | None = None) -> bytes:
value: Final = {
"id": identity,
"object": "chat.completion.chunk",
"created": 1,
"model": _BACKEND,
"choices": choices,
**({} if usage is None else {"usage": usage}),
}
return b"data: " + json.dumps(value).encode() + b"\n\n"
@pytest.mark.covers("other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log")
def test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log(gateway: Gateway) -> None:
identity: Final = f"databricks-stream-{uuid.uuid4().hex}"
frames: Final = (
_frame(
identity, [{"index": 0, "delta": {"role": "assistant", "content": "The briefing "}, "finish_reason": None}]
),
_frame(identity, [{"index": 0, "delta": {"content": "is short."}, "finish_reason": None}]),
_frame(identity, [{"index": 0, "delta": {}, "finish_reason": "stop"}]),
_frame(identity, [], usage=_PROVIDER_USAGE),
b"data: [DONE]\n\n",
)
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/chat/completions"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _BACKEND
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
assert body["stream"] is True
return Reply(content_type="text/event-stream", chunks=frames)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"databricks/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
with gateway.client.stream(
"POST",
"/v1/chat/completions",
json={
"model": model,
"messages": [{"role": "user", "content": _PROMPT}],
"stream": True,
"stream_options": {"include_usage": True},
},
headers={"Authorization": f"Bearer {gateway.key}"},
) as response:
assert response.status_code == 200, response.read()
lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: "))
assert lines[-1] == "data: [DONE]", lines
chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1])
assert {chunk.id for chunk in chunks} == {identity}
assert (
"".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
== "The briefing is short."
)
usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None)
assert len(usages) == 1, lines
assert (
usages[0].prompt_tokens,
usages[0].completion_tokens,
usages[0].total_tokens,
usages[0].prompt_tokens_details.cached_tokens if usages[0].prompt_tokens_details is not None else None,
) == (12011, 8, 12019, 12002), lines
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
rows: Final = eventually(
lambda: read_rows(
'SELECT prompt_tokens, completion_tokens, total_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
(identity,),
),
lambda values: len(values) == 1,
seconds=70,
)
assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"], rows[0]["total_tokens"]) == (12011, 8, 12019)

View file

@ -0,0 +1,92 @@
import base64
import json
import uuid
from pathlib import Path
from typing import Final
from urllib.parse import parse_qs
import pytest
from integration._support.client import Gateway
from integration._support.process import owned_proxy
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_MODEL: Final = "databricks/synthetic-vendor.chat-model.v1"
_CLIENT_ID: Final = "synthetic-databricks-client-id"
_CLIENT_SECRET: Final = "synthetic-databricks-client-secret"
_ACCESS_TOKEN: Final = "synthetic-databricks-oauth-token"
_PROMPT: Final = "Which workspace issued this token?"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _basic_credentials(client_id: str, client_secret: str) -> str:
return "Basic " + base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
def _completion(identity: str) -> bytes:
return json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": _MODEL.removeprefix("databricks/"),
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "the workspace origin"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13},
}
).encode()
@pytest.mark.covers("other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base")
def test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin(
gateway: Gateway, tmp_path: Path
) -> None:
identity: Final = f"databricks-oauth-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
if request.target == "/oidc/v1/token":
assert request.method == "POST"
assert request.headers["authorization"] == _basic_credentials(_CLIENT_ID, _CLIENT_SECRET)
assert request.headers["content-type"] == "application/x-www-form-urlencoded"
assert parse_qs(request.body.decode()) == {"grant_type": ["client_credentials"], "scope": ["all-apis"]}
return Reply(
body=json.dumps({"access_token": _ACCESS_TOKEN, "token_type": "Bearer", "expires_in": 3600}).encode()
)
if request.target == "/ai-gateway/mlflow/v1/chat/completions":
assert request.method == "POST"
assert request.headers["authorization"] == f"Bearer {_ACCESS_TOKEN}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _MODEL.removeprefix("databricks/")
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
return Reply(body=_completion(identity))
return Reply(status=401, body=json.dumps({"error": f"unauthenticated path {request.target}"}).encode())
overrides: Final = {"DATABRICKS_CLIENT_ID": _CLIENT_ID, "DATABRICKS_CLIENT_SECRET": _CLIENT_SECRET}
with wire_server(respond) as wire, owned_proxy(gateway, tmp_path, overrides) as candidate:
with candidate.scenario() as scenario:
model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/ai-gateway/mlflow/v1", api_key=None)
response: Final = candidate.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["id"] == identity
assert payload["choices"] == [
{
"finish_reason": "stop",
"index": 0,
"message": {"content": "the workspace origin", "role": "assistant"},
}
]
assert payload["usage"] == {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}
assert [(request.method, request.target) for request in wire.drain()] == [
("POST", "/oidc/v1/token"),
("POST", "/ai-gateway/mlflow/v1/chat/completions"),
]

View file

@ -0,0 +1,40 @@
from typing import Final
import httpx
import pytest
from pydantic import JsonValue
from tests.integration._support.client import JSON_OBJECT, Gateway, object_value
_VISION_MODEL: Final = "deepseek-v4-flash-vision-exp"
_API_KEY: Final = "synthetic-deepseek-key"
_VISION_CONTENT: Final[JsonValue] = [
{"type": "text", "text": "what is in this image?"},
{"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}},
]
@pytest.mark.covers("other.provider_wire.deepseek.vision_image_content_list_reaches_provider")
def test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text(gateway: Gateway) -> None:
with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream:
upstream.get("/__observations").raise_for_status()
model: Final = scenario.model(
model=f"deepseek/{_VISION_MODEL}",
api_key=_API_KEY,
model_info={"mode": "chat", "supports_vision": True},
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _VISION_CONTENT}]},
)
assert response.status_code == 200, response.text
observations: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]
assert isinstance(observations, list)
assert len(observations) == 1, response.text
observed: Final = object_value(observations[0])
assert observed["path"] == "/v1/chat/completions", response.text
assert observed["authorization"] == f"Bearer {_API_KEY}", response.text
body: Final = object_value(observed["body"])
assert body["model"] == _VISION_MODEL, response.text
assert body["messages"] == [{"role": "user", "content": _VISION_CONTENT}], response.text

View file

@ -0,0 +1,84 @@
import json
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_ROUTER_SLUG: Final = "routers/glm-latest"
_ROUTER_RESOURCE: Final = "accounts/fireworks/routers/glm-latest"
_API_KEY: Final = "synthetic-fireworks-key"
_PROMPT: Final = "route me through the router"
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _provider_body(request: Request, target: str) -> dict[str, JsonValue]:
assert request.method == "POST"
assert request.target == target
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
return _JSON_OBJECT.validate_json(request.body)
@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name")
def test_fireworks_router_slug_chat_sends_router_resource_not_models_path(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
body: Final = _provider_body(request, "/chat/completions")
assert body["model"] == _ROUTER_RESOURCE, body
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
return Reply(
body=json.dumps(
{
"id": "fw-router-chat",
"object": "chat.completion",
"created": 1,
"model": _ROUTER_RESOURCE,
"choices": [
{"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"}
],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["choices"] == [
{"finish_reason": "stop", "index": 0, "message": {"role": "assistant", "content": "routed"}}
]
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name")
def test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
body: Final = _provider_body(request, "/completions")
assert body["model"] == _ROUTER_RESOURCE, body
assert body["prompt"] == _PROMPT
return Reply(
body=json.dumps(
{
"id": "fw-router-text",
"object": "text_completion",
"created": 1,
"model": _ROUTER_RESOURCE,
"choices": [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}],
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request("POST", "/v1/completions", {"model": model, "prompt": _PROMPT})
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["choices"] == [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}]
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/completions")]

View file

@ -0,0 +1,66 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "gpt-5.4-mini"
_API_KEY: Final = "synthetic-openai-key"
_PROMPT: Final = "Summarize this conversation in one sentence."
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
def _completion(identity: str, content: str) -> bytes:
return json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": _BACKEND,
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 19, "completion_tokens": 7, "total_tokens": 26},
}
).encode()
@pytest.mark.covers("providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire")
def test_openai_chat_tool_choice_without_tools_is_not_forwarded(gateway: Gateway) -> None:
identity: Final = f"openai-toolless-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/chat/completions"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
body: Final = _JSON_OBJECT.validate_json(request.body)
assert body["model"] == _BACKEND
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
assert "tool_choice" not in body, body
assert "tools" not in body, body
return Reply(body=_completion(identity, "One sentence."))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], "tool_choice": "none"},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["id"] == identity
assert payload["choices"] == [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "One sentence.",
"provider_specific_fields": {"refusal": None},
},
"provider_specific_fields": {},
}
]
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]

View file

@ -0,0 +1,75 @@
import json
from email.message import Message
from email.parser import BytesParser
from email.policy import HTTP
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import BaseModel
_PNG_BYTES: Final = (
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff"
b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82"
)
_PROMPT: Final = "turn the red circle green"
_EDITED_IMAGE_B64: Final = "aW50ZWdyYXRpb24tZWRpdGVkLWltYWdl"
class _Image(BaseModel):
b64_json: str
class _ImageResponse(BaseModel):
data: tuple[_Image, ...]
def _multipart_parts(request: Request) -> tuple[Message, ...]:
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
assert parsed.is_multipart(), request.headers["content-type"]
return tuple(parsed.iter_parts())
def _text_fields(parts: tuple[Message, ...]) -> dict[str, str]:
return {
part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode()
for part in parts
if part.get_filename() is None
}
def _file_fields(parts: tuple[Message, ...]) -> dict[str, bytes]:
return {
part.get_param("name", header="content-disposition"): part.get_payload(decode=True)
for part in parts
if part.get_filename() is not None
}
@pytest.mark.covers("other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields")
def test_openai_compatible_image_edit_forwards_seed_form_field_to_backend(gateway: Gateway) -> None:
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/v1/images/edits"
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
parts: Final = _multipart_parts(request)
assert _text_fields(parts) == {"model": "gpt-image-1", "prompt": _PROMPT, "seed": "42"}
assert _file_fields(parts) == {"image[]": _PNG_BYTES}
return Reply(body=json.dumps({"created": 1700000000, "data": [{"b64_json": _EDITED_IMAGE_B64}]}).encode())
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/gpt-image-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
)
response: Final = gateway.request_multipart(
"/v1/images/edits",
{"model": model, "prompt": _PROMPT, "seed": "42"},
{"image": ("red_circle.png", _PNG_BYTES, "image/png")},
)
assert response.status_code == 200, response.text
payload: Final = _ImageResponse.model_validate_json(response.content)
assert [image.b64_json for image in payload.data] == [_EDITED_IMAGE_B64], response.text
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/images/edits")]

View file

@ -0,0 +1,64 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
@pytest.mark.covers("other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length")
def test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out(gateway: Gateway) -> None:
identity: Final = "responses-incomplete-" + uuid.uuid4().hex
def respond(request: Request) -> Reply:
assert request.method == "POST" and request.target == "/responses", request.target
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
body: Final = json.loads(request.body)
assert body["model"] == "gpt-5.3-codex"
assert body["max_output_tokens"] == 16
assert body["reasoning"] == {"effort": "high"}
assert body["input"] == [
{
"type": "message",
"role": "user",
"content": [{"type": "input_text", "text": f"explain the plan in detail {identity}"}],
}
]
return Reply(
body=json.dumps(
{
"id": f"resp_{identity}",
"object": "response",
"created_at": 1789788253,
"status": "incomplete",
"incomplete_details": {"reason": "max_output_tokens"},
"model": "gpt-5.3-codex",
"output": [{"type": "reasoning", "id": f"rs_{identity}", "summary": []}],
"usage": {"input_tokens": 12, "output_tokens": 16, "total_tokens": 28},
}
).encode()
)
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(
model="openai/responses/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key"
)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{
"model": model,
"messages": [{"role": "user", "content": f"explain the plan in detail {identity}"}],
"reasoning_effort": "high",
"max_completion_tokens": 16,
},
)
assert response.status_code == 200, response.text
body: Final = response.json()
assert len(wire.drain()) == 1
assert [choice["finish_reason"] for choice in body["choices"]] == ["length"], response.text
assert body["choices"][0]["message"]["content"] == "", response.text
assert body["choices"][0]["message"]["role"] == "assistant", response.text
assert body["usage"]["prompt_tokens"] == 12 and body["usage"]["completion_tokens"] == 16, response.text
assert body["usage"]["total_tokens"] == 28, response.text

View file

@ -0,0 +1,86 @@
import json
import uuid
from typing import Final
import pytest
from integration._support.client import Gateway
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue, TypeAdapter
_BACKEND: Final = "deepseek-v4-pro"
_API_KEY: Final = "synthetic-tencent-key"
_PROMPT: Final = "What is 17 + 26? Answer with just the number."
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
_REASONING_REQUESTS: Final[tuple[tuple[str, dict[str, JsonValue], dict[str, JsonValue]], ...]] = (
("thinking_enabled", {"thinking": {"type": "enabled"}}, {"type": "enabled"}),
("reasoning_effort_none", {"reasoning_effort": "none"}, {"type": "disabled"}),
)
def _completion(identity: str) -> bytes:
return json.dumps(
{
"id": identity,
"object": "chat.completion",
"created": 1,
"model": _BACKEND,
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64},
}
).encode()
@pytest.mark.covers("other.provider_wire.tencent.thinking_reaches_provider_in_request_body")
@pytest.mark.parametrize(
("reasoning_params", "expected_thinking"),
tuple(case[1:] for case in _REASONING_REQUESTS),
ids=tuple(case[0] for case in _REASONING_REQUESTS),
)
def test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request(
gateway: Gateway, reasoning_params: dict[str, JsonValue], expected_thinking: dict[str, JsonValue]
) -> None:
identity: Final = f"tencent-thinking-{uuid.uuid4().hex}"
def respond(request: Request) -> Reply:
assert request.method == "POST"
assert request.target == "/chat/completions"
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
assert request.headers["content-type"] == "application/json"
assert _JSON_OBJECT.validate_json(request.body) == {
"model": _BACKEND,
"messages": [{"role": "user", "content": _PROMPT}],
"thinking": expected_thinking,
}
return Reply(body=_completion(identity))
with wire_server(respond) as wire, gateway.scenario() as scenario:
model: Final = scenario.model(model=f"tencent/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
response: Final = gateway.request(
"POST",
"/v1/chat/completions",
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], **reasoning_params},
)
assert response.status_code == 200, response.text
payload: Final = _JSON_OBJECT.validate_json(response.content)
assert payload["id"] == identity
assert payload["choices"] == [
{
"finish_reason": "stop",
"index": 0,
"message": {
"role": "assistant",
"content": "43",
"reasoning_content": "17 plus 26 is 43.",
"provider_specific_fields": {"refusal": None},
},
"provider_specific_fields": {},
}
]
assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]

Some files were not shown because too many files have changed in this diff Show more