mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
chore: merge origin/main into litellm_config_vector_stores_read_only
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
commit
74fe1dad49
126 changed files with 7630 additions and 1009 deletions
33
.github/workflows/compat-matrix-image.yml
vendored
Normal file
33
.github/workflows/compat-matrix-image.yml
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
name: Compat Matrix Image
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- tests/e2e/claude_code/cron_vm/**
|
||||
- .github/workflows/compat-matrix-image.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
compat-matrix-image:
|
||||
name: compat-matrix-image
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build the Render cron image
|
||||
run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
|
||||
|
||||
- name: Run the pinned binaries as the cron user
|
||||
run: |
|
||||
docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version'
|
||||
10
.github/workflows/test-rust.yml
vendored
10
.github/workflows/test-rust.yml
vendored
|
|
@ -105,6 +105,16 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install Python dependencies for the bridge tests
|
||||
working-directory: .
|
||||
run: |
|
||||
uv sync --frozen --no-install-project
|
||||
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
||||
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG PROXY_EXTRAS_SOURCE=published
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.69"
|
||||
version = "0.1.70"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.69"
|
||||
version = "0.1.70"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
||||
ARG PGBOUNCER_VERSION=1.25.2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.100"
|
||||
version = "0.4.101"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.100"
|
||||
version = "0.4.101"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
21
litellm-rust/Cargo.lock
generated
21
litellm-rust/Cargo.lock
generated
|
|
@ -2946,6 +2946,7 @@ name = "litellm-core-utils"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -3075,6 +3076,7 @@ dependencies = [
|
|||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
|
|
@ -3093,6 +3095,7 @@ dependencies = [
|
|||
"litellm-callbacks-legacy-python",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
@ -3100,6 +3103,7 @@ dependencies = [
|
|||
"litellm-secrets-aws",
|
||||
"litellm-secrets-types",
|
||||
"litellm-token-counter",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"pyo3",
|
||||
"pyo3-async-runtimes",
|
||||
|
|
@ -3172,11 +3176,11 @@ dependencies = [
|
|||
"litellm-auth-aws",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3208,6 +3212,7 @@ dependencies = [
|
|||
"base64 0.22.1",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"moka",
|
||||
"percent-encoding",
|
||||
"reqwest 0.12.28",
|
||||
|
|
@ -3216,7 +3221,6 @@ dependencies = [
|
|||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3331,6 +3335,19 @@ dependencies = [
|
|||
"tiktoken-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"percent-encoding",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ license = "MIT"
|
|||
repository = "https://github.com/BerriAI/litellm"
|
||||
|
||||
[workspace.dependencies]
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
tracing = "0.1"
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-host = { path = "crates/host" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-types.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,109 +1 @@
|
|||
use fancy_regex::Regex;
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn minimum_custom_key_length() -> usize {
|
||||
std::env::var("MINIMUM_CUSTOM_KEY_LENGTH")
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
}
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
Self { pattern }
|
||||
}
|
||||
|
||||
/// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS")
|
||||
.is_ok_and(|value| value.eq_ignore_ascii_case("true"));
|
||||
(!disabled).then(|| Self::new(minimum_custom_key_length()))
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.pattern.replace_all(value, REDACTED).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
pub use litellm_tracing::{REDACTED, SecretRedactor};
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"]
|
|||
tiktoken = ["litellm-token-counter/tiktoken"]
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-host.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use crate::logger::run_async;
|
||||
use litellm_cache_response::PartialHits;
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py};
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_auth_aws::AwsAuthConfig;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization};
|
||||
use litellm_cache_redis::{RedisNode, RedisTopology};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
|
|
|
|||
|
|
@ -470,7 +470,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move {
|
||||
service
|
||||
|
|
@ -495,7 +495,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_lookup(&request, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -550,7 +550,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store(&request, response, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -619,7 +619,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store_batch(entries, now()).await },
|
||||
super::cache_error,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::{collections::VecDeque, time::Duration};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyException, PyRuntimeError},
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ pub(crate) fn call_config(
|
|||
.without_missing_files(&|path: &Path| path.exists());
|
||||
let resolution = Resolution::from(&settings);
|
||||
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
|
||||
PythonSettings::warn(py, &unsupported.to_string())?;
|
||||
crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}"));
|
||||
}
|
||||
Ok(resolution.config)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ mod credentials;
|
|||
mod diagnostics;
|
||||
mod errors;
|
||||
mod http;
|
||||
mod logger;
|
||||
mod marshal;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
|
|
@ -25,6 +26,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::errors::{RustBridgeDeclined, RustUpstreamError};
|
||||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{
|
||||
|
|
@ -87,6 +90,7 @@ mod tests {
|
|||
"chat_completions",
|
||||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"NativeDiagnosticProcessor",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
|
|
|
|||
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
use std::future::Future;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use serde::Serialize;
|
||||
|
||||
pub(crate) fn run_sync<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_async<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
|
||||
pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, Step},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_tracing::Logger;
|
||||
use pyo3::Python;
|
||||
|
||||
pub(crate) struct LoggedMachine<M> {
|
||||
machine: M,
|
||||
logger: OnceLock<Logger>,
|
||||
}
|
||||
|
||||
impl<M> LoggedMachine<M> {
|
||||
pub(crate) fn new(machine: M) -> Self {
|
||||
Self {
|
||||
machine,
|
||||
logger: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<M: Machine> Machine for LoggedMachine<M> {
|
||||
type Route = M::Route;
|
||||
type Complete = M::Complete;
|
||||
|
||||
fn resume(&mut self, result: Option<HostResult<Self::Route>>) -> Step<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result))))
|
||||
}
|
||||
|
||||
fn interrupt(
|
||||
&mut self,
|
||||
failure: HostFailure<<Self::Route as Route>::Error>,
|
||||
) -> Interrupted<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
|
||||
}
|
||||
}
|
||||
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
mod execution;
|
||||
mod machine;
|
||||
|
||||
pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
|
||||
pub(crate) use machine::LoggedMachine;
|
||||
|
||||
use litellm_host_python::Pythonized;
|
||||
use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink};
|
||||
use pyo3::exceptions::PyRuntimeError;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.logger";
|
||||
type NativeDiagnosticOutput = (String, Option<String>, Option<String>, Vec<String>, bool);
|
||||
|
||||
#[pyclass]
|
||||
pub(crate) struct NativeDiagnosticProcessor {
|
||||
inner: Processor,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl NativeDiagnosticProcessor {
|
||||
#[new]
|
||||
fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
inner: Processor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_text(&self, text: &str) -> PyResult<String> {
|
||||
self.inner.redact_text(text).map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_structured_text(key, text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_client_message(&self, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_client_message(text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (message, exception, stack, leaves, policy))]
|
||||
fn process_diagnostic(
|
||||
&self,
|
||||
message: String,
|
||||
exception: Option<String>,
|
||||
stack: Option<String>,
|
||||
leaves: Vec<(Option<String>, String)>,
|
||||
policy: (bool, i64, i64),
|
||||
) -> PyResult<NativeDiagnosticOutput> {
|
||||
let input = DiagnosticInput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
};
|
||||
let policy = Policy {
|
||||
redact: policy.0,
|
||||
base64_limit: policy.1,
|
||||
text_limit: policy.2,
|
||||
};
|
||||
self.inner
|
||||
.process_diagnostic(&input, policy)
|
||||
.map(|output| {
|
||||
(
|
||||
output.message,
|
||||
output.exception,
|
||||
output.stack,
|
||||
output.leaves,
|
||||
output.changed,
|
||||
)
|
||||
})
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn scrub_access_arguments(&self, arguments: Vec<String>) -> PyResult<Vec<String>> {
|
||||
self.inner
|
||||
.scrub_access_arguments(&arguments)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
}
|
||||
|
||||
fn processing_error(_: fancy_regex::Error) -> PyErr {
|
||||
PyRuntimeError::new_err("diagnostic processing failed")
|
||||
}
|
||||
|
||||
struct PythonSink {
|
||||
correlation: (String, String),
|
||||
}
|
||||
|
||||
fn level(level: &Level) -> u8 {
|
||||
match *level {
|
||||
Level::ERROR => 40,
|
||||
Level::WARN => 30,
|
||||
Level::INFO => 20,
|
||||
Level::DEBUG | Level::TRACE => 10,
|
||||
}
|
||||
}
|
||||
|
||||
fn report<T: Default>(py: Python<'_>, result: PyResult<T>) -> T {
|
||||
match result {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
error.write_unraisable(py, None);
|
||||
T::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink for PythonSink {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
|
||||
.and_then(|enabled| enabled.extract()),
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE).and_then(|module| {
|
||||
module
|
||||
.call_method1(
|
||||
"emit",
|
||||
(
|
||||
level(record.metadata.level()),
|
||||
&record.message,
|
||||
record.metadata.file().unwrap_or_default(),
|
||||
record.metadata.line().unwrap_or_default(),
|
||||
record.metadata.target(),
|
||||
Pythonized(&record.fields),
|
||||
(&self.correlation.0, &self.correlation.1),
|
||||
),
|
||||
)
|
||||
.map(|_| ())
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture(py: Python<'_>) -> Logger {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method0("context"))
|
||||
.and_then(|value| value.extract())
|
||||
.map(|correlation| Logger::new(PythonSink { correlation })),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
use std::{process::Command, task::Poll};
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
|
||||
route::Route,
|
||||
};
|
||||
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
struct DiagnosticMachine;
|
||||
|
||||
impl Route for DiagnosticMachine {
|
||||
type Response = ();
|
||||
type Error = String;
|
||||
type Op = ();
|
||||
type OpResult = ();
|
||||
type Chunk = ();
|
||||
type StreamHead = ();
|
||||
}
|
||||
|
||||
impl Machine for DiagnosticMachine {
|
||||
type Route = Self;
|
||||
type Complete = ();
|
||||
|
||||
fn resume(&mut self, _: Option<HostResult<Self>>) -> Step<'_, Self> {
|
||||
litellm_tracing::warn!("machine started");
|
||||
Box::pin(async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("machine warning");
|
||||
Ok(MachineStep::Complete(()))
|
||||
})
|
||||
}
|
||||
|
||||
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
|
||||
Box::pin(async {
|
||||
litellm_tracing::warn!("machine interrupted");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
|
||||
let mut future = Box::pin(async move {
|
||||
machine
|
||||
.resume(None)
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)?;
|
||||
machine
|
||||
.interrupt(HostFailure::Error("stop".into()))
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)
|
||||
});
|
||||
assert!(matches!(
|
||||
litellm_host_python::poll_async_value(py, future.as_mut())?,
|
||||
Poll::Pending
|
||||
));
|
||||
litellm_host_python::run_async_value(py, future)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn warning(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::warn!(attempt = 3, retry = true, "native warning");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn levels(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::trace!("trace");
|
||||
litellm_tracing::debug!("debug");
|
||||
litellm_tracing::info!("info");
|
||||
litellm_tracing::warn!("warn");
|
||||
litellm_tracing::error!("error");
|
||||
litellm_tracing::warn!(target: "unrelated_transport", "private wire data");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
super::run_async_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("async warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("sync warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
litellm_tracing::warn!("failure diagnostic");
|
||||
Err(pyo3::exceptions::PyValueError::new_err("request failed"))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn http_warning(py: Python<'_>) -> PyResult<()> {
|
||||
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
|
||||
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
|
||||
let output = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
std::thread::current().name().unwrap(),
|
||||
"--nocapture",
|
||||
])
|
||||
.env("LITELLM_LOGGER_TEST_PROCESS", "1")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals
|
||||
.set_item(
|
||||
"repo_root",
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"machine_warning",
|
||||
wrap_pyfunction!(machine_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_failure",
|
||||
wrap_pyfunction!(synchronous_failure, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("levels", wrap_pyfunction!(levels, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("warning", wrap_pyfunction!(warning, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"asynchronous_warning",
|
||||
wrap_pyfunction!(asynchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_warning",
|
||||
wrap_pyfunction!(synchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
|
||||
.unwrap();
|
||||
let importable = py
|
||||
.eval(
|
||||
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap()
|
||||
.is_truthy()
|
||||
.unwrap();
|
||||
if !importable {
|
||||
eprintln!("SKIP: litellm package dependencies are not importable in this interpreter");
|
||||
return;
|
||||
}
|
||||
py.run(c"
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
sys.path.insert(0, repo_root)
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, session_id_var, trace_id_var
|
||||
|
||||
class Capture(logging.Handler):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.records = []
|
||||
def emit(self, record):
|
||||
self.records.append(record)
|
||||
warning()
|
||||
|
||||
class Broken(logging.Handler):
|
||||
def emit(self, record):
|
||||
raise ValueError('handler failed')
|
||||
|
||||
capture = Capture()
|
||||
old_handlers = verbose_logger.handlers
|
||||
old_level = verbose_logger.level
|
||||
old_correlation = litellm.request_correlation_in_logs
|
||||
old_curve = litellm.ssl_ecdh_curve
|
||||
old_unraisable = sys.unraisablehook
|
||||
failures = []
|
||||
try:
|
||||
verbose_logger.handlers = [capture]
|
||||
litellm.request_correlation_in_logs = True
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert capture.records == []
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
record = capture.records[0]
|
||||
assert record.getMessage() == 'native warning'
|
||||
assert record.levelno == logging.WARNING
|
||||
assert record.rust_fields == {'attempt': 3, 'retry': True}
|
||||
assert record.pathname.endswith('logger/tests.rs')
|
||||
assert record.lineno > 0
|
||||
assert record.rust_target.endswith('logger::tests')
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
|
||||
async def request(name):
|
||||
session = session_id_var.set(name)
|
||||
trace = trace_id_var.set('trace-' + name)
|
||||
try:
|
||||
await asynchronous_warning()
|
||||
await machine_warning()
|
||||
synchronous_warning()
|
||||
assert session_id_var.get() == name
|
||||
assert trace_id_var.get() == 'trace-' + name
|
||||
finally:
|
||||
trace_id_var.reset(trace)
|
||||
session_id_var.reset(session)
|
||||
|
||||
async def concurrent():
|
||||
await asyncio.gather(request('first'), request('second'))
|
||||
|
||||
asyncio.run(concurrent())
|
||||
assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted(
|
||||
(message, name, 'trace-' + name)
|
||||
for name in ('first', 'second')
|
||||
for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted')
|
||||
)
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
before_levels = len(capture.records)
|
||||
levels()
|
||||
assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [
|
||||
('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO),
|
||||
('warn', logging.WARNING), ('error', logging.ERROR),
|
||||
]
|
||||
|
||||
before = len(capture.records)
|
||||
litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve'
|
||||
http_warning()
|
||||
http_warning()
|
||||
assert len(capture.records) == before + 1
|
||||
assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage()
|
||||
assert capture.records[-1].pathname.endswith('http.rs')
|
||||
|
||||
verbose_logger.handlers = [Broken()]
|
||||
sys.unraisablehook = failures.append
|
||||
warning()
|
||||
assert len(failures) == 1
|
||||
assert str(failures[0].exc_value) == 'handler failed'
|
||||
try:
|
||||
synchronous_failure()
|
||||
except ValueError as error:
|
||||
assert str(error) == 'request failed'
|
||||
else:
|
||||
raise AssertionError('request failure was lost')
|
||||
assert len(failures) == 2
|
||||
finally:
|
||||
sys.unraisablehook = old_unraisable
|
||||
verbose_logger.handlers = old_handlers
|
||||
verbose_logger.setLevel(old_level)
|
||||
litellm.request_correlation_in_logs = old_correlation
|
||||
litellm.ssl_ecdh_curve = old_curve
|
||||
", Some(&locals), Some(&locals)).unwrap();
|
||||
});
|
||||
}
|
||||
|
|
@ -44,11 +44,6 @@ impl PythonSettings {
|
|||
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
|
||||
Snapshot { group: self, value }
|
||||
}
|
||||
|
||||
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
|
||||
py.import(MODULE)?.getattr("warn")?.call1((message,))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::audio_transcription::{
|
||||
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::chat_completions::{
|
||||
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
|
||||
types::ChatCompletionsRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use litellm_types::utils::ChatCompletionsResponse;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ fn run_messages(
|
|||
py,
|
||||
SURFACE,
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
messages_machine(),
|
||||
crate::logger::LoggedMachine::new(messages_machine()),
|
||||
MessagesRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ fn run_ocr(
|
|||
py,
|
||||
if asynchronous { ASYNC_SURFACE } else { SURFACE },
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
ocr_machine(client),
|
||||
crate::logger::LoggedMachine::new(ocr_machine(client)),
|
||||
OcrRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection {
|
|||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let headers = marshal_headers(headers)?;
|
||||
let timeout = optional_timeout(timeout_seconds);
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
|
||||
.await
|
||||
.map_err(responses_error_to_pyerr)?;
|
||||
|
|
@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner
|
||||
.send_text(text)
|
||||
.await
|
||||
|
|
@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.recv_text().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
||||
fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.close().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_host_python::enter_native;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true
|
|||
litellm-core-utils.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
veil.workspace = true
|
||||
aws-sdk-kms = "1.120.0"
|
||||
aws-sdk-secretsmanager = "1.117.0"
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 {
|
|||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("secret created but replication failed");
|
||||
litellm_tracing::warn!("secret created but replication failed");
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ reqwest.workspace = true
|
|||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
veil.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
percent-encoding = "2.3"
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ impl CyberArkSecretManager {
|
|||
.unwrap_or(true);
|
||||
let mut builder = reqwest::Client::builder();
|
||||
if !verify {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
|
||||
);
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
|
|
@ -259,11 +259,13 @@ impl CyberArkSecretManager {
|
|||
.endpoint
|
||||
.join(&format!("policies/{}/policy/root", self.account));
|
||||
let Ok(policy_url) = policy_url else {
|
||||
tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
litellm_tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
return;
|
||||
};
|
||||
let Ok(authorization) = self.authorization_header(context).await else {
|
||||
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
|
||||
litellm_tracing::warn!(
|
||||
"Could not authenticate while ensuring CyberArk variable exists"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let body = format!(
|
||||
|
|
@ -288,19 +290,19 @@ impl CyberArkSecretManager {
|
|||
reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
|
||||
) =>
|
||||
{
|
||||
tracing::debug!(
|
||||
litellm_tracing::debug!(
|
||||
"CyberArk variable policy already exists or conflicts: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Ok(response) => {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"Could not ensure CyberArk variable exists: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -324,7 +326,7 @@ impl CyberArkSecretManager {
|
|||
_recovery_window_in_days: Option<u32>,
|
||||
_context: &SecretOperationContext,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
|
||||
);
|
||||
self.secrets.invalidate(name).await;
|
||||
|
|
|
|||
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
26
litellm-rust/crates/tracing/README.md
Normal file
26
litellm-rust/crates/tracing/README.md
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Native diagnostic tracing
|
||||
|
||||
`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber
|
||||
|
||||
Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work
|
||||
|
||||
Bindings implement `litellm_tracing::Sink` to connect events to their host runtime:
|
||||
|
||||
```rust
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
```
|
||||
|
||||
Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value
|
||||
|
||||
Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion
|
||||
|
||||
The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger
|
||||
|
||||
Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events
|
||||
|
||||
A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue
|
||||
|
||||
This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate
|
||||
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
use std::{
|
||||
cell::Cell,
|
||||
fmt,
|
||||
future::{Future, poll_fn},
|
||||
pin::pin,
|
||||
};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use tracing::{
|
||||
Dispatch, Event, Subscriber,
|
||||
field::{Field, Visit},
|
||||
subscriber::Interest,
|
||||
};
|
||||
use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
|
||||
|
||||
mod processing;
|
||||
mod redaction;
|
||||
|
||||
pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor};
|
||||
pub use redaction::{REDACTED, SecretRedactor};
|
||||
pub use tracing::{Level, Metadata, debug, error, info, trace, warn};
|
||||
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Record {
|
||||
pub metadata: &'static Metadata<'static>,
|
||||
pub message: String,
|
||||
pub fields: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Logger {
|
||||
dispatch: Dispatch,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
pub fn new(sink: impl Sink) -> Self {
|
||||
Self {
|
||||
dispatch: Dispatch::new(Registry::default().with(Output(sink))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
if EMITTING.get() {
|
||||
return operation();
|
||||
}
|
||||
tracing::dispatcher::with_default(&self.dispatch, operation)
|
||||
}
|
||||
|
||||
pub fn instrument<F: Future>(&self, future: F) -> impl Future<Output = F::Output> + use<F> {
|
||||
let logger = self.clone();
|
||||
async move {
|
||||
let mut future = pin!(future);
|
||||
poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static EMITTING: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
struct Emitting;
|
||||
|
||||
impl Emitting {
|
||||
fn enter() -> Option<Self> {
|
||||
EMITTING.with(|active| (!active.replace(true)).then_some(Self))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Emitting {
|
||||
fn drop(&mut self) {
|
||||
EMITTING.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
struct Output<S>(S);
|
||||
|
||||
impl<S: Sink, R: Subscriber> Layer<R> for Output<S> {
|
||||
fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {
|
||||
Interest::sometimes()
|
||||
}
|
||||
|
||||
fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return false;
|
||||
};
|
||||
self.0.enabled(metadata)
|
||||
}
|
||||
|
||||
fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return;
|
||||
};
|
||||
let mut record = Record {
|
||||
metadata: event.metadata(),
|
||||
message: String::new(),
|
||||
fields: Map::new(),
|
||||
};
|
||||
event.record(&mut record);
|
||||
self.0.emit(&record);
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
fn field(&mut self, field: &Field, value: Value) {
|
||||
if field.name() == "message" {
|
||||
self.message = match value {
|
||||
Value::String(message) => message,
|
||||
value => value.to_string(),
|
||||
};
|
||||
} else {
|
||||
self.fields.insert(field.name().to_owned(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit for Record {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
self.field(field, format!("{value:?}").into());
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_bool(&mut self, field: &Field, value: bool) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_i64(&mut self, field: &Field, value: i64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_u64(&mut self, field: &Field, value: u64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_f64(&mut self, field: &Field, value: f64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
}
|
||||
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
use fancy_regex::Result;
|
||||
use percent_encoding::percent_decode_str;
|
||||
|
||||
use crate::{REDACTED, SecretRedactor};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Policy {
|
||||
pub redact: bool,
|
||||
pub base64_limit: i64,
|
||||
pub text_limit: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DiagnosticInput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<(Option<String>, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct DiagnosticOutput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<String>,
|
||||
pub changed: bool,
|
||||
}
|
||||
|
||||
pub struct Processor {
|
||||
redactor: SecretRedactor,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
redactor: SecretRedactor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact_text(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact(text)
|
||||
}
|
||||
|
||||
pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_structured(key, text)
|
||||
}
|
||||
|
||||
pub fn redact_client_message(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_internal(text)
|
||||
}
|
||||
|
||||
pub fn process_diagnostic(
|
||||
&self,
|
||||
input: &DiagnosticInput,
|
||||
policy: Policy,
|
||||
) -> Result<DiagnosticOutput> {
|
||||
let message = self.process_text(&input.message, policy)?;
|
||||
let exception = input
|
||||
.exception
|
||||
.as_deref()
|
||||
.map(|text| self.process_text(text, policy))
|
||||
.transpose()?;
|
||||
let stack = input
|
||||
.stack
|
||||
.as_deref()
|
||||
.map(|text| {
|
||||
if policy.redact {
|
||||
self.redact_text(text)
|
||||
} else {
|
||||
Ok(text.to_owned())
|
||||
}
|
||||
})
|
||||
.transpose()?;
|
||||
let leaves = input
|
||||
.leaves
|
||||
.iter()
|
||||
.map(|(key, text)| {
|
||||
if policy.redact {
|
||||
self.redact_structured_text(key.as_deref(), text)
|
||||
} else {
|
||||
Ok(text.clone())
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let changed = message != input.message
|
||||
|| exception != input.exception
|
||||
|| stack != input.stack
|
||||
|| leaves
|
||||
.iter()
|
||||
.zip(&input.leaves)
|
||||
.any(|(processed, (_, original))| processed != original);
|
||||
Ok(DiagnosticOutput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
changed,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result<Vec<String>> {
|
||||
arguments
|
||||
.iter()
|
||||
.map(|argument| self.scrub_access_arg(argument))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn process_text(&self, text: &str, policy: Policy) -> Result<String> {
|
||||
let collapsed = if policy.base64_limit > 0 {
|
||||
collapse_base64(text, policy.base64_limit as usize)
|
||||
} else {
|
||||
text.to_owned()
|
||||
};
|
||||
let redacted = if policy.redact {
|
||||
self.redact_text(&collapsed)?
|
||||
} else {
|
||||
collapsed
|
||||
};
|
||||
Ok(
|
||||
if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize {
|
||||
truncate_text(&redacted, policy.text_limit as usize)
|
||||
} else {
|
||||
redacted
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn scrub_access_arg(&self, value: &str) -> Result<String> {
|
||||
let length = value.chars().count();
|
||||
let scanned = if length <= 512 {
|
||||
value
|
||||
} else {
|
||||
let head = &value[..char_offset(value, 512)];
|
||||
if head.contains('?') {
|
||||
&head[..head.rfind(['?', '&']).unwrap_or(0)]
|
||||
} else {
|
||||
head
|
||||
}
|
||||
};
|
||||
let scrubbed = self.redact_text(scanned)?;
|
||||
let (path, query) = scrubbed
|
||||
.split_once('?')
|
||||
.map_or((scrubbed.as_str(), None), |(path, query)| {
|
||||
(path, Some(query))
|
||||
});
|
||||
let safe = if self.hides_encoded_credential(path)? {
|
||||
REDACTED.to_owned()
|
||||
} else if query.is_some() && self.hides_encoded_credential(&scrubbed)? {
|
||||
format!("{path}?{REDACTED}")
|
||||
} else {
|
||||
scrubbed
|
||||
};
|
||||
Ok(if length > 512 {
|
||||
format!(
|
||||
"{safe}... ({} more chars truncated) ...",
|
||||
length - scanned.chars().count()
|
||||
)
|
||||
} else {
|
||||
safe
|
||||
})
|
||||
}
|
||||
|
||||
fn hides_encoded_credential(&self, value: &str) -> Result<bool> {
|
||||
if !value.as_bytes().contains(&b'%') {
|
||||
return Ok(false);
|
||||
}
|
||||
let decoded = percent_decode_str(value).decode_utf8_lossy();
|
||||
Ok(self.redact_text(&decoded)? != decoded)
|
||||
}
|
||||
}
|
||||
|
||||
fn char_offset(text: &str, count: usize) -> usize {
|
||||
text.char_indices()
|
||||
.nth(count)
|
||||
.map_or(text.len(), |(index, _)| index)
|
||||
}
|
||||
|
||||
fn marker(skipped_chars: usize) -> String {
|
||||
format!(
|
||||
"... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..."
|
||||
)
|
||||
}
|
||||
|
||||
fn truncate_text(text: &str, limit: usize) -> String {
|
||||
let length = text.chars().count();
|
||||
let kept = limit.saturating_sub(marker(length).len());
|
||||
if kept == 0 {
|
||||
return text[..char_offset(text, limit)].to_owned();
|
||||
}
|
||||
let head = kept / 2;
|
||||
let tail = kept - head;
|
||||
format!(
|
||||
"{}{}{}",
|
||||
&text[..char_offset(text, head)],
|
||||
marker(length - kept),
|
||||
&text[char_offset(text, length - tail)..]
|
||||
)
|
||||
}
|
||||
|
||||
fn base64_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/'
|
||||
}
|
||||
|
||||
fn looks_like_base64(run: &str) -> bool {
|
||||
let unpadded = run.trim_end_matches('=');
|
||||
let lower_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
|
||||
let upper_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
|
||||
let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]);
|
||||
(!lower_hex && !upper_hex) || repeated
|
||||
}
|
||||
|
||||
fn base64_size(chars: usize) -> String {
|
||||
let bytes = chars as f64 * 3.0 / 4.0;
|
||||
if bytes >= 1024.0 * 1024.0 {
|
||||
return format!("{:.2}MB", bytes / (1024.0 * 1024.0));
|
||||
}
|
||||
if bytes >= 1024.0 {
|
||||
return format!("{:.1}KB", bytes / 1024.0);
|
||||
}
|
||||
format!("{}B", bytes as usize)
|
||||
}
|
||||
|
||||
fn collapse_base64(text: &str, limit: usize) -> String {
|
||||
let bytes = text.as_bytes();
|
||||
let mut position = 0;
|
||||
let mut previous = 0;
|
||||
let mut output = String::new();
|
||||
while position < bytes.len() {
|
||||
if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) {
|
||||
position += 1;
|
||||
continue;
|
||||
}
|
||||
let start = position;
|
||||
while position < bytes.len() && base64_byte(bytes[position]) {
|
||||
position += 1;
|
||||
}
|
||||
let run_end = position;
|
||||
while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' {
|
||||
position += 1;
|
||||
}
|
||||
let run = &text[start..position];
|
||||
if run_end - start > limit && looks_like_base64(run) {
|
||||
output.push_str(&text[previous..start]);
|
||||
output.push_str(&format!(
|
||||
"[base64_data truncated: {}]",
|
||||
base64_size(run.len())
|
||||
));
|
||||
previous = position;
|
||||
}
|
||||
}
|
||||
output.push_str(&text[previous..]);
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() {
|
||||
let processor = Processor::new(16);
|
||||
let secret = format!("sk-{}", "q".repeat(48));
|
||||
let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000));
|
||||
let input = DiagnosticInput {
|
||||
message: text,
|
||||
exception: None,
|
||||
stack: None,
|
||||
leaves: vec![],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 0,
|
||||
text_limit: 500,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.chars().count() <= 500);
|
||||
assert!(!output.message.contains("sk-qq"));
|
||||
assert!(output.changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() {
|
||||
let processor = Processor::new(16);
|
||||
let input = DiagnosticInput {
|
||||
message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)),
|
||||
exception: Some(format!("upload failed: {}", "Q".repeat(100))),
|
||||
stack: Some("api_key=secret123".to_owned()),
|
||||
leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 20,
|
||||
text_limit: 0,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.contains("[base64_data truncated: 75B]"));
|
||||
assert!(output.message.contains(&"a1".repeat(50)));
|
||||
assert!(
|
||||
output
|
||||
.exception
|
||||
.unwrap()
|
||||
.contains("[base64_data truncated: 75B]")
|
||||
);
|
||||
assert_eq!(output.stack.as_deref(), Some(REDACTED));
|
||||
assert_eq!(output.leaves, vec![REDACTED]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() {
|
||||
let processor = Processor::new(16);
|
||||
let arguments = vec![
|
||||
"/v1/models?filter=gpt%2D4o&page=2".to_owned(),
|
||||
"/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(),
|
||||
];
|
||||
assert_eq!(
|
||||
processor.scrub_access_arguments(&arguments).unwrap(),
|
||||
vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()]
|
||||
);
|
||||
}
|
||||
}
|
||||
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
use fancy_regex::{NoExpand, Regex};
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
#[cfg(test)]
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
internal_pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
let internal_pattern = Regex::new(concat!(
|
||||
r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#,
|
||||
r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#,
|
||||
r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|",
|
||||
r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|",
|
||||
r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b",
|
||||
))
|
||||
.expect("internal detail patterns compile");
|
||||
Self {
|
||||
pattern,
|
||||
internal_pattern,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.try_redact(value)
|
||||
.unwrap_or_else(|_| REDACTED.to_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
self.pattern
|
||||
.try_replacen(value, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact_structured(
|
||||
&self,
|
||||
key: Option<&str>,
|
||||
value: &str,
|
||||
) -> fancy_regex::Result<String> {
|
||||
let scrubbed = self.try_redact(value)?;
|
||||
if scrubbed != value || key.is_none() {
|
||||
return Ok(scrubbed);
|
||||
}
|
||||
let rendered = format!("'{}': '{value}'", key.unwrap_or_default());
|
||||
Ok(if self.try_redact(&rendered)? != rendered {
|
||||
REDACTED.to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
let without_traceback = value
|
||||
.split_once("Traceback (most recent call last):")
|
||||
.map_or(value, |(prefix, _)| prefix.trim_end());
|
||||
self.internal_pattern
|
||||
.try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc,
|
||||
};
|
||||
|
||||
use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Output {
|
||||
enabled: Arc<AtomicBool>,
|
||||
sender: mpsc::Sender<(String, Value, Level, &'static str, Option<u32>)>,
|
||||
}
|
||||
|
||||
impl Sink for Output {
|
||||
fn enabled(&self, _: &Metadata<'_>) -> bool {
|
||||
self.enabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
self.sender
|
||||
.send((
|
||||
record.message.clone(),
|
||||
Value::Object(record.fields.clone()),
|
||||
*record.metadata.level(),
|
||||
record.metadata.target(),
|
||||
record.metadata.line(),
|
||||
))
|
||||
.unwrap();
|
||||
Logger::default().scope(|| warn!("a sink must not recursively emit"));
|
||||
}
|
||||
}
|
||||
|
||||
fn emit() {
|
||||
warn!(
|
||||
attempt = 3_u64,
|
||||
elapsed = 1.5,
|
||||
retry = true,
|
||||
reason = "timeout",
|
||||
"retry {}",
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let enabled = Arc::new(AtomicBool::new(false));
|
||||
let logger = Logger::new(Output {
|
||||
enabled: enabled.clone(),
|
||||
sender,
|
||||
});
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
enabled.store(true, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
let (message, fields, level, target, line) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "retry 3");
|
||||
assert_eq!(
|
||||
fields,
|
||||
json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"})
|
||||
);
|
||||
assert_eq!(level, Level::WARN);
|
||||
assert_eq!(target, module_path!());
|
||||
assert!(line.is_some());
|
||||
enabled.store(false, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() {
|
||||
let tasks = (0..2)
|
||||
.map(|id| {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let logger = Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
});
|
||||
let task = tokio::spawn(logger.instrument(async move {
|
||||
tokio::task::yield_now().await;
|
||||
info!(id, "worker");
|
||||
}));
|
||||
(id, task, receiver)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (id, task, receiver) in tasks {
|
||||
task.await.unwrap();
|
||||
let (message, fields, level, _, _) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "worker");
|
||||
assert_eq!(fields, json!({"id": id}));
|
||||
assert_eq!(level, Level::INFO);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_scopes_restore_the_previous_sink() {
|
||||
let (outer_sender, outer) = mpsc::channel();
|
||||
let (inner_sender, inner) = mpsc::channel();
|
||||
let logger = |sender| {
|
||||
Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
})
|
||||
};
|
||||
let outside = logger(outer_sender);
|
||||
let inside = logger(inner_sender);
|
||||
outside.scope(|| {
|
||||
info!("before");
|
||||
inside.scope(|| info!("inside"));
|
||||
info!("after");
|
||||
});
|
||||
assert_eq!(
|
||||
outer.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["before", "after"]
|
||||
);
|
||||
assert_eq!(
|
||||
inner.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["inside"]
|
||||
);
|
||||
}
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
import ast
|
||||
import contextvars
|
||||
import functools
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from typing import Any, Final, TextIO
|
||||
|
|
@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
_python_redact_string,
|
||||
_python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_string,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
set_verbose = False
|
||||
|
||||
|
|
@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str:
|
|||
pass through credential redaction.
|
||||
"""
|
||||
stripped: Final = "".join(ch for ch in value if ch.isprintable())
|
||||
return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
|
||||
return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
|
||||
|
||||
|
||||
def set_session_id(session_id: str) -> "contextvars.Token[str]":
|
||||
|
|
@ -74,12 +77,6 @@ def _redact_string(value: str) -> str:
|
|||
return redact_string(value)
|
||||
|
||||
|
||||
def _redact_structured_value(key: str | None, value: str) -> str:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return value
|
||||
return redact_structured_value(key, value)
|
||||
|
||||
|
||||
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
|
||||
_REDACTED_STAMP: Final = object()
|
||||
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
|
||||
|
|
@ -103,14 +100,6 @@ def _plain_text(value: object) -> str:
|
|||
return UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _redact_extra_value(key: str, value: object) -> object:
|
||||
try:
|
||||
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
|
||||
except Exception:
|
||||
return _redact_string(_plain_text(value))
|
||||
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
|
||||
|
||||
|
||||
def redact_secrets(value: str) -> str:
|
||||
"""Public API: redact known secret/credential patterns from an arbitrary string.
|
||||
|
||||
|
|
@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
|||
return None
|
||||
try:
|
||||
return color_message % record.args
|
||||
except TypeError:
|
||||
except Exception:
|
||||
return color_message
|
||||
|
||||
|
||||
|
|
@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter):
|
|||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
|
||||
return True
|
||||
|
||||
# Runs before args are cleared, and before the extra-field loop below
|
||||
# that redacts the substituted result.
|
||||
substituted_color_message: Final = _substituted_color_message(record)
|
||||
if substituted_color_message is not None:
|
||||
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
try:
|
||||
record.msg = _redact_string(record.getMessage())
|
||||
record.args = None
|
||||
except Exception:
|
||||
if isinstance(record.msg, str):
|
||||
record.msg = _redact_string(record.msg)
|
||||
|
||||
# Redact exception tracebacks
|
||||
if record.exc_info and record.exc_info[1] is not None:
|
||||
try:
|
||||
record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(record.stack_info, str):
|
||||
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
# Redact extra fields passed via logger.debug("msg", extra={...})
|
||||
record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items())
|
||||
for key, value in record_items:
|
||||
if key in _STANDARD_RECORD_ATTRS:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
setattr(record, key, _redact_structured_value(key, value))
|
||||
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
|
||||
setattr(record, key, _redact_extra_value(key, value))
|
||||
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
return _process_record(record, base64_limit=0, text_limit=0, redact=True)
|
||||
|
||||
|
||||
_secret_filter: Final = SecretRedactionFilter()
|
||||
|
|
@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
|
|||
def _hides_a_credential(value: str) -> bool:
|
||||
"""Whether *value* only looks clean until it is percent-decoded."""
|
||||
decoded: Final = unquote(value)
|
||||
return _redact_string(decoded) != decoded
|
||||
return _python_redact_string(decoded) != decoded
|
||||
|
||||
|
||||
def _drop_encoded_credential(scrubbed: str) -> str:
|
||||
|
|
@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str:
|
|||
pattern and would then be logged raw.
|
||||
"""
|
||||
if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
|
||||
return _drop_encoded_credential(_redact_string(value))
|
||||
return _drop_encoded_credential(_python_redact_string(value))
|
||||
head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
|
||||
kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
|
||||
scrubbed: Final = _drop_encoded_credential(_redact_string(kept))
|
||||
scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
|
||||
return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
|
||||
|
||||
|
||||
|
|
@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter):
|
|||
if not _ENABLE_SECRET_REDACTION:
|
||||
return True
|
||||
if isinstance(record.args, tuple) and record.args:
|
||||
strings: Final = tuple(arg for arg in record.args if isinstance(arg, str))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.scrub_access_arguments(strings),
|
||||
lambda: tuple(_scrub_access_arg(arg) for arg in strings),
|
||||
)
|
||||
scrubbed: Final = (
|
||||
candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings)
|
||||
)
|
||||
values: Final = iter(scrubbed)
|
||||
record.args = tuple( # rebind-ok: a Filter scrubs records in place
|
||||
_scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args
|
||||
next(values) if isinstance(arg, str) else arg for arg in record.args
|
||||
)
|
||||
return True
|
||||
# No positional args means everything is in msg, where collapsing is correct.
|
||||
|
|
@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str:
|
|||
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
|
||||
|
||||
|
||||
def _extra_structure(key: str, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return safe_json_structure(value, key=key)
|
||||
except Exception:
|
||||
return _plain_text(value)
|
||||
|
||||
|
||||
def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]:
|
||||
if isinstance(value, str):
|
||||
yield key, value
|
||||
elif isinstance(value, dict):
|
||||
yield from itertools.chain.from_iterable(
|
||||
_string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str)
|
||||
)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value)
|
||||
|
||||
|
||||
def _replace_string_leaves(value: object, values: Iterator[str]) -> object:
|
||||
if isinstance(value, str):
|
||||
return next(values)
|
||||
if isinstance(value, dict):
|
||||
return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers
|
||||
key: _replace_string_leaves(child, values) for key, child in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers
|
||||
_replace_string_leaves(child, values) for child in value
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_replace_string_leaves(child, values) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def _sort_processed_sets(original: object, processed: object) -> object:
|
||||
if isinstance(original, set) and isinstance(processed, list):
|
||||
return sorted(processed)
|
||||
if isinstance(original, dict) and isinstance(processed, dict):
|
||||
return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict
|
||||
key: _sort_processed_sets(original.get(key), value) for key, value in processed.items()
|
||||
}
|
||||
if isinstance(original, list) and isinstance(processed, list):
|
||||
return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list
|
||||
_sort_processed_sets(before, after) for before, after in zip(original, processed)
|
||||
]
|
||||
if isinstance(original, tuple) and isinstance(processed, tuple):
|
||||
return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed))
|
||||
return processed
|
||||
|
||||
|
||||
def _python_process_diagnostic(
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
redact: bool,
|
||||
base64_limit: int,
|
||||
text_limit: int,
|
||||
) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
|
||||
def process_text(text: str) -> str:
|
||||
collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
|
||||
scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
|
||||
return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
|
||||
|
||||
processed_message: Final = process_text(message)
|
||||
processed_exception: Final = process_text(exception) if exception is not None else None
|
||||
processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
|
||||
processed_leaves: Final = tuple(
|
||||
_python_redact_structured_value(key, text) if redact else text for key, text in leaves
|
||||
)
|
||||
changed: Final = (
|
||||
processed_message != message
|
||||
or processed_exception != exception
|
||||
or processed_stack != stack
|
||||
or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves))
|
||||
)
|
||||
return processed_message, processed_exception, processed_stack, processed_leaves, changed
|
||||
|
||||
|
||||
def _render_message(record: logging.LogRecord) -> str:
|
||||
try:
|
||||
return record.getMessage()
|
||||
except Exception:
|
||||
return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _render_exception(record: logging.LogRecord) -> str | None:
|
||||
if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None:
|
||||
return None
|
||||
try:
|
||||
return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info)
|
||||
except Exception:
|
||||
return "REDACTED"
|
||||
|
||||
|
||||
def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool:
|
||||
if _is_redacted(record):
|
||||
return True
|
||||
message: Final = _render_message(record)
|
||||
exception: Final = _render_exception(record)
|
||||
stack: Final = record.stack_info if isinstance(record.stack_info, str) else None
|
||||
substituted_color: Final = _substituted_color_message(record)
|
||||
extras: Final = (
|
||||
tuple(
|
||||
(
|
||||
key,
|
||||
value,
|
||||
_extra_structure(
|
||||
key, substituted_color if key == "color_message" and substituted_color is not None else value
|
||||
),
|
||||
)
|
||||
for key, value in record.__dict__.items()
|
||||
if key not in _STANDARD_RECORD_ATTRS
|
||||
and key != _REDACTED_RECORD_ATTR
|
||||
and not isinstance(value, _UNREDACTED_SCALAR_TYPES)
|
||||
)
|
||||
if redact
|
||||
else ()
|
||||
)
|
||||
extra_leaves: Final = tuple(
|
||||
itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras)
|
||||
)
|
||||
raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None
|
||||
color_template: Final = record.__dict__.get("color_message")
|
||||
raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None
|
||||
leaves: Final = (
|
||||
extra_leaves
|
||||
+ (((None, raw_template),) if raw_template is not None else ())
|
||||
+ (((None, raw_color),) if raw_color is not None else ())
|
||||
)
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)),
|
||||
lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit),
|
||||
)
|
||||
processed_message, processed_exception, processed_stack, processed_leaves, _ = (
|
||||
candidate
|
||||
if len(candidate[3]) == len(leaves)
|
||||
else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit)
|
||||
)
|
||||
raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template
|
||||
safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message
|
||||
if redact or safe_message != message:
|
||||
record.msg = safe_message # rebind-ok: the Filter interface mutates the record
|
||||
record.args = None # rebind-ok: the rendered message replaces interpolation inputs
|
||||
if processed_exception is not None:
|
||||
record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record
|
||||
if processed_stack is not None:
|
||||
record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record
|
||||
processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
|
||||
for key, original, prepared in extras:
|
||||
replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
|
||||
if not _scrubbing_changed_nothing(replacement, original):
|
||||
setattr(record, key, replacement)
|
||||
raw_color_changed: Final = (
|
||||
raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color
|
||||
)
|
||||
if raw_color_changed and getattr(record, "color_message", None) == substituted_color:
|
||||
setattr(record, "color_message", "REDACTED")
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
|
||||
|
||||
def _redact_json_record(value: object) -> object:
|
||||
prepared: Final = safe_json_structure(value)
|
||||
leaves: Final = tuple(_string_leaves(None, prepared))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
|
||||
lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
|
||||
)
|
||||
replacements: Final = (
|
||||
candidate
|
||||
if len(candidate) == len(leaves)
|
||||
else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
|
||||
)
|
||||
return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
|
||||
|
||||
|
||||
class StdoutLogTruncationFilter(logging.Filter):
|
||||
"""Bounds how much of an oversized log line reaches stdout.
|
||||
|
||||
|
|
@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter):
|
|||
return True
|
||||
|
||||
|
||||
_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
|
||||
class DiagnosticProcessingFilter(StdoutLogTruncationFilter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return _process_record(
|
||||
record,
|
||||
base64_limit=_get_max_base64_length_stdout_log(),
|
||||
text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0,
|
||||
redact=_ENABLE_SECRET_REDACTION,
|
||||
)
|
||||
|
||||
|
||||
_diagnostic_filter: Final = DiagnosticProcessingFilter()
|
||||
|
||||
|
||||
class CorrelationContextFilter(logging.Filter):
|
||||
|
|
@ -633,7 +785,9 @@ class JsonFormatter(Formatter):
|
|||
if record.exc_info:
|
||||
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
|
||||
|
||||
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
|
||||
return safe_dumps(
|
||||
json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record)
|
||||
)
|
||||
|
||||
|
||||
class CorrelationPlainFormatter(logging.Formatter):
|
||||
|
|
@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter):
|
|||
# Create a handler with JSON formatting for exceptions
|
||||
error_handler: Final = logging.StreamHandler()
|
||||
error_handler.setFormatter(formatter)
|
||||
error_handler.addFilter(_stdout_truncation_filter)
|
||||
error_handler.addFilter(_diagnostic_filter)
|
||||
error_handler.addFilter(_secret_filter)
|
||||
error_handler.addFilter(_correlation_filter)
|
||||
|
||||
|
|
@ -734,10 +888,10 @@ verbose_logger.addHandler(handler)
|
|||
|
||||
# Filters attached to the logger, not the handler, survive callers swapping in their own
|
||||
# handlers (JSON mode, uvicorn log config, a host app's root handler).
|
||||
verbose_router_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_router_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
|
||||
verbose_logger.addFilter(_diagnostic_filter)
|
||||
|
||||
|
||||
def _suppress_loggers():
|
||||
|
|
|
|||
|
|
@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
|
|||
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
|
||||
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
|
||||
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
|
||||
provider_first_model_name_with_quality: Final = (
|
||||
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
|
||||
if quality and custom_llm_provider
|
||||
else None
|
||||
)
|
||||
|
||||
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
|
||||
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
|
||||
|
|
@ -2386,6 +2391,7 @@ def default_image_cost_calculator(
|
|||
|
||||
models_to_check: Final = (
|
||||
model_name_with_quality,
|
||||
provider_first_model_name_with_quality,
|
||||
base_model_name,
|
||||
model_name_with_v2_quality,
|
||||
model_with_quality_without_provider,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import re
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
REDACTED: Final = "REDACTED"
|
||||
|
||||
|
|
@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]":
|
|||
_SECRET_RE: Final = _build_secret_patterns()
|
||||
|
||||
|
||||
def _python_redact_string(value: str) -> str:
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
|
||||
|
||||
def redact_string(value: str) -> str:
|
||||
"""Scrub known secret/credential patterns from *value* and return the result."""
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value))
|
||||
|
||||
|
||||
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
|
||||
|
|
@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile(
|
|||
_TRACEBACK_MARKER: Final = "Traceback (most recent call last):"
|
||||
|
||||
|
||||
def redact_internal_details(value: str) -> str:
|
||||
def _python_redact_internal_details(value: str) -> str:
|
||||
"""Drop an embedded traceback and scrub filesystem paths and internal hostnames,
|
||||
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
|
||||
marker_index: Final = value.find(_TRACEBACK_MARKER)
|
||||
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback))
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
def redact_internal_details(value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value)
|
||||
)
|
||||
|
||||
|
||||
def _python_redact_structured_value(key: str | None, value: str) -> str:
|
||||
"""Scrub *value* as it appeared under *key* inside a structured record.
|
||||
|
||||
redact_string() replaces a whole ``key: value`` span with REDACTED, which is
|
||||
|
|
@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str:
|
|||
repr would, so the key-name patterns still fire, but collapses only the value
|
||||
so the caller's structure survives.
|
||||
"""
|
||||
scrubbed: Final = redact_string(value)
|
||||
scrubbed: Final = _python_redact_string(value)
|
||||
if scrubbed != value or key is None:
|
||||
return scrubbed
|
||||
rendered: Final = f"'{key}': '{value}'"
|
||||
return REDACTED if redact_string(rendered) != rendered else value
|
||||
return REDACTED if _python_redact_string(rendered) != rendered else value
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_structured_text(key, value),
|
||||
lambda: _python_redact_structured_value(key, value),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
return self._is_deepseek_r1_model(model=model, base_model=base_model)
|
||||
|
||||
@classmethod
|
||||
def _supports_sampling_params(cls, model: str) -> bool:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if base_model.startswith("anthropic"):
|
||||
return True
|
||||
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
|
||||
for candidate in candidates:
|
||||
if (
|
||||
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
|
||||
candidate, "supports_sampling_params"
|
||||
)
|
||||
) is not None:
|
||||
return flag
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
supports_sampling: Final = self._supports_sampling_params(model)
|
||||
supported_params: Final = [
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
"stream_options",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
*(("temperature", "top_p") if supports_sampling else ()),
|
||||
"extra_headers",
|
||||
"response_format",
|
||||
"requestMetadata",
|
||||
|
|
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
|
|||
value = [value]
|
||||
optional_params["stopSequences"] = value
|
||||
if param == "temperature" or param == "top_p":
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
if base_model.startswith("anthropic"):
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
elif not self._supports_sampling_params(model):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}={value}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
else:
|
||||
optional_params["topP" if param == "top_p" else param] = value
|
||||
if param == "tools" and isinstance(value, list):
|
||||
self._apply_tool_call_transformation(
|
||||
tools=cast(list[OpenAIChatCompletionToolParam], value),
|
||||
|
|
|
|||
|
|
@ -231,6 +231,12 @@ async def make_call(
|
|||
sync_stream=False,
|
||||
)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
elif bedrock_invoke_provider == "moonshot":
|
||||
decoder = AmazonOpenAICompatibleStreamDecoder(
|
||||
model=model,
|
||||
sync_stream=False,
|
||||
)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
else:
|
||||
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
|
|
@ -329,6 +335,12 @@ def make_sync_call(
|
|||
sync_stream=True,
|
||||
)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
elif bedrock_invoke_provider == "moonshot":
|
||||
decoder = AmazonOpenAICompatibleStreamDecoder(
|
||||
model=model,
|
||||
sync_stream=True,
|
||||
)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
else:
|
||||
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
|
|
@ -795,6 +807,24 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
|
|||
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
|
||||
|
||||
class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
sync_stream: bool,
|
||||
) -> None:
|
||||
super().__init__(model=model)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
|
||||
|
||||
self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
|
||||
streaming_response=None,
|
||||
sync_stream=sync_stream,
|
||||
)
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
|
||||
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
|
||||
|
||||
class MockResponseIterator: # for returning ai21 streaming responses
|
||||
def __init__(self, model_response, json_mode: bool | None = False):
|
||||
self.model_response = model_response
|
||||
|
|
|
|||
|
|
@ -770,9 +770,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
aws_decoder: Final = AmazonAnthropicClaudeMessagesStreamDecoder(
|
||||
model=model,
|
||||
)
|
||||
completion_stream: Final = aws_decoder.aiter_bytes(
|
||||
httpx_response.aiter_bytes(chunk_size=aws_decoder.DEFAULT_CHUNK_SIZE)
|
||||
)
|
||||
completion_stream: Final = aws_decoder.aiter_bytes(httpx_response.aiter_bytes())
|
||||
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
|
||||
return self.bedrock_sse_wrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -919,16 +917,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
|
||||
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""
|
||||
Iterator to return Bedrock invoke response in anthropic /messages format
|
||||
"""
|
||||
super().__init__(model=model)
|
||||
self.DEFAULT_CHUNK_SIZE = 1024
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
|
||||
"""
|
||||
Parse the chunk data into anthropic /messages format
|
||||
|
|
|
|||
|
|
@ -287,7 +287,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
Check if the model is Gemini 3 or newer.
|
||||
"""
|
||||
model_name = model.split("/")[-1].lower()
|
||||
if not model_name:
|
||||
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
|
||||
model.startswith("gemini/") and not model_name.startswith("gemini-")
|
||||
)
|
||||
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
|
||||
return False
|
||||
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
|
||||
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1004,6 +1004,8 @@ def _stamp_deployment_attribution(
|
|||
return attribution
|
||||
metadata.setdefault("model_info", attribution["model_info"])
|
||||
metadata.setdefault("deployment", attribution["deployment"])
|
||||
if isinstance(model_group, str):
|
||||
metadata.setdefault("model_group", model_group)
|
||||
return attribution
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ...
|
|||
class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
|
||||
class ProcessReservedForForking(RuntimeError): ...
|
||||
|
||||
@final
|
||||
class NativeDiagnosticProcessor:
|
||||
def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: Sequence[tuple[str | None, str]],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, list[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ...
|
||||
|
||||
def ocr(
|
||||
request: LiteLLMOcrRequest,
|
||||
args: tuple[object, ...],
|
||||
|
|
@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ...
|
|||
__all__ = [
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"NativeDiagnosticProcessor",
|
||||
"ProcessReservedForForking",
|
||||
"ResponsesWebSocketConnection",
|
||||
"RustBridgeDeclined",
|
||||
|
|
|
|||
|
|
@ -86,11 +86,25 @@ class SecretManagerRule:
|
|||
return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerContext:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerRule:
|
||||
rollout: Rollout
|
||||
|
||||
def matches(self, context: Context) -> bool:
|
||||
return isinstance(context, LoggerContext)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule
|
||||
Rules: TypeAlias = tuple[Rule, ...]
|
||||
|
||||
RULES: Final[Rules] = (
|
||||
LoggerRule(Rollout.RUST_OPT_IN),
|
||||
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
|
||||
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
|
||||
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
|
||||
|
|
|
|||
59
litellm/rust_bridge/diagnostics.py
Normal file
59
litellm/rust_bridge/diagnostics.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from functools import lru_cache
|
||||
from typing import Final, Protocol, TypeVar, cast
|
||||
|
||||
from litellm.rust_bridge.bindings import NativeBinding
|
||||
|
||||
ResultT: Final = TypeVar("ResultT")
|
||||
|
||||
|
||||
class NativeDiagnosticProcessor(Protocol):
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, Sequence[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ...
|
||||
|
||||
|
||||
class NativeDiagnosticFactory(Protocol):
|
||||
def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
|
||||
|
||||
def _as_factory(value: object) -> NativeDiagnosticFactory | None:
|
||||
if not isinstance(value, type):
|
||||
return None
|
||||
return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type
|
||||
|
||||
|
||||
PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory)
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor:
|
||||
return factory(minimum_custom_key_length)
|
||||
|
||||
|
||||
def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT:
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge.catalog import LoggerContext, decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
||||
selected: Final = decision(LoggerContext())
|
||||
if selected is Decision.PYTHON:
|
||||
return python()
|
||||
factory: Final = PROCESSOR.load()
|
||||
if factory is None:
|
||||
return python()
|
||||
try:
|
||||
return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH))
|
||||
except Exception:
|
||||
return python()
|
||||
63
litellm/rust_bridge/logger.py
Normal file
63
litellm/rust_bridge/logger.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm._logging import (
|
||||
CorrelationContextFilter,
|
||||
DiagnosticProcessingFilter,
|
||||
session_id_var,
|
||||
set_session_id,
|
||||
set_trace_id,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
)
|
||||
|
||||
_REDACTION: Final = DiagnosticProcessingFilter()
|
||||
_CORRELATION: Final = CorrelationContextFilter()
|
||||
|
||||
|
||||
def context() -> tuple[str, str]:
|
||||
return session_id_var.get(), trace_id_var.get()
|
||||
|
||||
|
||||
def enabled(level: int) -> bool:
|
||||
return verbose_logger.isEnabledFor(level)
|
||||
|
||||
|
||||
def emit(
|
||||
level: int,
|
||||
message: str,
|
||||
pathname: str,
|
||||
lineno: int,
|
||||
target: str,
|
||||
fields: Mapping[str, JsonValue],
|
||||
correlation: tuple[str, str],
|
||||
) -> None:
|
||||
if not enabled(level):
|
||||
return
|
||||
session_token: Final = set_session_id(correlation[0])
|
||||
trace_token: Final = set_trace_id(correlation[1])
|
||||
try:
|
||||
record: Final = verbose_logger.makeRecord(
|
||||
verbose_logger.name,
|
||||
level,
|
||||
pathname,
|
||||
lineno,
|
||||
message,
|
||||
(),
|
||||
None,
|
||||
func=target,
|
||||
extra={
|
||||
"rust_target": target,
|
||||
"rust_fields": dict(fields),
|
||||
}, # mutable-ok: LogRecord requires JSON dict extras
|
||||
)
|
||||
_REDACTION.filter(record)
|
||||
_CORRELATION.filter(record)
|
||||
verbose_logger.handle(record)
|
||||
finally:
|
||||
trace_id_var.reset(trace_token)
|
||||
session_id_var.reset(session_token)
|
||||
|
|
@ -58,12 +58,6 @@ class SecretManagerBinding:
|
|||
settings_object: object
|
||||
|
||||
|
||||
def warn(message: str) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.warning("%s", message)
|
||||
|
||||
|
||||
def secret_manager() -> SecretManager:
|
||||
from litellm.secret_managers.main import (
|
||||
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm"
|
||||
version = "1.103.0"
|
||||
version = "1.104.0"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10, <3.15"
|
||||
|
|
@ -75,8 +75,8 @@ proxy = [
|
|||
"mcp>=2.2.0,<3",
|
||||
"httpx2>=2.5.0,<3",
|
||||
"pydantic>=2.12.0,<3",
|
||||
"litellm-proxy-extras==0.4.100",
|
||||
"litellm-enterprise==0.1.69",
|
||||
"litellm-proxy-extras==0.4.101",
|
||||
"litellm-enterprise==0.1.70",
|
||||
"RestrictedPython>=8.5,<9.0",
|
||||
"rich>=13.9.4,<14.0",
|
||||
"InquirerPy>=0.3.4,<1.0",
|
||||
|
|
@ -355,7 +355,7 @@ litellm-enterprise = { workspace = true }
|
|||
members = ["enterprise", "litellm-proxy-extras"]
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.103.0"
|
||||
version = "1.104.0"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [
|
|||
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
|
||||
"_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible).
|
||||
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
|
||||
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
41
tests/e2e/claude_code/cron_vm/Dockerfile
Normal file
41
tests/e2e/claude_code/cron_vm/Dockerfile
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
|
||||
|
||||
ARG GH_VERSION=2.101.0
|
||||
ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8
|
||||
ARG UV_VERSION=0.10.9
|
||||
ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594
|
||||
ARG CLAUDE_CODE_VERSION=2.1.228
|
||||
ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \
|
||||
&& rm /tmp/gh.tar.gz
|
||||
|
||||
RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \
|
||||
&& echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \
|
||||
&& rm /tmp/uv.tar.gz
|
||||
|
||||
RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \
|
||||
&& echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/claude /usr/local/bin/claude \
|
||||
&& rm /tmp/claude
|
||||
|
||||
RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator
|
||||
|
||||
ENV HOME=/home/populator \
|
||||
LITELLM_REPO=/opt/litellm \
|
||||
DISABLE_AUTOUPDATER=1
|
||||
|
||||
COPY --chown=populator:populator . /opt/litellm/tests/e2e/
|
||||
|
||||
USER populator
|
||||
WORKDIR /home/populator
|
||||
CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"]
|
||||
|
|
@ -1,66 +1,59 @@
|
|||
# Cron VM setup for the Claude Code compatibility-matrix populator
|
||||
# Render cron job for the Claude Code compatibility-matrix populator
|
||||
|
||||
The populator runs daily on a dedicated GCP VM
|
||||
(`litellm-compatibility-matrix-populator`) rather than as a GitHub
|
||||
Action. Trade-offs:
|
||||
The populator runs daily as the Render cron job `litellm-compat-matrix`
|
||||
(Docker runtime, built from the `Dockerfile` in this directory) rather
|
||||
than as a GitHub Action or on a dedicated VM. Trade-offs:
|
||||
|
||||
- ✅ Real VM means we can `gh auth login` against an account that's
|
||||
already a collaborator on `BerriAI/litellm-docs`, instead of
|
||||
provisioning a GitHub App with `pull-requests: write`.
|
||||
- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`)
|
||||
is reused across runs, so each daily run does a fast `git checkout` +
|
||||
incremental `uv sync` rather than a fresh clone + cold sync.
|
||||
- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`.
|
||||
- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers
|
||||
from short outages, but a multi-day outage means the matrix goes
|
||||
stale until the VM is back.
|
||||
- ⚠️ Provider credentials live on the VM filesystem
|
||||
(`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat
|
||||
the VM as an environment with comparable blast radius to a CI runner.
|
||||
|
||||
This directory used to live at `tests/claude_code/cron_vm/` (paired with
|
||||
the standalone `tests/claude_code/` suite); it now runs the maintained
|
||||
`tests/e2e/claude_code/` suite instead. The pytest env interface changed
|
||||
accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY`
|
||||
(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure
|
||||
column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
||||
`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and
|
||||
`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`.
|
||||
- ✅ No machine to keep on or patch. Render builds the image from this
|
||||
directory on every push to `main` that touches `tests/e2e/**` and
|
||||
runs it on the schedule.
|
||||
- ✅ Credentials live in Render env vars and secret files, scoped to
|
||||
this one service, instead of on a VM filesystem.
|
||||
- ✅ The publish token still uses the `mateo-berri` account, which is a
|
||||
collaborator on `BerriAI/litellm-docs`, so no GitHub App with
|
||||
`pull-requests: write` has to be provisioned.
|
||||
- ⚠️ The disk is ephemeral, so every run starts from a fresh (blobless)
|
||||
clone of litellm plus a cold `uv sync`. That adds a few minutes on
|
||||
top of the ~10 minute test run; the job's 12 hour ceiling is nowhere
|
||||
near.
|
||||
- ⚠️ The Claude Code CLI version under test is pinned in the
|
||||
`Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a
|
||||
PR, see the gotchas below.
|
||||
|
||||
## Layout
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
|
||||
| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). |
|
||||
| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
|
||||
| `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. |
|
||||
| `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. |
|
||||
| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. |
|
||||
| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. |
|
||||
| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. |
|
||||
| `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. |
|
||||
|
||||
## What `run_daily.sh` does
|
||||
|
||||
1. **Resolves the latest LiteLLM final release tag** (newest bare
|
||||
`vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the
|
||||
GitHub Releases API (`curl | jq`).
|
||||
2. **Reads the local Claude Code CLI version** via `claude --version`.
|
||||
The cron does not auto-upgrade the CLI — operators do that
|
||||
out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`.
|
||||
3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`:
|
||||
`git fetch --tags --force`, `git reset --hard`,
|
||||
`git clean -fdx -e .venv -e .uv-bin`, `git checkout --force <tag>`.
|
||||
The `.venv` is preserved across runs so `uv sync --frozen` is
|
||||
incremental. Then **shims the test suite**: `tests/e2e/` in the
|
||||
worktree is rebuilt from the dev checkout — the `claude_code/` suite
|
||||
plus the five shared transport helpers it imports (`proxy_client.py`,
|
||||
`e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the
|
||||
cron always runs *today's* tests against the latest stable proxy. The
|
||||
tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`,
|
||||
whose imports the stable venv doesn't install) is deliberately not
|
||||
used.
|
||||
2. **Reads the Claude Code CLI version** via `claude --version`. That
|
||||
is whatever the `Dockerfile` pins; the job never upgrades it on its
|
||||
own.
|
||||
3. **Clones the worktree** at `~/litellm-cron-worktree/` (a
|
||||
`--filter=blob:none` clone, so only the checked-out tag's blobs are
|
||||
fetched), `git checkout --force <tag>`, then `uv sync --frozen
|
||||
--no-install-project` against a uv-managed CPython 3.12 followed by
|
||||
`uv pip install --no-build litellm==<version>`, so the proxy under
|
||||
test is the published PyPI wheel (what users install) rather than a
|
||||
source build: the tag builds a Rust extension through maturin, and
|
||||
the image ships no C or Rust toolchain. Then **shims the test suite**:
|
||||
`tests/e2e/` in the worktree is replaced by the image's copy of this
|
||||
whole tree, so the cron always runs *today's* tests against the
|
||||
latest stable proxy, and pytest runs with `--confcutdir` pointed at
|
||||
`claude_code/` so the tree's EKS-harness `conftest.py` (whose imports
|
||||
the stable venv doesn't install) is never loaded. The tag's own
|
||||
`tests/e2e/` is deliberately not used.
|
||||
4. **Boots the proxy** as a `setsid` background process on port `4100`
|
||||
(so it can't collide with a developer's `:4000`), then polls
|
||||
`/health/liveliness` until it's up.
|
||||
bound to loopback, then polls `/health/liveliness` until it's up.
|
||||
5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL`
|
||||
pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest
|
||||
hook writes the per-test results artifact. Test failures become
|
||||
|
|
@ -74,8 +67,9 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
|||
`mateo-berri` token has write access, so this is a same-repo branch,
|
||||
not a fork), `gh pr create`. A re-run on the same day fast-forwards
|
||||
the existing branch and `gh pr create` no-ops ("a pull request for
|
||||
branch ... already exists" is treated as success). These PRs are no
|
||||
longer gated on a second human review.
|
||||
branch ... already exists" is treated as success). If the JSON is
|
||||
byte-identical to what `main` already publishes, the push is skipped
|
||||
entirely. These PRs are not gated on a second human review.
|
||||
8. **Gates auto-merge on a regression check**: before enabling
|
||||
auto-merge, `check_regressions.py` diffs the new matrix against the
|
||||
one currently on `main`. Auto-merge (`gh pr merge --auto --squash`)
|
||||
|
|
@ -89,107 +83,155 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
|||
human reviews before it lands on the public table. The check fails
|
||||
*closed*: if it errors, auto-merge is withheld.
|
||||
9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every
|
||||
other open `compat-matrix/*` PR on the docs repo is closed (and its
|
||||
bot-owned branch deleted), so at most one compat-matrix PR is ever
|
||||
open — the newest.
|
||||
other open `compat-matrix/*` PR that the publishing account opened
|
||||
from a branch on the docs repo itself is closed (and its bot-owned
|
||||
branch deleted), so at most one compat-matrix PR is ever open — the
|
||||
newest. A contributor's PR under that prefix is never touched.
|
||||
|
||||
## One-time VM setup
|
||||
## The Render service
|
||||
|
||||
Run as `mateo` on the cron VM:
|
||||
Everything below is what the live service is set to; recreate it with
|
||||
the same values if it ever has to be rebuilt.
|
||||
|
||||
| Setting | Value |
|
||||
| --- | --- |
|
||||
| Workspace | Litellm (the one that already builds the other litellm services) |
|
||||
| Type | Cron job, Docker runtime |
|
||||
| Repo / branch | `BerriAI/litellm` @ `main` |
|
||||
| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` |
|
||||
| Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) |
|
||||
| Build filter | included paths `tests/e2e/**` |
|
||||
| Schedule | `0 6 * * *` (06:00 UTC daily) |
|
||||
| Plan / region | `4c-16g` (4 CPU, 16 GB, what the dashboard calls Pro Max; the suite fans out to ~75 concurrent CLI calls) / Oregon |
|
||||
| Env vars | every key in `litellm-compat-matrix.env.example` |
|
||||
| Secret files | `github-token` (the publish PAT, one line) and `vertex-service-account.json` (the Vertex service-account key) |
|
||||
|
||||
Render mounts secret files at `/etc/secrets/<name>`, which is where
|
||||
`CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env
|
||||
example point. Render also passes env vars to `docker build` as build
|
||||
args, which is why the `Dockerfile` declares no `ARG` that could ever
|
||||
be given a secret's name.
|
||||
|
||||
Creating it through the API looks like this (fill `envVars` and
|
||||
`secretFiles` from the env example and the two secrets; `ownerId` is
|
||||
the workspace id from `GET /v1/owners`):
|
||||
|
||||
```bash
|
||||
# 1. Toolchain
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git nodejs npm jq curl
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
sudo apt-get install -y gh # or follow https://cli.github.com/
|
||||
|
||||
# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this
|
||||
# line out-of-band when you want a fresh CLI to be tested)
|
||||
sudo npm install -g @anthropic-ai/claude-code@latest
|
||||
|
||||
# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the
|
||||
# source of the .service / .timer files. The cron itself runs out
|
||||
# of the separate worktree at ~/litellm-cron-worktree/.
|
||||
mkdir -p ~/litellm
|
||||
git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm
|
||||
git -C ~/litellm/litellm checkout litellm_internal_staging
|
||||
|
||||
# 4. gh auth — must be a collaborator on BerriAI/litellm-docs.
|
||||
gh auth login # follow prompts; pick HTTPS + token paste flow
|
||||
|
||||
# 5. Provider credentials + the publish token.
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \
|
||||
/etc/litellm-compat-matrix.env
|
||||
sudoedit /etc/litellm-compat-matrix.env # fill in real values
|
||||
sudo chmod 0600 /etc/litellm-compat-matrix.env
|
||||
# The mateo-berri PAT lives in its own file, mapped into the service via
|
||||
# systemd LoadCredential so it stays out of the test processes' env
|
||||
# (see the env.example comment for why).
|
||||
sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token
|
||||
sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT
|
||||
|
||||
# 6. systemd units.
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now litellm-compat-matrix.timer
|
||||
curl -fsS https://api.render.com/v1/services \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"type": "cron_job",
|
||||
"name": "litellm-compat-matrix",
|
||||
"ownerId": "<workspace id>",
|
||||
"repo": "https://github.com/BerriAI/litellm",
|
||||
"branch": "main",
|
||||
"autoDeploy": "yes",
|
||||
"buildFilter": {"paths": ["tests/e2e/**"], "ignoredPaths": []},
|
||||
"envVars": [{"key": "ANTHROPIC_API_KEY", "value": "..."}],
|
||||
"secretFiles": [{"name": "github-token", "content": "..."},
|
||||
{"name": "vertex-service-account.json", "content": "..."}],
|
||||
"serviceDetails": {
|
||||
"runtime": "docker",
|
||||
"schedule": "0 6 * * *",
|
||||
"plan": "4c-16g",
|
||||
"region": "oregon",
|
||||
"envSpecificDetails": {
|
||||
"dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile",
|
||||
"dockerContext": "tests/e2e"
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
## Operating it
|
||||
|
||||
```bash
|
||||
# When does it run next?
|
||||
systemctl list-timers litellm-compat-matrix.timer
|
||||
# Trigger a real run right now (PRs to litellm-docs). The id is the
|
||||
# service id (`crn-...`) from the dashboard URL or `GET /v1/services`.
|
||||
curl -fsS -X POST "https://api.render.com/v1/cron-jobs/${CRON_ID}/runs" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Trigger a real run right now (PRs to litellm-docs).
|
||||
sudo systemctl start litellm-compat-matrix.service
|
||||
# Follow a run: the Logs tab on the service, or the API.
|
||||
curl -fsS "https://api.render.com/v1/logs?ownerId=${OWNER_ID}&resource=${CRON_ID}&limit=100" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Trigger a run that does NOT open a PR (good for first-time validation).
|
||||
SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
# Rebuild the image after a merge that touches tests/e2e/** (see the
|
||||
# auto-deploy gotcha below). The deploy is done once its status is
|
||||
# `live`; a run triggered before that still uses the previous image.
|
||||
curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}" \
|
||||
-H 'Content-Type: application/json' -d '{"clearCache": "do_not_clear"}'
|
||||
curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Narrow to one cell while debugging.
|
||||
SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \
|
||||
~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
# A run that does NOT open a PR (first-time validation, CLI bumps):
|
||||
# set SKIP_PUBLISH=1 on the service, trigger a run, then remove it.
|
||||
# The matrix JSON is printed at the end of the run's log (nothing on
|
||||
# the container's disk outlives the run) and saved to
|
||||
# ~/compatibility-matrix.json for a local docker run.
|
||||
# PYTEST_K='basic_messaging_non_streaming and anthropic' narrows the
|
||||
# run to one cell the same way.
|
||||
|
||||
# Watch the most recent run.
|
||||
journalctl -u litellm-compat-matrix.service -f
|
||||
|
||||
# Read older runs.
|
||||
journalctl -u litellm-compat-matrix.service --since '2 days ago'
|
||||
|
||||
# Disable until further notice (e.g. while debugging).
|
||||
sudo systemctl disable --now litellm-compat-matrix.timer
|
||||
# Build and run the image locally (docker on Apple silicon needs the
|
||||
# platform flag; the context is tests/e2e, see the table above).
|
||||
docker build --platform linux/amd64 \
|
||||
-f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e
|
||||
docker run --rm --platform linux/amd64 \
|
||||
--env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \
|
||||
-v "$PWD/secrets:/etc/secrets:ro" compat-matrix
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The
|
||||
e2e suite uses PEP 695 `type` aliases, which the VM's system Python
|
||||
(3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython
|
||||
e2e suite uses PEP 695 `type` aliases, which the image's Debian
|
||||
Python can't parse; `run_daily.sh` has uv fetch a managed CPython
|
||||
into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against
|
||||
it. The first run after a version bump is a cold venv rebuild.
|
||||
- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd
|
||||
into the same VM with their own `:4000` proxy doesn't collide with a
|
||||
cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env`
|
||||
if you need to.
|
||||
it.
|
||||
- **The proxy port is `4100`, not `4000`.** Kept from the VM days so a
|
||||
developer running the script locally next to their own `:4000` proxy
|
||||
doesn't collide. Override with `PROXY_PORT=...`.
|
||||
- **`uv sync --frozen` requires the resolved tag to be tagged on
|
||||
GitHub.** If the latest stable release was made but not pushed as a
|
||||
git tag, the `git checkout` step fails. Push the tag, then rerun.
|
||||
GitHub, and the wheel install requires it on PyPI.** If the latest
|
||||
stable release was made but not pushed as a git tag, the `git
|
||||
checkout` step fails; push the tag, then rerun. PyPI has had every
|
||||
stable version days before its GitHub release so far (1.102.0 was
|
||||
uploaded 2026-09-20, released on GitHub 2026-09-22), so the
|
||||
`--no-build` install failing means the wheel is genuinely missing,
|
||||
not late.
|
||||
- **Pushes do not redeploy the service; deploy by hand.** `autoDeploy`
|
||||
is `yes` on the service, but Render only hears about pushes through
|
||||
its GitHub app, which is not installed on the `BerriAI` org (an org
|
||||
admin step), so no push to the branch has ever started a deploy.
|
||||
After a merge that changes anything under `tests/e2e/**`, run the
|
||||
deploy command from the operating section (or "Manual Deploy" on the
|
||||
dashboard) and wait for `live` before triggering a run, otherwise
|
||||
the next scheduled run still executes the old image.
|
||||
- **Publish-token rotation is your problem.** The cron does not
|
||||
refresh the token; if `mateo-berri`'s PAT in
|
||||
`/etc/litellm-compat-matrix-github-token` expires, the run fails at
|
||||
the `git push`/`gh pr create` step with a 401 ("Bad credentials" /
|
||||
"Authentication failed"). Mint a fresh PAT and update that file.
|
||||
The token needs write access to `BerriAI/litellm-docs` (classic
|
||||
`repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is
|
||||
delivered via systemd `LoadCredential`, not the env file, so pytest,
|
||||
the proxy, and the claude CLI never inherit it; manual runs export
|
||||
`GITHUB_TOKEN` instead.
|
||||
- **First run after upgrading the Claude Code CLI is the riskiest one.**
|
||||
If the new CLI changes its wire format the matrix run can produce
|
||||
systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI
|
||||
upgrade before letting the next scheduled fire happen.
|
||||
- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory
|
||||
is ~1 GB. Plan for at least 5 GB free on the VM, otherwise
|
||||
`uv sync` will fail mid-run and leave you with a half-installed venv.
|
||||
refresh the token; if `mateo-berri`'s PAT in the `github-token`
|
||||
secret file expires, the run fails at the `git push`/`gh pr create`
|
||||
step with a 401 ("Bad credentials" / "Authentication failed"). Mint
|
||||
a fresh PAT and replace the secret file on the service. The token
|
||||
needs write access to `BerriAI/litellm-docs` (classic `repo` scope,
|
||||
or fine-grained Contents:RW + Pull requests:RW). It is delivered as
|
||||
a file, not an env var, so pytest, the proxy, and the claude CLI
|
||||
never inherit it; manual runs export `GITHUB_TOKEN` instead.
|
||||
- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION`
|
||||
in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64`
|
||||
checksum from
|
||||
`https://downloads.claude.ai/claude-code-releases/<version>/manifest.json`.
|
||||
The first run on a new CLI is the riskiest one: if the new CLI
|
||||
changes its wire format the matrix run can produce systematic
|
||||
failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled
|
||||
fire. `gh` and `uv` bump the same way, with the checksum from the
|
||||
release's `gh_<version>_checksums.txt` and the tarball's `.sha256`
|
||||
sidecar respectively.
|
||||
- **A local build on Apple silicon only proves the image assembles.**
|
||||
Under QEMU the Claude Code binary (a Bun executable) dies with
|
||||
`CPU lacks AVX support` and `gh` panics in the Go runtime, so
|
||||
`claude --version` and a full run are verified with a
|
||||
`SKIP_PUBLISH=1` run on Render, not locally.
|
||||
- **Nothing persists between runs.** A failed run leaves no
|
||||
half-installed venv behind, but also no cache: don't expect a rerun
|
||||
to be faster than the first one.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
# Environment file consumed by `litellm-compat-matrix.service`.
|
||||
# Environment variables of the Render cron job `litellm-compat-matrix`.
|
||||
#
|
||||
# Install at `/etc/litellm-compat-matrix.env` and chmod 0600.
|
||||
# `EnvironmentFile=-` in the unit means the service is allowed to start
|
||||
# even if this file is missing, but the populator will fail at the
|
||||
# first provider request without these credentials.
|
||||
# Set every value here on the Render service (Environment tab, or the
|
||||
# `envVars` list of the create-service call in README.md). A local run
|
||||
# passes a filled-in copy with `docker run --env-file`.
|
||||
|
||||
# Anthropic
|
||||
ANTHROPIC_API_KEY=
|
||||
|
|
@ -17,11 +16,12 @@ AWS_BEARER_TOKEN_BEDROCK=
|
|||
AWS_REGION_NAME=us-east-1
|
||||
|
||||
# Vertex AI (vertex_ai + vertex_ai_gpt columns).
|
||||
# On the GCP VM, the default service-account ADC from the metadata server
|
||||
# is used -- no JSON key file is needed. If you ever need to run outside
|
||||
# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json.
|
||||
# The service-account key JSON is the Render secret file
|
||||
# `vertex-service-account.json`, mounted at /etc/secrets, and
|
||||
# GOOGLE_APPLICATION_CREDENTIALS points google-auth at it.
|
||||
VERTEXAI_PROJECT=
|
||||
VERTEXAI_LOCATION=global
|
||||
GOOGLE_APPLICATION_CREDENTIALS=/etc/secrets/vertex-service-account.json
|
||||
|
||||
# Azure AI Foundry (azure column — Claude models on Foundry)
|
||||
AZURE_AI_API_KEY=
|
||||
|
|
@ -35,19 +35,20 @@ AZURE_API_BASE=
|
|||
AZURE_API_KEY=
|
||||
|
||||
# The publish PAT (mateo-berri, write access on BerriAI/litellm-docs)
|
||||
# deliberately does NOT live in this file. Everything here lands in the
|
||||
# process environment of pytest, the proxy, and the model-driven claude
|
||||
# CLI, where any same-UID reader can lift it from /proc/<pid>/environ.
|
||||
# Instead, install the token at /etc/litellm-compat-matrix-github-token
|
||||
# (chmod 0600, single line); the service maps it in via systemd
|
||||
# LoadCredential and run_daily.sh keeps it out of every child process
|
||||
# env. Used to (a) resolve the latest stable release, (b) push the
|
||||
# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open
|
||||
# the same-repo PR, and (d) enable squash auto-merge on it. Scopes:
|
||||
# deliberately is NOT an env var. Everything here lands in the process
|
||||
# environment of pytest, the proxy, and the model-driven claude CLI,
|
||||
# where any same-UID reader can lift it from /proc/<pid>/environ.
|
||||
# Instead, the token is the Render secret file `github-token` (single
|
||||
# line), mounted under CREDENTIALS_DIRECTORY, and run_daily.sh reads it
|
||||
# from there and keeps it out of every child process env. Used to
|
||||
# (a) resolve the latest stable release, (b) push the daily
|
||||
# compat-matrix branch directly to BerriAI/litellm-docs, (c) open the
|
||||
# same-repo PR, and (d) enable squash auto-merge on it. Scopes:
|
||||
# classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs
|
||||
# with Contents:RW + Pull requests:RW + Workflows:RW.
|
||||
# Manual runs export GITHUB_TOKEN instead, or skip publishing entirely
|
||||
# with SKIP_PUBLISH=1 (only writes the matrix JSON locally).
|
||||
CREDENTIALS_DIRECTORY=/etc/secrets
|
||||
|
||||
# Optional: the bedrock_mantle column is opt-in because the AWS account
|
||||
# needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the
|
||||
|
|
@ -59,9 +60,9 @@ AZURE_API_KEY=
|
|||
# usually run them. Skipped cells are recorded as not_tested.
|
||||
# COMPAT_OPENAI_GPT_CELLS=1
|
||||
|
||||
# Optional overrides; defaults are sensible for the cron VM.
|
||||
# Optional overrides; defaults are sensible for the cron job.
|
||||
# PROXY_PORT=4100
|
||||
# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree
|
||||
# LITELLM_WORKTREE=/home/populator/litellm-cron-worktree
|
||||
# DOCS_REPO=BerriAI/litellm-docs
|
||||
# DOCS_BRANCH=main
|
||||
# DOCS_TARGET_PATH=src/data/compatibility-matrix.json
|
||||
|
|
|
|||
|
|
@ -1,113 +0,0 @@
|
|||
# systemd service for the Claude Code compatibility-matrix populator.
|
||||
#
|
||||
# Triggered by `litellm-compat-matrix.timer`; not started directly. The
|
||||
# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics
|
||||
# describe "run once per day" cleanly — there's no long-lived daemon to
|
||||
# supervise; each invocation runs the populator end-to-end and exits.
|
||||
#
|
||||
# Install
|
||||
# -------
|
||||
#
|
||||
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
|
||||
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl enable --now litellm-compat-matrix.timer
|
||||
#
|
||||
# Paths are hard-coded to /home/mateo rather than using systemd's %h
|
||||
# specifier. Why: in *system* units (this one), %h is expanded at
|
||||
# parse time against the *manager's* home -- which is /root for PID 1
|
||||
# -- and *not* against the User= directive. That mismatch makes
|
||||
# ReadWritePaths point at /root/.cache (which doesn't exist), causing
|
||||
# the namespace setup to fail with status=226/NAMESPACE before the
|
||||
# script ever runs. The runtime user (`User=mateo`) must:
|
||||
#
|
||||
# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the
|
||||
# publisher module is importable;
|
||||
# * have a uv venv at `~/litellm/litellm/.venv` (created by
|
||||
# `uv sync --frozen` inside that checkout once);
|
||||
# * have `gh` already authenticated against an account with
|
||||
# `pull-requests: write` on `BerriAI/litellm-docs`;
|
||||
# * have provider credentials exported in `/etc/litellm-compat-matrix.env`
|
||||
# (see `litellm-compat-matrix.env.example` in this directory);
|
||||
# * have the mateo-berri publish PAT at
|
||||
# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single
|
||||
# line), delivered via `LoadCredential=` below.
|
||||
|
||||
[Unit]
|
||||
Description=Claude Code compatibility-matrix populator (oneshot)
|
||||
Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=mateo
|
||||
Group=mateo
|
||||
|
||||
# Provider credentials + any gh/PROXY_PORT overrides live here. Format
|
||||
# is the standard `KEY=value` one line per env var.
|
||||
EnvironmentFile=-/etc/litellm-compat-matrix.env
|
||||
|
||||
# The mateo-berri publish PAT is mapped in via the credential store, NOT
|
||||
# the EnvironmentFile, so it never lands in the process environment that
|
||||
# pytest, the proxy, and the model-driven claude CLI inherit (any
|
||||
# same-UID process can read /proc/<pid>/environ). run_daily.sh reads
|
||||
# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call.
|
||||
# Unlike EnvironmentFile= above, this is deliberately NOT optional: a
|
||||
# missing token file fails the unit at start instead of 30 minutes in.
|
||||
LoadCredential=github-token:/etc/litellm-compat-matrix-github-token
|
||||
|
||||
# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin).
|
||||
# `uv` and `claude` are installed under the runtime user's `~/.local/bin`
|
||||
# so we have to prepend it explicitly; otherwise run_daily.sh fails at
|
||||
# the up-front command-presence check.
|
||||
Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be
|
||||
# explicit so anything that reads $HOME (e.g. uv's cache lookup, the
|
||||
# claude CLI's per-session dir) sees the right value even if a future
|
||||
# refactor flips DynamicUser= or PrivateUsers= on.
|
||||
Environment=HOME=/home/mateo
|
||||
|
||||
WorkingDirectory=/home/mateo/litellm/litellm
|
||||
|
||||
ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
|
||||
# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new
|
||||
# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM,
|
||||
# plus the full feature x provider grid of pytest cells hitting several
|
||||
# cloud providers.
|
||||
TimeoutStartSec=90min
|
||||
|
||||
# A failed run shouldn't restart automatically — the next timer fire is
|
||||
# the right retry. Reruns of the same day's matrix are idempotent.
|
||||
Restart=no
|
||||
|
||||
# Security hardening: the populator only reads the litellm checkout and
|
||||
# the env-file; everything else it writes lives in either the worktree
|
||||
# (managed) or `/tmp` (cleaned up by tempfile).
|
||||
#
|
||||
# ReadWritePaths whitelist:
|
||||
# * litellm-cron-worktree - the long-lived stable-tag checkout +
|
||||
# its `.venv` (`uv sync` rewrites every
|
||||
# run) + `.uv-bin` (pinned `uv` binary
|
||||
# cache).
|
||||
# * .cache - uv's wheel cache (~/.cache/uv) so we
|
||||
# don't redownload pinned deps each run.
|
||||
# * .claude - `claude` CLI's per-session state under
|
||||
# `~/.claude/projects/<sha>/`; created
|
||||
# on every `claude --print` invocation.
|
||||
# * .config/gh - `gh` CLI host config; technically not
|
||||
# needed when we pass GH_TOKEN inline,
|
||||
# but cheap to whitelist and prevents
|
||||
# future regressions if a code path
|
||||
# ever falls back to the host config.
|
||||
# * /tmp - mktemp -d workdir + proxy logs.
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Daily timer for the compatibility-matrix populator.
|
||||
#
|
||||
# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so
|
||||
# operators in US/EU timezones see fresh PRs at the start of their work
|
||||
# day.
|
||||
#
|
||||
# `Persistent=true` causes a missed run (VM was off / suspended) to
|
||||
# fire the next time the timer is started, which is the property we
|
||||
# want for a once-a-day job: the matrix should refresh as soon as the
|
||||
# VM is reachable again, not wait another 24h.
|
||||
#
|
||||
# `RandomizedDelaySec=10min` smears load if multiple matrix-style
|
||||
# pipelines are ever colocated on the same VM in the future.
|
||||
|
||||
[Unit]
|
||||
Description=Run the Claude Code compatibility-matrix populator daily
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 06:00:00 UTC
|
||||
Persistent=true
|
||||
RandomizedDelaySec=10min
|
||||
Unit=litellm-compat-matrix.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# Daily Claude Code compatibility-matrix populator.
|
||||
#
|
||||
# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the
|
||||
# systemd timer in this directory. The flow is:
|
||||
# Runs daily as the Render cron job `litellm-compat-matrix`, built from
|
||||
# the Dockerfile in this directory (see README.md). The flow is:
|
||||
#
|
||||
# 1. Resolve the latest LiteLLM final release tag from the GitHub
|
||||
# Releases API.
|
||||
|
|
@ -33,12 +33,12 @@
|
|||
# rather than spawning a new one. If the JSON is byte-identical to the
|
||||
# docs branch, we skip the push entirely.
|
||||
#
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm.
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, claude.
|
||||
# Required state: a litellm checkout at $LITELLM_REPO (this file lives in
|
||||
# it), $WORKTREE is created on first run, gh is already authenticated.
|
||||
# it); $WORKTREE is created on first run.
|
||||
#
|
||||
# Override any default by setting the matching env var; see the systemd
|
||||
# unit for the production wiring.
|
||||
# Override any default by setting the matching env var; see README.md
|
||||
# for the production wiring.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
|
|
@ -52,12 +52,12 @@ DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}"
|
|||
SKIP_PUBLISH="${SKIP_PUBLISH:-0}"
|
||||
PYTEST_K="${PYTEST_K:-}"
|
||||
# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python
|
||||
# >= 3.12 (also what repo CI runs) even when the VM's system python is
|
||||
# >= 3.12 (also what repo CI runs) even when the host's system python is
|
||||
# older. uv fetches a managed CPython of this version on first use --
|
||||
# checksum-verified against the manifest baked into the pinned uv
|
||||
# binary -- and installs it under ${WORKTREE}/.uv-python (see
|
||||
# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the
|
||||
# systemd sandbox lets us write to.
|
||||
# UV_PYTHON_INSTALL_DIR below) so everything the run writes lives inside
|
||||
# the worktree.
|
||||
CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}"
|
||||
# Merge method for auto-merge. BerriAI/litellm-docs only allows squash
|
||||
# merges (merge-commit and rebase are disabled at the repo level), so
|
||||
|
|
@ -113,9 +113,9 @@ for cmd in git uv gh jq curl claude; do
|
|||
done
|
||||
|
||||
# Publishing pushes the branch straight to BerriAI/litellm-docs and opens
|
||||
# the PR as mateo-berri, who has write access on the docs repo. Under
|
||||
# systemd the PAT arrives as a file via LoadCredential=, NOT via the
|
||||
# EnvironmentFile: several suite cells let the model-driven claude CLI
|
||||
# the PR as mateo-berri, who has write access on the docs repo. On
|
||||
# Render the PAT arrives as a secret file under ${CREDENTIALS_DIRECTORY},
|
||||
# NOT via an env var: several suite cells let the model-driven claude CLI
|
||||
# read arbitrary files as this user, and /proc/<pid>/environ of the
|
||||
# script, pytest, and the proxy would hand an env-borne token to any
|
||||
# same-UID reader. Kept as an unexported shell variable and passed per
|
||||
|
|
@ -125,13 +125,18 @@ done
|
|||
# quota.
|
||||
if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then
|
||||
GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")"
|
||||
log "publish token source: systemd credential store"
|
||||
log "publish token source: ${CREDENTIALS_DIRECTORY}/github-token"
|
||||
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
log "publish token source: process environment"
|
||||
fi
|
||||
if [[ "${SKIP_PUBLISH}" != "1" ]]; then
|
||||
[[ -n "${GITHUB_TOKEN:-}" ]] \
|
||||
|| die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|
||||
|| die "publish token required: the github-token secret file under CREDENTIALS_DIRECTORY, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|
||||
# The stale-PR sweep below closes only PRs this account opened, so the
|
||||
# login is resolved from the token once rather than hardcoded.
|
||||
PUBLISH_LOGIN="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq .login)" \
|
||||
|| die "could not resolve the publishing account from the github token"
|
||||
log "publishing as ${PUBLISH_LOGIN}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -205,7 +210,7 @@ log "local claude code: ${CLAUDE_CODE_VERSION}"
|
|||
if [[ ! -d "${WORKTREE}/.git" ]]; then
|
||||
log "first run: cloning litellm into ${WORKTREE}"
|
||||
mkdir -p "$(dirname "${WORKTREE}")"
|
||||
git clone https://github.com/BerriAI/litellm.git "${WORKTREE}"
|
||||
git clone --filter=blob:none https://github.com/BerriAI/litellm.git "${WORKTREE}"
|
||||
fi
|
||||
|
||||
log "updating worktree to ${LITELLM_VERSION}"
|
||||
|
|
@ -221,40 +226,27 @@ git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python
|
|||
git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}"
|
||||
|
||||
# Always rebuild tests/e2e/ in the worktree from the dev checkout,
|
||||
# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two
|
||||
# reasons:
|
||||
# regardless of what the resolved ${LITELLM_VERSION} tag ships: the
|
||||
# matrix populator's job is to exercise *today's* tests against the
|
||||
# latest stable proxy, and the dev checkout carries the most recent
|
||||
# test fixes that haven't yet rolled into a stable release.
|
||||
#
|
||||
# * The matrix populator's job is to exercise *today's* tests against
|
||||
# the latest stable proxy. The dev checkout carries the most recent
|
||||
# test fixes that haven't yet rolled into a stable release, and we
|
||||
# want every cron run to pick those up the moment they land on
|
||||
# ${LITELLM_REPO}, not whenever the next stable release happens.
|
||||
# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose
|
||||
# top-level conftest.py imports modules (e2e_db, lifecycle,
|
||||
# otel_client, ...) that the stable venv does not install. Copying
|
||||
# the whole tree would make pytest collection blow up on those
|
||||
# imports.
|
||||
#
|
||||
# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY
|
||||
# the claude_code suite plus the shared transport helpers it imports.
|
||||
# The whole tree is copied rather than an allowlist of the helpers the
|
||||
# suite imports: the helpers import each other (proxy_client ->
|
||||
# e2e_config -> fixture_mode -> ...), so a new edge in that graph turned
|
||||
# an allowlist into a ModuleNotFoundError at conftest load. The tree's
|
||||
# top-level conftest.py pulls in the full EKS harness (e2e_db,
|
||||
# lifecycle, ...), which the stable venv does not install, so the pytest
|
||||
# run below points --confcutdir at claude_code/ and never loads it.
|
||||
# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while
|
||||
# claude_code/ does), which is what resolves both the `claude_code.*`
|
||||
# and the bare `proxy_client` / `e2e_http` imports inside the suite.
|
||||
E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py)
|
||||
if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then
|
||||
die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
|
||||
fi
|
||||
for helper in "${E2E_HELPER_FILES[@]}"; do
|
||||
[[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \
|
||||
|| die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}"
|
||||
done
|
||||
log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)"
|
||||
[[ -d "${LITELLM_REPO}/tests/e2e/claude_code" ]] \
|
||||
|| die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
|
||||
log "shimming tests/e2e/ from ${LITELLM_REPO} (always-overwrite)"
|
||||
rm -rf "${WORKTREE}/tests/e2e"
|
||||
mkdir -p "${WORKTREE}/tests/e2e"
|
||||
cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/"
|
||||
for helper in "${E2E_HELPER_FILES[@]}"; do
|
||||
cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/"
|
||||
done
|
||||
cp -r "${LITELLM_REPO}/tests/e2e/." "${WORKTREE}/tests/e2e/"
|
||||
|
||||
# litellm pins an exact uv version in pyproject.toml's [tool.uv]
|
||||
# `required-version` field, so a system uv that's newer or older
|
||||
|
|
@ -305,10 +297,18 @@ fi
|
|||
# actually serve. `--group proxy-dev` brings in pytest and the rest of
|
||||
# what tests/e2e/claude_code/ needs. `--python` pins the venv to
|
||||
# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates
|
||||
# the venv from scratch (a one-time cold sync).
|
||||
# the venv from scratch (a one-time cold sync). `--no-install-project`
|
||||
# leaves litellm itself out: the tag builds a Rust extension through
|
||||
# maturin, which needs a C and Rust toolchain the image does not carry,
|
||||
# so the published PyPI wheel (what users install) goes in right after,
|
||||
# and every later `uv run` passes `--no-sync` so uv never tries to put
|
||||
# the source build back.
|
||||
export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python"
|
||||
log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
|
||||
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}")
|
||||
log "uv sync --frozen --group proxy-dev --extra proxy --no-install-project --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
|
||||
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --no-install-project --python "${CRON_PYTHON_VERSION}")
|
||||
LITELLM_WHEEL_VERSION="${LITELLM_VERSION#v}"
|
||||
log "installing the published litellm==${LITELLM_WHEEL_VERSION} wheel from PyPI"
|
||||
"${WORKTREE_UV}" pip install --python "${WORKTREE}/.venv/bin/python" --no-deps --no-build "litellm==${LITELLM_WHEEL_VERSION}"
|
||||
|
||||
PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
|
||||
[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)"
|
||||
|
|
@ -321,10 +321,10 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
|
|||
# Bind the proxy to loopback only. The populator proxy is talked to
|
||||
# exclusively by the pytest run on the same host (the health check and
|
||||
# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`),
|
||||
# so there's no reason to expose it on the VM's external interfaces.
|
||||
# so there's no reason to expose it on the container's external interfaces.
|
||||
# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with
|
||||
# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would
|
||||
# allow anything that can reach :${PROXY_PORT} on the VM to authenticate
|
||||
# allow anything that can reach :${PROXY_PORT} on the host to authenticate
|
||||
# and burn upstream provider credentials.
|
||||
#
|
||||
# `setsid` puts the proxy in its own session+pgroup so cleanup() can
|
||||
|
|
@ -334,7 +334,7 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
|
|||
setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c '
|
||||
echo "$$" > "$0"
|
||||
cd "$1"
|
||||
exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4"
|
||||
exec "$2" run --no-sync litellm --config "$3" --host 127.0.0.1 --port "$4"
|
||||
' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \
|
||||
>"${WORKDIR}/proxy.log" 2>&1 &
|
||||
disown
|
||||
|
|
@ -359,6 +359,7 @@ RESULTS_JSON="${WORKDIR}/compat-results.json"
|
|||
# the cron skips them if/when they land in the suite.
|
||||
PYTEST_ARGS=(
|
||||
tests/e2e/claude_code/
|
||||
--confcutdir=tests/e2e/claude_code
|
||||
"--ignore-glob=*_unit_tests*"
|
||||
)
|
||||
if [[ -n "${PYTEST_K}" ]]; then
|
||||
|
|
@ -373,7 +374,7 @@ set +e
|
|||
&& LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \
|
||||
LITELLM_MASTER_KEY="${PROXY_API_KEY}" \
|
||||
COMPAT_RESULTS_PATH="${RESULTS_JSON}" \
|
||||
"${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}"
|
||||
"${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}"
|
||||
)
|
||||
PYTEST_EXIT=$?
|
||||
set -e
|
||||
|
|
@ -392,7 +393,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json"
|
|||
log "building ${MATRIX_JSON}"
|
||||
(
|
||||
cd "${WORKTREE}" \
|
||||
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/build_matrix.py" \
|
||||
--manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \
|
||||
--results "${RESULTS_JSON}" \
|
||||
--output "${MATRIX_JSON}" \
|
||||
|
|
@ -405,8 +406,9 @@ log "building ${MATRIX_JSON}"
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [[ "${SKIP_PUBLISH}" == "1" ]]; then
|
||||
cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json"
|
||||
log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json"
|
||||
cp "${MATRIX_JSON}" "${HOME}/compatibility-matrix.json"
|
||||
log "SKIP_PUBLISH=1; matrix saved to ${HOME}/compatibility-matrix.json and printed below"
|
||||
cat "${MATRIX_JSON}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -415,7 +417,7 @@ BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}
|
|||
DOCS_CLONE="${WORKDIR}/litellm-docs"
|
||||
|
||||
log "cloning ${DOCS_REPO}@${DOCS_BRANCH}"
|
||||
gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
|
||||
GH_TOKEN="${GITHUB_TOKEN}" gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
|
||||
|
||||
cd "${DOCS_CLONE}"
|
||||
git config user.email "litellm-bot@berri.ai"
|
||||
|
|
@ -452,7 +454,7 @@ log "checking for green->red regressions vs the published matrix"
|
|||
set +e
|
||||
REGRESSION_REPORT="$(
|
||||
cd "${WORKTREE}" \
|
||||
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/check_regressions.py" \
|
||||
--old "${PUBLISHED_MATRIX}" \
|
||||
--new "${MATRIX_JSON}"
|
||||
)"
|
||||
|
|
@ -492,7 +494,7 @@ git commit -m "${COMMIT_MSG}"
|
|||
#
|
||||
# Plain --force (not --force-with-lease) is acceptable here: the
|
||||
# compat-matrix/* branch is bot-owned, only this script ever writes to
|
||||
# it, and runs are serialized by the systemd timer. --force-with-lease
|
||||
# it, and runs are serialized by the cron schedule. --force-with-lease
|
||||
# would require a fetch to populate the remote-tracking ref before each
|
||||
# push and adds no safety in this single-writer setup.
|
||||
PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git"
|
||||
|
|
@ -553,7 +555,7 @@ Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without mergi
|
|||
EOF
|
||||
)"
|
||||
|
||||
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)"
|
||||
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as ${PUBLISH_LOGIN})"
|
||||
# GH_TOKEN is mateo-berri's write-scoped token, the same identity used
|
||||
# for release-listing above. The branch lives on ${DOCS_REPO} itself, so
|
||||
# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`.
|
||||
|
|
@ -644,15 +646,25 @@ fi
|
|||
#
|
||||
# Non-fatal: a sweep failure (rate limit, transient API error) leaves
|
||||
# stale PRs for the next run to retry; it must not fail the pipeline.
|
||||
#
|
||||
# The docs repo carries a few hundred open PRs, so the list has to page
|
||||
# past gh's default 30 (and the earlier 100, which never reached a
|
||||
# week-old compat-matrix PR and left it open for good).
|
||||
#
|
||||
# `compat-matrix/` is only a naming convention, so the prefix alone does
|
||||
# not make a PR this job's: a contributor can open a fork PR under that
|
||||
# name. Only PRs the publishing account itself opened from a branch on
|
||||
# the docs repo qualify; anything else stays untouched.
|
||||
log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})"
|
||||
set +e
|
||||
STALE_PRS="$(
|
||||
GH_TOKEN="${GITHUB_TOKEN}" gh pr list \
|
||||
--repo "${DOCS_REPO}" \
|
||||
--state open \
|
||||
--limit 100 \
|
||||
--json number,headRefName \
|
||||
--jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"'
|
||||
--author "${PUBLISH_LOGIN}" \
|
||||
--limit 1000 \
|
||||
--json number,headRefName,isCrossRepository \
|
||||
--jq '.[] | select((.headRefName | startswith("compat-matrix/")) and (.isCrossRepository | not)) | "\(.number)\t\(.headRefName)"'
|
||||
)"
|
||||
while IFS=$'\t' read -r stale_pr stale_head; do
|
||||
[[ -z "${stale_pr}" ]] && continue
|
||||
|
|
@ -660,7 +672,7 @@ while IFS=$'\t' read -r stale_pr stale_head; do
|
|||
GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \
|
||||
--repo "${DOCS_REPO}" \
|
||||
--delete-branch \
|
||||
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /'
|
||||
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open" 2>&1 | sed 's/^/ /'
|
||||
if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
|
||||
log "closed stale compat-matrix PR #${stale_pr} (${stale_head})"
|
||||
else
|
||||
|
|
|
|||
|
|
@ -345,6 +345,50 @@ def request_with_retry[T: RetryableResponse](
|
|||
return issue()
|
||||
|
||||
|
||||
PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError"
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4
|
||||
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0
|
||||
|
||||
|
||||
def tolerate_provider_rate_limit[R: BaseModel](
|
||||
issue: Callable[[], Result[R]],
|
||||
*,
|
||||
attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> Result[R]:
|
||||
"""Retry a call up to `attempts` times while the proxy relays the provider's own 429;
|
||||
any other outcome, the proxy's own 429 included, comes back at once."""
|
||||
for attempt in range(1, attempts):
|
||||
match issue():
|
||||
case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body:
|
||||
delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1))
|
||||
print(
|
||||
f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s",
|
||||
flush=True,
|
||||
)
|
||||
sleep(delay)
|
||||
case result:
|
||||
return result
|
||||
return issue()
|
||||
|
||||
|
||||
class ProxyErrorDetail(BaseModel):
|
||||
message: str
|
||||
type: str
|
||||
code: str
|
||||
|
||||
|
||||
class _ProxyErrorBody(BaseModel):
|
||||
error: ProxyErrorDetail
|
||||
|
||||
|
||||
def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None:
|
||||
"""The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own."""
|
||||
if PROVIDER_RATE_LIMIT_MARKER not in outcome.body:
|
||||
return None
|
||||
return _ProxyErrorBody.model_validate_json(outcome.body).error
|
||||
|
||||
|
||||
class ClassifiableResponse(Protocol):
|
||||
"""What classifying an outcome reads off a response. requests.Response satisfies
|
||||
it, and so does a fake, so the classification rules are testable on their own."""
|
||||
|
|
@ -931,7 +975,10 @@ class PreparedForward:
|
|||
|
||||
|
||||
def prepare_forward(
|
||||
method: str, url: str, headers: dict[str, str], body: bytes | None,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes | None,
|
||||
) -> PreparedForward | NetworkError:
|
||||
try:
|
||||
with requests.Session() as session:
|
||||
|
|
@ -950,7 +997,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream
|
|||
except requests.RequestException as exc:
|
||||
return NetworkError(message=str(exc))
|
||||
return StreamHead(
|
||||
resp.status_code, {name.lower(): value for name, value in resp.headers.items()},
|
||||
resp.status_code,
|
||||
{name.lower(): value for name, value in resp.headers.items()},
|
||||
primed_steps(_stream_steps(resp)),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
|
|||
call, so a never-seen prefix must come back cached on its very first call
|
||||
(Gemini's implicit caching cannot hit a cold prefix), the cached count must
|
||||
cover the marked block, and the spend row must be billed below the uncached
|
||||
price of the prompt.
|
||||
price of the cached tokens. Vertex's cache create is nondeterministic:
|
||||
identical bodies come back 200 or with the minimum-token 400 ("The cached
|
||||
content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to
|
||||
eight never-seen prefixes are tried with a pause after each rejection. The
|
||||
billing check prices the cached tokens rather than prompt_tokens, which
|
||||
Vertex reports inclusive of the cached prefix on some calls and exclusive
|
||||
of it on others.
|
||||
- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over
|
||||
the OpenAI-compatible route; the second call must report cache-read tokens > 0.
|
||||
- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the
|
||||
|
|
@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
|
|||
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
|
||||
OPENAI_MODEL = "openai/gpt-5.6"
|
||||
VERTEX_CACHE_TTL: Final = "300s"
|
||||
VERTEX_COLD_CALL_ATTEMPTS: Final = 3
|
||||
VERTEX_COLD_CALL_ATTEMPTS: Final = 8
|
||||
VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0
|
||||
VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024
|
||||
CACHED_SHARE_OF_PROMPT: Final = 0.9
|
||||
VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching"
|
||||
|
|
@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons
|
|||
return unwrap(result)
|
||||
|
||||
|
||||
def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None:
|
||||
for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1):
|
||||
candidate = _cold_cache_call(send)
|
||||
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS:
|
||||
return candidate
|
||||
if attempt < VERTEX_COLD_CALL_ATTEMPTS:
|
||||
print(
|
||||
f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; "
|
||||
f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS)
|
||||
return None
|
||||
|
||||
|
||||
def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse:
|
||||
completion: Final = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS))
|
||||
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS
|
||||
),
|
||||
None,
|
||||
)
|
||||
completion: Final = _first_engaged_cold_call(send)
|
||||
assert completion is not None, (
|
||||
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either "
|
||||
f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} "
|
||||
"cached tokens on their first call; explicit context caching did not engage"
|
||||
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over "
|
||||
f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by "
|
||||
f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on "
|
||||
"their first call; explicit context caching did not engage"
|
||||
)
|
||||
assert completion.choices, f"{model}: cached call returned no choices: {completion}"
|
||||
usage: Final = completion.usage
|
||||
cached: Final = _cached_read_tokens(usage)
|
||||
assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
|
||||
f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the "
|
||||
"cache; the cache_control block was not cached whole"
|
||||
assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}"
|
||||
assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
|
||||
f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the "
|
||||
"cache_control block was not cached whole"
|
||||
)
|
||||
return completion
|
||||
|
||||
|
|
@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str,
|
|||
assert row.prompt_tokens == usage.prompt_tokens, (
|
||||
f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}"
|
||||
)
|
||||
uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model)
|
||||
assert row.spend is not None and row.spend < uncached_prompt_cost, (
|
||||
f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for "
|
||||
f"{usage.prompt_tokens} tokens); cache-read pricing was not applied"
|
||||
cached: Final = _cached_read_tokens(usage)
|
||||
uncached_read_cost: Final = cached * _input_rate(client, model)
|
||||
assert row.spend is not None and row.spend < uncached_read_cost, (
|
||||
f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache "
|
||||
f"({uncached_read_cost}); cache-read pricing was not applied"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]:
|
|||
|
||||
def _assert_describes_cat(response: ChatResponse) -> None:
|
||||
assert response.choices, f"vision returned no choices: {response}"
|
||||
message = response.choices[0].message
|
||||
content = (message.content if message else None) or ""
|
||||
choice = response.choices[0]
|
||||
content = (choice.message.content if choice.message else None) or ""
|
||||
assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), (
|
||||
f"vision response did not describe the image: {content[:200]}"
|
||||
f"vision response did not describe the image: {content[:200]!r} "
|
||||
f"(finish_reason={choice.finish_reason!r}, usage={response.usage})"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -421,7 +422,11 @@ class TestVertexChatCompletions:
|
|||
model = self._register(client, resources, "e2e-vertex-vision")
|
||||
key = resources.key()
|
||||
|
||||
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
|
||||
response = unwrap(
|
||||
client.proxy.chat(
|
||||
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none")
|
||||
)
|
||||
)
|
||||
_assert_describes_cat(response)
|
||||
|
||||
@pytest.mark.covers(
|
||||
|
|
|
|||
|
|
@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
|
|||
fails when no proxy answers and also fails once a request reaches it: the proxy
|
||||
fetches each provider's referenced secrets, so a
|
||||
missing credential surfaces as a live provider error rather than silent green.
|
||||
|
||||
The provider keys are shared with other pipelines, so a provider's rate limit can
|
||||
hold across the bounded retries. The case then accepts the gateway's faithful relay
|
||||
of that 429 (throttling_error, code 429) as its second expected outcome; any other
|
||||
non-success still fails at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from typing import Final, Protocol
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import assert_client_error, unwrap
|
||||
from e2e_http import (
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
RateLimitedError,
|
||||
Success,
|
||||
assert_client_error,
|
||||
relayed_provider_rate_limit,
|
||||
tolerate_provider_rate_limit,
|
||||
)
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
|
||||
from proxy_client import ProxyClient
|
||||
|
|
@ -146,24 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None:
|
|||
assert response.pages[0].markdown is not None, "first page has no markdown"
|
||||
|
||||
|
||||
def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None:
|
||||
detail: Final = relayed_provider_rate_limit(outcome)
|
||||
assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}"
|
||||
assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}"
|
||||
print(
|
||||
f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; "
|
||||
f"the gateway relayed it as {detail.type} {detail.code}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
class TestRustOcrGateway:
|
||||
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
|
||||
def test_rust_ocr_response(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase
|
||||
) -> None:
|
||||
def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None:
|
||||
model = f"rust-ocr-{case.suffix}-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, case.provider.litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document)))
|
||||
_assert_ocr_document(response)
|
||||
match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))):
|
||||
case Success(data=response):
|
||||
_assert_ocr_document(response)
|
||||
case RateLimitedError() as outcome:
|
||||
_assert_provider_rate_limit_relayed(model, outcome)
|
||||
case outcome:
|
||||
pytest.fail(f"{model}: {outcome!r}")
|
||||
|
||||
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")
|
||||
@pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works")
|
||||
def test_missing_document_returns_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
|
||||
model = f"rust-ocr-val-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, MistralOcr().litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
|
|
@ -174,4 +198,3 @@ class TestRustOcrGateway:
|
|||
json=_OptionalOcrBody(model=model),
|
||||
)
|
||||
assert_client_error(result, "ocr missing document")
|
||||
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ reliability behavior.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import ValidationError
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from proxy_client import ProxyClient
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker
|
||||
|
|
@ -367,6 +368,26 @@ def model_id_of(resp: StreamingResponse) -> str | None:
|
|||
return resp.headers.get("x-litellm-model-id")
|
||||
|
||||
|
||||
class _AzurePromptFilterResult(BaseModel):
|
||||
content_filter_results: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class _AzureAnnotatedChatBody(BaseModel):
|
||||
prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None
|
||||
|
||||
|
||||
def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool:
|
||||
"""True when Azure's 200 recorded no prompt-filter verdict (every `content_filter_results`
|
||||
empty), so the prompt has to be sent again."""
|
||||
try:
|
||||
annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body)
|
||||
except ValidationError:
|
||||
return False
|
||||
if not annotated.prompt_filter_results:
|
||||
return False
|
||||
return all(not entry.content_filter_results for entry in annotated.prompt_filter_results)
|
||||
|
||||
|
||||
def _parsed(resp: StreamingResponse) -> ChatResponse | None:
|
||||
try:
|
||||
return ChatResponse.model_validate_json(resp.body)
|
||||
|
|
|
|||
|
|
@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and
|
|||
reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure
|
||||
OpenAI content filter rejecting a jailbreak prompt, and a control call first
|
||||
proves the refusal reaches the customer as a 400 when no reroute is configured.
|
||||
Azure intermittently answers without running its prompt filter at all (the
|
||||
body's prompt_filter_results carry no verdict), which is not a pass, so both
|
||||
calls send the prompt again, a bounded number of times, until the filter ran.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from complexity_router_client import ComplexityRouterClient
|
||||
|
|
@ -29,6 +35,7 @@ from lifecycle import ResourceManager
|
|||
from models import RouterSettingsOverride
|
||||
from reliability_support import (
|
||||
CONTENT_POLICY_PROMPT,
|
||||
azure_prompt_filter_skipped,
|
||||
chat_override,
|
||||
completion_tokens_of,
|
||||
content_of,
|
||||
|
|
@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None:
|
|||
assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}"
|
||||
|
||||
|
||||
AZURE_FILTER_ATTEMPTS: Final = 3
|
||||
|
||||
|
||||
def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse:
|
||||
for attempt in range(1, AZURE_FILTER_ATTEMPTS):
|
||||
resp = send()
|
||||
if not azure_prompt_filter_skipped(resp):
|
||||
return resp
|
||||
print(
|
||||
"e2e: azure answered without running its prompt filter; sending the jailbreak prompt again "
|
||||
f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})",
|
||||
flush=True,
|
||||
)
|
||||
return send()
|
||||
|
||||
|
||||
def _filter_verdict(resp: StreamingResponse) -> str:
|
||||
if azure_prompt_filter_skipped(resp):
|
||||
return "azure skipped its prompt filter on every attempt"
|
||||
return "the filter ran and let the prompt through"
|
||||
|
||||
|
||||
class TestReliabilityFallbacks:
|
||||
@pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback")
|
||||
def test_5xx_routes_to_fallback(
|
||||
|
|
@ -124,17 +153,21 @@ class TestReliabilityFallbacks:
|
|||
model_id = create_content_filtered_deployment(client.proxy, primary)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
|
||||
refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
|
||||
refused = _chat_once_azure_runs_its_filter(
|
||||
lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
|
||||
)
|
||||
assert refused.status_code == 400, (
|
||||
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: "
|
||||
f"{refused.body[:300]}"
|
||||
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} "
|
||||
f"({_filter_verdict(refused)}): {refused.body[:300]}"
|
||||
)
|
||||
|
||||
resp = chat_override(
|
||||
client.proxy,
|
||||
scoped_key,
|
||||
primary,
|
||||
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
|
||||
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
|
||||
resp = _chat_once_azure_runs_its_filter(
|
||||
lambda: chat_override(
|
||||
client.proxy,
|
||||
scoped_key,
|
||||
primary,
|
||||
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
|
||||
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
|
||||
)
|
||||
)
|
||||
_assert_served_by_fallback(resp)
|
||||
|
|
|
|||
|
|
@ -290,6 +290,12 @@
|
|||
"tests/integration/spend/test_spend_calculate.py::test_spend_calculate_rejects_unpriced_model_with_400": [
|
||||
"quota_management.spend_tracking.spend_calculate.rejects_unpriced_model"
|
||||
],
|
||||
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini-live-2.5-flash-preview-native-audio-09-2025]": [
|
||||
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
|
||||
],
|
||||
"tests/integration/spend/test_spend_calculate.py::test_live_preview_entry_charges_cached_tokens_at_the_fresh_rate[gemini/gemini-live-2.5-flash-preview-native-audio-09-2025]": [
|
||||
"quota_management.spend_tracking.spend_calculate.live_preview_cached_tokens_cost_fresh_rate"
|
||||
],
|
||||
"tests/integration/management/test_partial_update_sequences.py::test_restricted_actor_cannot_detach_key_from_project": [
|
||||
"mgmt.key.update.project_detach_denied_to_restricted_actor"
|
||||
],
|
||||
|
|
@ -1810,6 +1816,126 @@
|
|||
"tests/integration/management/test_vector_store_config_ownership.py::test_redis_outage_keeps_config_store_served_and_recovers": [
|
||||
"mgmt.vector_store.chaos.redis_outage_keeps_config_store_and_recovers"
|
||||
],
|
||||
"tests/integration/providers/test_anthropic_advisor_wire.py::test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated": [
|
||||
"providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment"
|
||||
],
|
||||
"tests/integration/providers/test_azure_ai_chat_wire.py::test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages": [
|
||||
"providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry"
|
||||
],
|
||||
"tests/integration/providers/test_azure_ai_flux2_image_wire.py::test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro": [
|
||||
"other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body"
|
||||
],
|
||||
"tests/integration/providers/test_azure_ai_rerank_auth_wire.py::test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider": [
|
||||
"other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_auth_wire.py::test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature": [
|
||||
"providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_batch_files_wire.py::test_completions_and_responses_batch_records_upload_as_anthropic_user_messages": [
|
||||
"other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_claude_thinking_wire.py::test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens": [
|
||||
"other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_converse_config_blocks_wire.py::test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config": [
|
||||
"other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_embedding_wire.py::test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions": [
|
||||
"other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_gpt5_reasoning_wire.py::test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort": [
|
||||
"providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_invoke_tool_search_wire.py::test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field": [
|
||||
"providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_mantle_codex_input_wire.py::test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items": [
|
||||
"other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_mantle_responses_wire.py::test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle": [
|
||||
"providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_mantle_wire.py::test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long": [
|
||||
"other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_messages_web_search_replay_wire.py::test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers": [
|
||||
"providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_passthrough_stream_wire.py::test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type": [
|
||||
"other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_rerank_wire.py::test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature": [
|
||||
"providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned"
|
||||
],
|
||||
"tests/integration/providers/test_bedrock_thinking_tokens_wire.py::test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens": [
|
||||
"other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text"
|
||||
],
|
||||
"tests/integration/providers/test_dashscope_chat_wire.py::test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider": [
|
||||
"other.provider_wire.dashscope.reasoning_effort_reaches_provider"
|
||||
],
|
||||
"tests/integration/providers/test_databricks_chat_wire.py::test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log": [
|
||||
"other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log"
|
||||
],
|
||||
"tests/integration/providers/test_databricks_oauth_wire.py::test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin": [
|
||||
"other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base"
|
||||
],
|
||||
"tests/integration/providers/test_deepseek_vision_wire.py::test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text": [
|
||||
"other.provider_wire.deepseek.vision_image_content_list_reaches_provider"
|
||||
],
|
||||
"tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_chat_sends_router_resource_not_models_path": [
|
||||
"other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name"
|
||||
],
|
||||
"tests/integration/providers/test_fireworks_ai_router_slug_wire.py::test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path": [
|
||||
"other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name"
|
||||
],
|
||||
"tests/integration/providers/test_openai_chat_wire.py::test_openai_chat_tool_choice_without_tools_is_not_forwarded": [
|
||||
"providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire"
|
||||
],
|
||||
"tests/integration/providers/test_openai_image_edit_wire.py::test_openai_compatible_image_edit_forwards_seed_form_field_to_backend": [
|
||||
"other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields"
|
||||
],
|
||||
"tests/integration/providers/test_responses_bridge_incomplete.py::test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out": [
|
||||
"other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length"
|
||||
],
|
||||
"tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[reasoning_effort_none]": [
|
||||
"other.provider_wire.tencent.thinking_reaches_provider_in_request_body"
|
||||
],
|
||||
"tests/integration/providers/test_tencent_chat_wire.py::test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request[thinking_enabled]": [
|
||||
"other.provider_wire.tencent.thinking_reaches_provider_in_request_body"
|
||||
],
|
||||
"tests/integration/providers/test_websearch_interception_wire.py::test_capped_websearch_interception_loop_ends_turn_instead_of_exposing_internal_tool_use": [
|
||||
"other.provider_wire.anthropic.websearch_interception_capped_loop_ends_turn_without_internal_tool_use"
|
||||
],
|
||||
"tests/integration/providers/test_websearch_interception_wire.py::test_streamed_web_search_turn_capped_by_max_agentic_loops_ends_turn_with_snippets_and_ordered_blocks": [
|
||||
"other.provider_wire.bedrock.websearch_interception_streamed_capped_turn_ends_with_native_results"
|
||||
],
|
||||
"tests/integration/providers/test_xai_web_search_wire.py::test_xai_chat_web_search_is_sent_to_responses_with_instructions_and_nested_filters": [
|
||||
"other.provider_wire.xai.chat_web_search_reaches_responses_with_instructions_and_filters"
|
||||
],
|
||||
"tests/integration/routing/test_priority_rate_limit_headers.py::test_non_streaming_v1_messages_success_carries_v3_priority_rate_limit_headers": [
|
||||
"other.routing.priority_rate_limits.v1_messages_success_exposes_v3_priority_headers"
|
||||
],
|
||||
"tests/integration/routing/test_stale_cost_map_boot.py::test_config_deployment_dropped_by_stale_boot_cost_map_is_restored_after_reload": [
|
||||
"other.routing.cost_map.config_deployment_dropped_by_stale_boot_map_is_restored_after_reload"
|
||||
],
|
||||
"tests/integration/streaming/test_stream_contracts.py::test_messages_stream_completes_through_trailing_empty_choices_usage_chunk": [
|
||||
"other.streaming.messages_bridge.empty_choices_usage_chunk_completes_stream"
|
||||
],
|
||||
"tests/integration/streaming/test_stream_contracts.py::test_perplexity_stream_with_cost_breakdown_object_completes_and_bills_total_cost": [
|
||||
"other.streaming.usage.provider_cost_object_completes_stream_and_bills_total_cost"
|
||||
],
|
||||
"tests/integration/streaming/test_stream_contracts.py::test_primary_stream_with_empty_first_chunk_then_disconnect_falls_back_and_bills_the_fallback": [
|
||||
"other.streaming.fallback.empty_leading_chunk_then_disconnect_streams_fallback_with_usage_and_spend"
|
||||
],
|
||||
"tests/integration/streaming/test_stream_contracts.py::test_responses_stream_completes_through_empty_choices_metadata_and_usage_chunks": [
|
||||
"other.streaming.responses_bridge.empty_choices_chunks_complete_stream"
|
||||
],
|
||||
"tests/integration/streaming/test_stream_parallel_slot_release.py::test_failing_stream_logging_callback_does_not_leak_max_parallel_requests_slot": [
|
||||
"streaming.max_parallel_requests.slot_released_when_stream_logging_callback_fails"
|
||||
],
|
||||
"tests/integration/streaming/test_ttft_keepalive.py::test_stream_emits_sse_ping_comments_before_the_first_data_frame_while_upstream_is_silent": [
|
||||
"streaming.keepalive.sse_pings_fill_silent_time_to_first_token"
|
||||
],
|
||||
"tests/integration/observability/test_callback_delivery.py::test_response_survives_raising_success_deployment_hook[chat]": [
|
||||
"other.observability.callbacks.raising_success_deployment_hook_keeps_response"
|
||||
],
|
||||
|
|
|
|||
113
tests/integration/providers/test_anthropic_advisor_wire.py
Normal file
113
tests/integration/providers/test_anthropic_advisor_wire.py
Normal file
|
|
@ -0,0 +1,113 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_ADVISOR_KEY: Final = "synthetic-advisor-key"
|
||||
_QUESTION: Final = "which index should this query use"
|
||||
_ADVICE: Final = "use the composite index on (tenant_id, created_at)"
|
||||
_FINAL_ANSWER: Final = "done, the composite index is the right one"
|
||||
|
||||
|
||||
_ADVISOR_CALL_MESSAGE: Final = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "advisor-call",
|
||||
"type": "function",
|
||||
"function": {"name": "advisor", "arguments": json.dumps({"question": _QUESTION})},
|
||||
}
|
||||
],
|
||||
}
|
||||
_FINAL_MESSAGE: Final = {"role": "assistant", "content": _FINAL_ANSWER}
|
||||
|
||||
|
||||
def _chat_completion(identity: str, message: dict[str, object], finish_reason: str) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"chatcmpl-{identity}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "llama-3.3-70b-versatile",
|
||||
"choices": [{"index": 0, "message": message, "finish_reason": finish_reason}],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 4, "total_tokens": 14},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _executor_reply(body: dict[str, object], identity: str) -> Reply:
|
||||
messages: Final = body["messages"]
|
||||
assert isinstance(messages, list)
|
||||
if any(message.get("role") == "tool" for message in messages):
|
||||
assert messages[-1]["content"] == _ADVICE
|
||||
return _chat_completion(identity, _FINAL_MESSAGE, "stop")
|
||||
tools: Final = body["tools"]
|
||||
assert isinstance(tools, list)
|
||||
assert tools[0]["function"]["name"] == "advisor"
|
||||
return _chat_completion(identity, _ADVISOR_CALL_MESSAGE, "tool_calls")
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.anthropic_messages_advisor.sub_call_uses_the_configured_advisor_deployment")
|
||||
def test_advisor_sub_call_reaches_the_router_deployment_with_its_key_instead_of_anthropic_unauthenticated(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
identity: Final = "advisor-wire-" + uuid.uuid4().hex
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
body: Final = json.loads(request.body)
|
||||
if request.target == "/v1/chat/completions":
|
||||
assert request.headers["authorization"] == "Bearer integration-provider-key"
|
||||
return _executor_reply(body, identity)
|
||||
assert request.target == "/v1/messages"
|
||||
assert request.headers["x-api-key"] == _ADVISOR_KEY
|
||||
assert body["model"] == "claude-opus-4-1-20250805"
|
||||
assert body["messages"] == [
|
||||
{"role": "user", "content": "please plan the migration"},
|
||||
{"role": "user", "content": _QUESTION},
|
||||
]
|
||||
assert "tools" not in body
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"msg-{identity}",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-4-1-20250805",
|
||||
"content": [{"type": "text", "text": _ADVICE}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 6},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
executor: Final = scenario.model(model="hosted_vllm/llama-3.3-70b", api_base=wire.url + "/v1")
|
||||
advisor: Final = scenario.model(
|
||||
model="anthropic/claude-opus-4-1-20250805", api_base=wire.url, api_key=_ADVISOR_KEY
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": executor,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "please plan the migration"}],
|
||||
"tools": [{"type": "advisor_20260301", "name": "advisor", "model": advisor}],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
assert body["content"] == [{"type": "text", "text": _FINAL_ANSWER}], response.text
|
||||
assert body["stop_reason"] == "end_turn", response.text
|
||||
assert [request.target for request in wire.drain()] == [
|
||||
"/v1/chat/completions",
|
||||
"/v1/messages",
|
||||
"/v1/chat/completions",
|
||||
]
|
||||
85
tests/integration/providers/test_azure_ai_chat_wire.py
Normal file
85
tests/integration/providers/test_azure_ai_chat_wire.py
Normal file
|
|
@ -0,0 +1,85 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "kimi-k2-thinking"
|
||||
_API_KEY: Final = "synthetic-azure-ai-key"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_THINKING_BLOCK: Final[JsonValue] = {
|
||||
"type": "thinking",
|
||||
"thinking": "The user wants the sum of 17 and 26.",
|
||||
"signature": "synthetic-signature",
|
||||
}
|
||||
_HISTORY_WITH_ANTHROPIC_FIELDS: Final[JsonValue] = [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a calculator.",
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{"role": "user", "content": "What is 17 + 26?"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "43",
|
||||
"thinking_blocks": [_THINKING_BLOCK],
|
||||
"provider_specific_fields": {"citations": None},
|
||||
},
|
||||
{"role": "user", "content": "And doubled?"},
|
||||
]
|
||||
_HISTORY_AS_OPENAI_SPEC: Final[JsonValue] = [
|
||||
{"role": "system", "content": "You are a calculator."},
|
||||
{"role": "user", "content": "What is 17 + 26?"},
|
||||
{"role": "assistant", "content": "43"},
|
||||
{"role": "user", "content": "And doubled?"},
|
||||
]
|
||||
|
||||
|
||||
def _completion(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "86"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 31, "completion_tokens": 2, "total_tokens": 33},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.azure_ai.anthropic_message_fields_are_stripped_before_foundry")
|
||||
def test_azure_ai_strips_thinking_blocks_and_cache_control_from_forwarded_messages(gateway: Gateway) -> None:
|
||||
identity: Final = f"azure-ai-strip-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/chat/completions"
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == _BACKEND
|
||||
assert body["messages"] == _HISTORY_AS_OPENAI_SPEC
|
||||
return Reply(body=_completion(identity))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"azure_ai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": _HISTORY_WITH_ANTHROPIC_FIELDS},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["id"] == identity
|
||||
assert payload["choices"] == [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "86"},
|
||||
"provider_specific_fields": {},
|
||||
}
|
||||
]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
|
|
@ -0,0 +1,48 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_FLEX_MODEL: Final = "azure_ai/FLUX.2-flex"
|
||||
_PROMPT: Final = "a red fox in the snow"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.azure_ai.flux2_flex_generation_targets_flex_path_with_bfl_body")
|
||||
def test_azure_flux2_flex_generation_hits_flex_provider_path_not_pro(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview"
|
||||
assert request.headers["api-key"] == "synthetic-azure-key"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"model": "FLUX.2-flex",
|
||||
"prompt": _PROMPT,
|
||||
"num_images": 2,
|
||||
"width": 1536,
|
||||
"height": 1024,
|
||||
"guidance": 4.5,
|
||||
"steps": 32,
|
||||
}
|
||||
return Reply(body=json.dumps({"data": [{"b64_json": "aW1n"}, {"b64_json": "aW1n"}]}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=_FLEX_MODEL, api_base=wire.url, api_key="synthetic-azure-key", api_version="preview"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/images/generations",
|
||||
{"model": model, "prompt": _PROMPT, "n": 2, "size": "1536x1024", "guidance": 4.5, "steps": 32},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["data"] == [
|
||||
{"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None},
|
||||
{"url": None, "b64_json": "aW1n", "revised_prompt": None, "provider_specific_fields": None},
|
||||
]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/providers/blackforestlabs/v1/flux-2-flex?api-version=preview")
|
||||
]
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "azure_ai/Cohere-rerank-v4.0-fast"
|
||||
ENTRA_TOKEN: Final = "synthetic-entra-access-token"
|
||||
QUERY: Final = "which document mentions the gateway"
|
||||
DOCUMENTS: Final = ("the gateway proxies rerank calls", "unrelated synthetic text")
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "synthetic-rerank-id",
|
||||
"results": [{"index": 0, "relevance_score": 0.91}, {"index": 1, "relevance_score": 0.03}],
|
||||
"meta": {"api_version": {"version": "2"}, "billed_units": {"search_units": 1}},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def entra_rerank_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/providers/cohere/v2/rerank"
|
||||
assert request.headers["authorization"] == f"Bearer {ENTRA_TOKEN}"
|
||||
assert "api-key" not in request.headers
|
||||
body: Final = json.loads(request.body)
|
||||
assert body == {"model": "Cohere-rerank-v4.0-fast", "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2}
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.azure_ai.rerank_entra_token_without_api_key_reaches_provider")
|
||||
def test_azure_ai_rerank_with_entra_token_and_no_api_key_sends_bearer_to_provider(gateway: Gateway) -> None:
|
||||
with wire_server(entra_rerank_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=None,
|
||||
api_base=f"{wire.url}/providers/cohere/v2",
|
||||
azure_ad_token=ENTRA_TOKEN,
|
||||
model_info={"mode": "rerank"},
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/rerank", {"model": model, "query": QUERY, "documents": list(DOCUMENTS), "top_n": 2}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
assert [(result["index"], result["relevance_score"]) for result in body["results"]] == [(0, 0.91), (1, 0.03)]
|
||||
assert len(wire.drain()) == 1, "Expected exactly one provider rerank call"
|
||||
|
|
@ -7,18 +7,22 @@ from typing import Final
|
|||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/converse/anthropic.claude-3-haiku-20240307-v1:0"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
RESPONSE: Final = json.dumps({
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}},
|
||||
"stopReason": "end_turn", "usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}).encode()
|
||||
ACCESS_KEY: Final = "AKIAINTEGRATION000002"
|
||||
CLIENT_OAUTH_TOKEN: Final = "Bearer sk-ant-oat01-synthetic-client-subscription-token"
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "bedrock wire control"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def bearer_peer(request: Request) -> Reply:
|
||||
|
|
@ -34,31 +38,58 @@ def bearer_peer(request: Request) -> Reply:
|
|||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.bearer_sdk_skips_credential_chain")
|
||||
async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
|
||||
async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credentials(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
empty: Final = tmp_path / "empty-aws-config"
|
||||
empty.write_text("")
|
||||
for name in tuple(name for name in os.environ if name.startswith("AWS_")):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
for name, value in {"AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"}.items():
|
||||
for name, value in {
|
||||
"AWS_CONFIG_FILE": str(empty),
|
||||
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
"LITELLM_RUST": "false",
|
||||
}.items():
|
||||
monkeypatch.setenv(name, value)
|
||||
with wire_server(bearer_peer) as wire:
|
||||
with pytest.raises(litellm.APIConnectionError, match=r"config profile .* could not be found"):
|
||||
await asyncio.to_thread(litellm.completion, model=MODEL, aws_profile_name="integration-profile-must-not-be-read", aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url, messages=[{"role": "user", "content": "synthetic credential control"}], timeout=5, num_retries=0)
|
||||
await asyncio.to_thread(
|
||||
litellm.completion,
|
||||
model=MODEL,
|
||||
aws_profile_name="integration-profile-must-not-be-read",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
messages=[{"role": "user", "content": "synthetic credential control"}],
|
||||
timeout=5,
|
||||
num_retries=0,
|
||||
)
|
||||
assert wire.drain() == ()
|
||||
for source in ("argument", "environment"):
|
||||
if source == "environment":
|
||||
monkeypatch.setenv("AWS_BEARER_TOKEN_BEDROCK", TOKEN)
|
||||
parameters: Final = {
|
||||
"model": MODEL, "api_key": TOKEN if source == "argument" else None,
|
||||
"aws_region_name": "us-east-1", "aws_profile_name": "integration-profile-must-not-be-read",
|
||||
"aws_bedrock_runtime_endpoint": wire.url, "timeout": 5, "num_retries": 0,
|
||||
"messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}],
|
||||
"model": MODEL,
|
||||
"api_key": TOKEN if source == "argument" else None,
|
||||
"aws_region_name": "us-east-1",
|
||||
"aws_profile_name": "integration-profile-must-not-be-read",
|
||||
"aws_bedrock_runtime_endpoint": wire.url,
|
||||
"timeout": 5,
|
||||
"num_retries": 0,
|
||||
"messages": [
|
||||
{"role": "system", "content": "synthetic system"},
|
||||
{"role": "user", "content": "synthetic bearer request"},
|
||||
],
|
||||
"max_tokens": 16,
|
||||
}
|
||||
for asynchronous in (False, True):
|
||||
result: Final = await litellm.acompletion(**parameters) if asynchronous else await asyncio.to_thread(litellm.completion, **parameters)
|
||||
result: Final = (
|
||||
await litellm.acompletion(**parameters)
|
||||
if asynchronous
|
||||
else await asyncio.to_thread(litellm.completion, **parameters)
|
||||
)
|
||||
assert result.choices[0].message.content == "bedrock wire control"
|
||||
assert result.choices[0].finish_reason == "stop"
|
||||
assert result.usage.prompt_tokens == 11 and result.usage.completion_tokens == 4
|
||||
|
|
@ -66,28 +97,57 @@ async def test_bearer_only_sdk_sync_async_requests_do_not_require_aws_credential
|
|||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.bearer_db_yaml_survives_reload")
|
||||
def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload(gateway: Gateway, tmp_path: Path) -> None:
|
||||
def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
empty: Final = tmp_path / "empty-aws-config"
|
||||
empty.write_text("")
|
||||
with wire_server(bearer_peer) as wire:
|
||||
parameters: Final = {
|
||||
"model": MODEL, "api_key": "os.environ/INTEGRATION_BEARER_TOKEN", "aws_region_name": "us-east-1",
|
||||
"aws_profile_name": "integration-profile-must-not-be-read", "aws_bedrock_runtime_endpoint": wire.url,
|
||||
"model": MODEL,
|
||||
"api_key": "os.environ/INTEGRATION_BEARER_TOKEN",
|
||||
"aws_region_name": "us-east-1",
|
||||
"aws_profile_name": "integration-profile-must-not-be-read",
|
||||
"aws_bedrock_runtime_endpoint": wire.url,
|
||||
}
|
||||
alias: Final = f"integration-yaml-{uuid.uuid4().hex}"
|
||||
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
configuration["model_list"] = [{"model_name": alias, "litellm_params": parameters, "model_info": {"id": alias}}]
|
||||
path: Final = tmp_path / "bedrock.yaml"
|
||||
path.write_text(yaml.safe_dump(configuration))
|
||||
overrides: Final = {"INTEGRATION_BEARER_TOKEN": TOKEN, "AWS_CONFIG_FILE": str(empty), "AWS_SHARED_CREDENTIALS_FILE": str(empty), "AWS_EC2_METADATA_DISABLED": "true", "LITELLM_RUST": "false"}
|
||||
with owned_proxy(gateway, tmp_path, overrides, config=path, remove_environment=tuple(name for name in os.environ if name.startswith("AWS_"))) as candidate, candidate.scenario() as scenario:
|
||||
overrides: Final = {
|
||||
"INTEGRATION_BEARER_TOKEN": TOKEN,
|
||||
"AWS_CONFIG_FILE": str(empty),
|
||||
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
"LITELLM_RUST": "false",
|
||||
}
|
||||
with (
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
overrides,
|
||||
config=path,
|
||||
remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")),
|
||||
) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
database_model: Final = scenario.model(**parameters)
|
||||
for generation in range(2):
|
||||
for model in (alias, database_model):
|
||||
response: Final = candidate.request("POST", "/v1/chat/completions", {
|
||||
"model": model, "messages": [{"role": "system", "content": "synthetic system"}, {"role": "user", "content": "synthetic bearer request"}],
|
||||
"max_tokens": 16, "cache": {"no-cache": True},
|
||||
})
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [
|
||||
{"role": "system", "content": "synthetic system"},
|
||||
{"role": "user", "content": "synthetic bearer request"},
|
||||
],
|
||||
"max_tokens": 16,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control"
|
||||
assert response.json()["usage"]["total_tokens"] == 15
|
||||
|
|
@ -95,5 +155,61 @@ def test_bearer_environment_reference_loads_from_db_and_yaml_and_survives_reload
|
|||
if generation == 0:
|
||||
entries: Final = candidate.get("/model/info")["data"]
|
||||
target: Final = next(entry for entry in entries if entry["model_name"] == database_model)
|
||||
response: Final = candidate.request("PATCH", f"/model/{target['model_info']['id']}/update", {"model_info": {"description": "bearer reload"}})
|
||||
response: Final = candidate.request(
|
||||
"PATCH",
|
||||
f"/model/{target['model_info']['id']}/update",
|
||||
{"model_info": {"description": "bearer reload"}},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
|
||||
INVOKE_MODEL: Final = "bedrock/invoke/anthropic.claude-3-haiku-20240307-v1:0"
|
||||
INVOKE_RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "msg_synthetic",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"content": [{"type": "text", "text": "bedrock invoke wire control"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 11, "output_tokens": 4},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def sigv4_invoke_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1:0/invoke"
|
||||
assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/"), dict(
|
||||
request.headers
|
||||
)
|
||||
assert CLIENT_OAUTH_TOKEN not in request.headers.values(), dict(request.headers)
|
||||
assert json.loads(request.body)["messages"] == [{"role": "user", "content": "synthetic oauth isolation request"}]
|
||||
return Reply(body=INVOKE_RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_auth.client_anthropic_oauth_token_never_replaces_sigv4_authorization")
|
||||
def test_client_anthropic_oauth_authorization_header_does_not_replace_bedrock_sigv4_signature(gateway: Gateway) -> None:
|
||||
with wire_server(sigv4_invoke_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=INVOKE_MODEL,
|
||||
api_key=None,
|
||||
aws_access_key_id=ACCESS_KEY,
|
||||
aws_secret_access_key="synthetic-secret-key-for-testing",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
api_base=wire.url,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "synthetic oauth isolation request"}],
|
||||
"max_tokens": 16,
|
||||
},
|
||||
headers={"Authorization": CLIENT_OAUTH_TOKEN, "x-litellm-api-key": f"Bearer {gateway.key}"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["content"] == [{"type": "text", "text": "bedrock invoke wire control"}], response.text
|
||||
assert len(wire.drain()) == 1, response.text
|
||||
|
|
|
|||
78
tests/integration/providers/test_bedrock_batch_files_wire.py
Normal file
78
tests/integration/providers/test_bedrock_batch_files_wire.py
Normal file
|
|
@ -0,0 +1,78 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/anthropic.claude-3-haiku-20240307-v1:0"
|
||||
BUCKET: Final = "integration-batch-bucket"
|
||||
PROMPT: Final = "synthetic completions prompt"
|
||||
RESPONSES_INPUT: Final = "synthetic responses input"
|
||||
INPUT_LINES: Final = (
|
||||
{
|
||||
"custom_id": "completions-record",
|
||||
"method": "POST",
|
||||
"url": "/v1/completions",
|
||||
"body": {"model": MODEL, "prompt": PROMPT, "max_tokens": 64},
|
||||
},
|
||||
{
|
||||
"custom_id": "responses-record",
|
||||
"method": "POST",
|
||||
"url": "/v1/responses",
|
||||
"body": {"model": MODEL, "input": RESPONSES_INPUT, "max_output_tokens": 16},
|
||||
},
|
||||
)
|
||||
EXPECTED_S3_OBJECT: Final = (
|
||||
{
|
||||
"recordId": "completions-record",
|
||||
"modelInput": {
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": PROMPT}]}],
|
||||
"max_tokens": 64,
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
},
|
||||
},
|
||||
{
|
||||
"recordId": "responses-record",
|
||||
"modelInput": {
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": RESPONSES_INPUT}]}],
|
||||
"max_tokens": 16,
|
||||
"anthropic_version": "bedrock-2023-05-31",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def s3_peer(request: Request) -> Reply:
|
||||
assert request.method == "PUT" and request.target.startswith(f"/{BUCKET}/"), request.target
|
||||
assert request.headers["authorization"].startswith("AWS4-HMAC-SHA256 ")
|
||||
return Reply(body=b"")
|
||||
|
||||
|
||||
@pytest.mark.covers(
|
||||
"other.provider_wire.bedrock.batch_file_completions_and_responses_records_reach_s3_as_user_messages"
|
||||
)
|
||||
def test_completions_and_responses_batch_records_upload_as_anthropic_user_messages(gateway: Gateway) -> None:
|
||||
with wire_server(s3_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=None,
|
||||
api_base=None,
|
||||
aws_access_key_id="AKIAIOSFODNN7EXAMPLE",
|
||||
aws_secret_access_key="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY",
|
||||
aws_region_name="us-east-1",
|
||||
s3_bucket_name=BUCKET,
|
||||
s3_endpoint_url=wire.url,
|
||||
)
|
||||
jsonl: Final = "\n".join(json.dumps(line, separators=(",", ":")) for line in INPUT_LINES) + "\n"
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/files",
|
||||
{"purpose": "batch", "model": model},
|
||||
{"file": ("in.jsonl", jsonl.encode(), "application/jsonl")},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["object"] == "file" and response.json()["purpose"] == "batch", response.text
|
||||
uploads: Final = wire.drain()
|
||||
assert len(uploads) == 1, f"Expected exactly one S3 PUT, saw {[upload.target for upload in uploads]}"
|
||||
stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip())
|
||||
assert stored == EXPECTED_S3_OBJECT
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/invoke/us.anthropic.claude-opus-4-8"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "msg_adaptive_control",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "us.anthropic.claude-opus-4-8",
|
||||
"content": [{"type": "text", "text": "adaptive thinking control"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 5},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def adaptive_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/us.anthropic.claude-opus-4-8/invoke"
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "synthetic effort request"}]}]
|
||||
assert body["thinking"]["type"] == "adaptive", body
|
||||
assert body["output_config"] == {"effort": "high"}, body
|
||||
assert "budget_tokens" not in json.dumps(body), body
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.prefixed_opus_4_8_reasoning_effort_sends_adaptive_thinking")
|
||||
def test_prefixed_opus_4_8_reasoning_effort_reaches_bedrock_as_adaptive_thinking_not_budget_tokens(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
with wire_server(adaptive_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=TOKEN,
|
||||
aws_region_name="us-east-1",
|
||||
api_base=wire.url,
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "synthetic effort request"}],
|
||||
"max_tokens": 4096,
|
||||
"reasoning_effort": "high",
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["choices"][0]["message"]["content"] == "adaptive thinking control"
|
||||
assert response.json()["usage"]["prompt_tokens"] == 12 and response.json()["usage"]["completion_tokens"] == 5
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from integration.providers.test_bedrock_auth_wire import MODEL, RESPONSE, TOKEN
|
||||
|
||||
GUARDRAIL: Final = {"guardrailIdentifier": "integration-guardrail", "guardrailVersion": "DRAFT", "trace": "enabled"}
|
||||
PERFORMANCE: Final = {"latency": "optimized"}
|
||||
|
||||
|
||||
def converse_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/anthropic.claude-3-haiku-20240307-v1%3A0/converse"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["inferenceConfig"] == {"maxTokens": 16, "temperature": 0.2}, body
|
||||
assert body["guardrailConfig"] == GUARDRAIL, body
|
||||
assert body["performanceConfig"] == PERFORMANCE, body
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.converse_config_blocks_sent_once_at_top_level")
|
||||
def test_guardrail_and_performance_config_are_not_duplicated_inside_inference_config(gateway: Gateway) -> None:
|
||||
with wire_server(converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=TOKEN,
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
guardrailConfig=GUARDRAIL,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "synthetic guardrail request"}],
|
||||
"max_tokens": 16,
|
||||
"temperature": 0.2,
|
||||
"performanceConfig": PERFORMANCE,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["choices"][0]["message"]["content"] == "bedrock wire control"
|
||||
assert len(wire.drain()) == 1, response.text
|
||||
58
tests/integration/providers/test_bedrock_embedding_wire.py
Normal file
58
tests/integration/providers/test_bedrock_embedding_wire.py
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/cohere.embed-english-v3"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
INPUT: Final = "hello world"
|
||||
VECTOR: Final = [0.1, 0.2, 0.3]
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"embeddings": {"float": [VECTOR]},
|
||||
"id": "synthetic-cohere-embed",
|
||||
"response_type": "embeddings_by_type",
|
||||
"texts": [INPUT],
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def cohere_english_v3_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/cohere.embed-english-v3/invoke"
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
assert json.loads(request.body) == {
|
||||
"texts": [INPUT],
|
||||
"input_type": "search_document",
|
||||
"embedding_types": ["float"],
|
||||
"output_dimension": 512,
|
||||
}
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.cohere_embed_english_v3_accepts_encoding_format")
|
||||
def test_cohere_embed_english_v3_accepts_encoding_format_and_dimensions(gateway: Gateway) -> None:
|
||||
with wire_server(cohere_english_v3_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=TOKEN,
|
||||
api_base=wire.url,
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
for encoding_format in ("float", "base64"):
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/embeddings",
|
||||
{
|
||||
"model": model,
|
||||
"input": INPUT,
|
||||
"encoding_format": encoding_format,
|
||||
"dimensions": 512,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, f"encoding_format={encoding_format}: {response.text}"
|
||||
assert response.json()["data"] == [
|
||||
{"object": "embedding", "index": 0, "embedding": VECTOR, "type": "float"},
|
||||
], response.text
|
||||
assert len(wire.drain()) == 1, f"encoding_format={encoding_format} never reached Bedrock"
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/converse/us.openai.gpt-5.6-sol"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": "gpt-5 reasoning wire control"}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 9, "outputTokens": 5, "totalTokens": 14},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def gpt5_converse_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/us.openai.gpt-5.6-sol/converse"
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["messages"] == [{"role": "user", "content": [{"text": "synthetic reasoning request"}]}]
|
||||
assert body["additionalModelRequestFields"] == {"reasoning": {"effort": "high"}}, body
|
||||
assert body["inferenceConfig"] == {"maxTokens": 16}, body
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_converse.gpt5_reasoning_effort_reaches_provider_as_reasoning_effort")
|
||||
def test_gpt5_reasoning_effort_is_accepted_and_sent_as_converse_reasoning_effort(gateway: Gateway) -> None:
|
||||
with wire_server(gpt5_converse_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "synthetic reasoning request"}],
|
||||
"reasoning_effort": "high",
|
||||
"max_tokens": 16,
|
||||
"cache": {"no-cache": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["choices"][0]["message"]["content"] == "gpt-5 reasoning wire control", response.text
|
||||
assert response.json()["usage"]["total_tokens"] == 14, response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL_ID: Final = "us.anthropic.claude-sonnet-5"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
TOOL_SEARCH_TOOL: Final = {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}
|
||||
DEFERRED_TOOL: Final = {
|
||||
"name": "get_weather",
|
||||
"description": "Weather lookup",
|
||||
"input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]},
|
||||
"defer_loading": True,
|
||||
}
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "msg_tool_search_control",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": MODEL_ID,
|
||||
"content": [
|
||||
{
|
||||
"type": "server_tool_use",
|
||||
"id": "srvtoolu_control",
|
||||
"name": "tool_search_tool_regex",
|
||||
"input": {"pattern": "weather"},
|
||||
},
|
||||
{
|
||||
"type": "tool_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_control",
|
||||
"content": {
|
||||
"type": "tool_search_tool_search_result",
|
||||
"tool_references": [{"type": "tool_reference", "tool_name": "get_weather"}],
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "tool search wire control"},
|
||||
],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 6},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def tool_search_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == f"/model/{MODEL_ID}/invoke", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["anthropic_beta"] == ["tool-search-tool-2025-10-19"], body
|
||||
assert body["messages"] == [{"role": "user", "content": "find the weather tool"}]
|
||||
assert body["tools"] == [TOOL_SEARCH_TOOL, DEFERRED_TOOL], body["tools"]
|
||||
assert body["max_tokens"] == 64
|
||||
assert "model" not in body
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_invoke.tool_search_gen5_claude_sends_bedrock_beta_and_reports_support")
|
||||
def test_gen5_claude_bedrock_invoke_messages_tool_search_sends_bedrock_beta_field(gateway: Gateway) -> None:
|
||||
with wire_server(tool_search_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/invoke/{MODEL_ID}",
|
||||
api_key=TOKEN,
|
||||
aws_region_name="us-east-1",
|
||||
api_base=wire.url,
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [{"role": "user", "content": "find the weather tool"}],
|
||||
"tools": [TOOL_SEARCH_TOOL, DEFERRED_TOOL],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
assert body["content"][2] == {"type": "text", "text": "tool search wire control"}, response.text
|
||||
assert body["stop_reason"] == "end_turn"
|
||||
assert body["usage"]["input_tokens"] == 12 and body["usage"]["output_tokens"] == 6
|
||||
assert len(wire.drain()) == 1
|
||||
entries: Final = gateway.get("/v1/model/info")["data"]
|
||||
assert isinstance(entries, list)
|
||||
info: Final = next(entry for entry in entries if isinstance(entry, dict) and entry["model_name"] == model)
|
||||
assert isinstance(info["model_info"], dict)
|
||||
assert info["model_info"]["supports_tool_search"] is True, info["model_info"]
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol"
|
||||
TOKEN: Final = "synthetic-mantle-bearer"
|
||||
CIPHERTEXT: Final = "synthetic-compaction-ciphertext"
|
||||
CALL_ID: Final = "call_synthetic_shell"
|
||||
JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000}
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "resp_synthetic_mantle",
|
||||
"object": "response",
|
||||
"created_at": 1789788253,
|
||||
"status": "completed",
|
||||
"model": "openai.gpt-5.6-sol",
|
||||
"output": [
|
||||
{
|
||||
"type": "message",
|
||||
"id": "msg_synthetic_mantle",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}],
|
||||
}
|
||||
],
|
||||
"usage": {"input_tokens": 21, "output_tokens": 4, "total_tokens": 25},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def user_turn(text: str) -> JsonValue:
|
||||
return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}
|
||||
|
||||
|
||||
def codex_history(marker: str) -> tuple[JsonValue, ...]:
|
||||
return (
|
||||
user_turn(f"first turn {marker}"),
|
||||
{"type": "agent_message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]},
|
||||
{"type": "context_compaction", "encrypted_content": CIPHERTEXT},
|
||||
{"type": "local_shell_call", "call_id": CALL_ID, "status": "completed", "action": ACTION},
|
||||
{"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"},
|
||||
user_turn(f"next turn {marker}"),
|
||||
)
|
||||
|
||||
|
||||
def mantle_history(marker: str) -> tuple[JsonValue, ...]:
|
||||
return (
|
||||
user_turn(f"first turn {marker}"),
|
||||
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent reply"}]},
|
||||
{"type": "compaction", "encrypted_content": CIPHERTEXT},
|
||||
{"type": "function_call", "call_id": CALL_ID, "name": "local_shell", "arguments": json.dumps(ACTION)},
|
||||
{"type": "function_call_output", "call_id": CALL_ID, "output": "synthetic shell output"},
|
||||
user_turn(f"next turn {marker}"),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock_mantle.codex_history_items_reach_mantle_as_supported_types")
|
||||
def test_codex_agent_message_context_compaction_and_local_shell_call_reach_mantle_as_supported_items(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
expected_input: Final = list(mantle_history(marker))
|
||||
|
||||
def mantle_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/openai/v1/responses", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == "openai.gpt-5.6-sol", body
|
||||
assert body["input"] == expected_input, body["input"]
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
with wire_server(mantle_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=MODEL, api_key=TOKEN, api_base=wire.url, aws_region_name="us-east-2")
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": list(codex_history(marker)), "store": False}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["output"][0]["content"][0]["text"] == "mantle wire control", response.text
|
||||
assert response.json()["usage"]["total_tokens"] == 25, response.text
|
||||
forwarded: Final = wire.drain()
|
||||
assert len(forwarded) == 1, forwarded
|
||||
assert JSON_OBJECT.validate_json(forwarded[0].body)["input"] == expected_input, forwarded[0].body
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
import json
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_MODEL: Final = "bedrock_mantle/openai.gpt-5.6-sol"
|
||||
_TOKEN: Final = "synthetic-mantle-bearer"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_SHELL_ACTION: Final[dict[str, JsonValue]] = {"type": "exec", "command": ["ls", "-la"], "timeout_ms": 1000}
|
||||
_OUTPUT_MESSAGE: Final[dict[str, JsonValue]] = {
|
||||
"type": "message",
|
||||
"id": "msg_mantle",
|
||||
"status": "completed",
|
||||
"role": "assistant",
|
||||
"content": [{"type": "output_text", "text": "mantle wire control", "annotations": []}],
|
||||
}
|
||||
_RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"id": "resp_mantle",
|
||||
"object": "response",
|
||||
"status": "completed",
|
||||
"created_at": 1700000000,
|
||||
"model": "gpt-5.6-sol",
|
||||
"output": [_OUTPUT_MESSAGE],
|
||||
"usage": {
|
||||
"input_tokens": 11,
|
||||
"output_tokens": 4,
|
||||
"total_tokens": 15,
|
||||
"input_tokens_details": {"cached_tokens": 0},
|
||||
"output_tokens_details": {"reasoning_tokens": 0},
|
||||
},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _codex_history(marker: str) -> list[JsonValue]:
|
||||
return [
|
||||
{"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"},
|
||||
{
|
||||
"type": "agent_message",
|
||||
"id": "msg_agent",
|
||||
"content": [{"type": "text", "text": "sub-agent said "}, {"type": "text", "encrypted_content": "hello"}],
|
||||
},
|
||||
{"type": "context_compaction", "id": "cmp_1", "encrypted_content": "compacted-history"},
|
||||
{
|
||||
"type": "local_shell_call",
|
||||
"id": "lsc_1",
|
||||
"call_id": "call_shell",
|
||||
"status": "completed",
|
||||
"action": _SHELL_ACTION,
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "call_shell", "output": "total 0"},
|
||||
]
|
||||
|
||||
|
||||
def _mantle_history(marker: str) -> list[JsonValue]:
|
||||
return [
|
||||
{"type": "message", "role": "user", "content": f"delegate to a subagent {marker}"},
|
||||
{"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "sub-agent said hello"}]},
|
||||
{"type": "compaction", "encrypted_content": "compacted-history"},
|
||||
{
|
||||
"type": "function_call",
|
||||
"call_id": "call_shell",
|
||||
"name": "local_shell",
|
||||
"arguments": json.dumps(_SHELL_ACTION),
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "call_shell", "output": "total 0"},
|
||||
]
|
||||
|
||||
|
||||
def _mantle_peer(marker: str) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/openai/v1/responses", request.target
|
||||
assert request.headers["authorization"] == f"Bearer {_TOKEN}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["input"] == _mantle_history(marker), json.dumps(body["input"])
|
||||
return Reply(body=_RESPONSE)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_mantle.codex_history_items_reach_the_wire_as_supported_input_items")
|
||||
def test_codex_agent_message_compaction_and_local_shell_items_are_rewritten_for_mantle(gateway: Gateway) -> None:
|
||||
marker: Final = uuid4().hex
|
||||
with wire_server(_mantle_peer(marker)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_MODEL, api_base=wire.url, api_key=_TOKEN, aws_region_name="us-east-1")
|
||||
response: Final = gateway.request(
|
||||
"POST", "/v1/responses", {"model": model, "input": _codex_history(marker), "stream": False}
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["output"] == [
|
||||
{
|
||||
**_OUTPUT_MESSAGE,
|
||||
"phase": None,
|
||||
"content": [
|
||||
{"type": "output_text", "text": "mantle wire control", "annotations": [], "logprobs": None}
|
||||
],
|
||||
}
|
||||
], response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/openai/v1/responses")]
|
||||
53
tests/integration/providers/test_bedrock_mantle_wire.py
Normal file
53
tests/integration/providers/test_bedrock_mantle_wire.py
Normal file
|
|
@ -0,0 +1,53 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "openai.gpt-5.6-sol"
|
||||
_API_KEY: Final = "synthetic-mantle-bearer"
|
||||
_PROMPT: Final = "synthetic long conversation control"
|
||||
_PROMPT_TOKENS: Final = 1055489
|
||||
_MODEL_MAXIMUM: Final = 1050000
|
||||
_RESPONSES_PATH: Final = "/openai/v1/responses"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_OVERFLOW_BODY: Final = json.dumps(
|
||||
{
|
||||
"error": {
|
||||
"code": "validation_error",
|
||||
"message": f"prompt tokens ({_PROMPT_TOKENS}) exceed model maximum ({_MODEL_MAXIMUM}) for {_BACKEND}",
|
||||
"type": "invalid_request_error",
|
||||
}
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _overflow_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == _RESPONSES_PATH
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == _BACKEND
|
||||
assert _PROMPT in json.dumps(body["input"]), body
|
||||
return Reply(status=400, body=_OVERFLOW_BODY)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock_mantle.context_overflow_is_reported_as_prompt_too_long")
|
||||
def test_bedrock_mantle_context_overflow_returns_400_saying_prompt_is_too_long(gateway: Gateway) -> None:
|
||||
with wire_server(_overflow_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"bedrock_mantle/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
error: Final = _JSON_OBJECT.validate_json(response.content)["error"]
|
||||
assert isinstance(error, dict), response.text
|
||||
assert error["code"] == "400", response.text
|
||||
message: Final = error["message"]
|
||||
assert isinstance(message, str), response.text
|
||||
assert f"prompt is too long: {_PROMPT_TOKENS} tokens > {_MODEL_MAXIMUM} maximum" in message, response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", _RESPONSES_PATH)]
|
||||
|
|
@ -0,0 +1,88 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
BEDROCK_MODEL: Final = "us.anthropic.claude-opus-5-v1:0"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
SNIPPET: Final = "synthetic snippet about the integration harness"
|
||||
INTERCEPTED_TURN: Final = (
|
||||
{"type": "server_tool_use", "id": "srvtoolu_synthetic", "name": "web_search", "input": {"query": "harness docs"}},
|
||||
{
|
||||
"type": "web_search_tool_result",
|
||||
"tool_use_id": "srvtoolu_synthetic",
|
||||
"content": [
|
||||
{
|
||||
"type": "web_search_result",
|
||||
"url": "https://example.test/harness",
|
||||
"title": "Harness",
|
||||
"page_age": None,
|
||||
"encrypted_content": "",
|
||||
"snippet": SNIPPET,
|
||||
},
|
||||
],
|
||||
},
|
||||
{"type": "text", "text": "The harness is documented at example.test"},
|
||||
)
|
||||
FLATTENED_TURN: Final = (
|
||||
{
|
||||
"type": "text",
|
||||
"text": f"Web search results for 'harness docs':\n\nTitle: Harness\nURL: https://example.test/harness\nSnippet: {SNIPPET}",
|
||||
},
|
||||
{"type": "text", "text": "The harness is documented at example.test"},
|
||||
)
|
||||
REPLY: Final = json.dumps(
|
||||
{
|
||||
"id": "msg_synthetic_replay",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": BEDROCK_MODEL,
|
||||
"content": [{"type": "text", "text": "replay accepted"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 30, "output_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def bedrock_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == f"/model/{BEDROCK_MODEL}/invoke"
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["messages"] == [
|
||||
{"role": "user", "content": "where is the harness documented"},
|
||||
{"role": "assistant", "content": list(FLATTENED_TURN)},
|
||||
{"role": "user", "content": "and what does it say"},
|
||||
], request.body.decode()
|
||||
assert "tools" not in body, request.body.decode()
|
||||
return Reply(body=REPLY)
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_messages.replayed_intercepted_web_search_turn_is_flattened_to_text")
|
||||
def test_replayed_intercepted_web_search_turn_reaches_bedrock_as_text_and_answers(gateway: Gateway) -> None:
|
||||
with wire_server(bedrock_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"bedrock/{BEDROCK_MODEL}",
|
||||
api_key=TOKEN,
|
||||
api_base=wire.url,
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "user", "content": "where is the harness documented"},
|
||||
{"role": "assistant", "content": list(INTERCEPTED_TURN)},
|
||||
{"role": "user", "content": "and what does it say"},
|
||||
],
|
||||
},
|
||||
headers={"x-api-key": gateway.key, "anthropic-version": "2023-06-01"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["content"] == [{"type": "text", "text": "replay accepted"}], response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -0,0 +1,42 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.upstream import _aws_event_frame
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
_MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0"
|
||||
_EVENT_STREAM: Final = "application/vnd.amazon.eventstream"
|
||||
_REQUEST_BODY: Final = {"messages": [{"role": "user", "content": [{"text": "synthetic passthrough stream"}]}]}
|
||||
_EVENTS: Final = (
|
||||
("messageStart", {"role": "assistant"}),
|
||||
("contentBlockDelta", {"delta": {"text": "bedrock stream control"}, "contentBlockIndex": 0}),
|
||||
("messageStop", {"stopReason": "end_turn"}),
|
||||
("metadata", {"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}}),
|
||||
)
|
||||
_STREAM_BYTES: Final = b"".join(_aws_event_frame(kind, payload, "sc", "u") for kind, payload in _EVENTS)
|
||||
|
||||
|
||||
def event_stream_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == f"/model/{_MODEL_ID}/converse-stream"
|
||||
assert json.loads(request.body)["messages"] == _REQUEST_BODY["messages"]
|
||||
return Reply(body=_STREAM_BYTES, content_type=_EVENT_STREAM)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.passthrough_stream_keeps_event_stream_content_type")
|
||||
def test_bedrock_passthrough_converse_stream_response_carries_event_stream_content_type(gateway: Gateway) -> None:
|
||||
with wire_server(event_stream_peer) as wire, gateway.scenario() as scenario:
|
||||
deployment: Final = scenario.model(
|
||||
model=f"bedrock/{_MODEL_ID}",
|
||||
api_base=wire.url,
|
||||
aws_access_key_id="AKIASCRIPTEDPROVIDER",
|
||||
aws_secret_access_key="scripted-secret",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
response: Final = gateway.request("POST", f"/bedrock/model/{deployment}/converse-stream", _REQUEST_BODY)
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(wire.drain()) == 1, response.text
|
||||
assert response.headers.get("content-type") == _EVENT_STREAM, dict(response.headers)
|
||||
assert response.content == _STREAM_BYTES, response.text
|
||||
92
tests/integration/providers/test_bedrock_rerank_wire.py
Normal file
92
tests/integration/providers/test_bedrock_rerank_wire.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
MODEL: Final = "bedrock/arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
|
||||
ACCESS_KEY: Final = "AKIAINTEGRATION000002"
|
||||
FORWARDED_FOR: Final = "203.0.113.5"
|
||||
RESPONSE: Final = json.dumps(
|
||||
{"results": [{"index": 1, "relevanceScore": 0.9}, {"index": 0, "relevanceScore": 0.1}]}
|
||||
).encode()
|
||||
|
||||
|
||||
def signed_headers(authorization: str) -> tuple[str, ...]:
|
||||
return tuple(authorization.split("SignedHeaders=")[1].split(",")[0].split(";"))
|
||||
|
||||
|
||||
def rerank_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/rerank"
|
||||
assert request.headers["authorization"].startswith(f"AWS4-HMAC-SHA256 Credential={ACCESS_KEY}/")
|
||||
assert signed_headers(request.headers["authorization"]) == ("content-type", "host", "x-amz-date"), request.headers[
|
||||
"authorization"
|
||||
]
|
||||
assert request.headers["x-forwarded-for"] == FORWARDED_FOR
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["queries"] == [{"textQuery": {"text": "synthetic rerank query"}, "type": "TEXT"}]
|
||||
assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["modelConfiguration"] == {
|
||||
"modelArn": "arn:aws:bedrock:us-east-1::foundation-model/cohere.rerank-v3-5:0"
|
||||
}
|
||||
assert body["rerankingConfiguration"]["bedrockRerankingConfiguration"]["numberOfResults"] == 2
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.bedrock_rerank.forwarded_client_headers_are_sent_unsigned")
|
||||
def test_forwarded_client_header_on_rerank_is_excluded_from_the_sigv4_signature(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
empty: Final = tmp_path / "empty-aws-config"
|
||||
empty.write_text("")
|
||||
configuration: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
configuration["general_settings"]["forward_client_headers_to_llm_api"] = True
|
||||
path: Final = tmp_path / "forwarding.yaml"
|
||||
path.write_text(yaml.safe_dump(configuration))
|
||||
overrides: Final = {
|
||||
"AWS_CONFIG_FILE": str(empty),
|
||||
"AWS_SHARED_CREDENTIALS_FILE": str(empty),
|
||||
"AWS_EC2_METADATA_DISABLED": "true",
|
||||
"LITELLM_RUST": "false",
|
||||
}
|
||||
with wire_server(rerank_peer) as wire:
|
||||
with (
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
overrides,
|
||||
config=path,
|
||||
remove_environment=tuple(name for name in os.environ if name.startswith("AWS_")),
|
||||
) as candidate,
|
||||
candidate.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model=MODEL,
|
||||
api_key=None,
|
||||
api_base=None,
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
aws_access_key_id=ACCESS_KEY,
|
||||
aws_secret_access_key="synthetic-rerank-secret-key-for-testing",
|
||||
)
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/rerank",
|
||||
{
|
||||
"model": model,
|
||||
"query": "synthetic rerank query",
|
||||
"documents": ["first synthetic document", "second synthetic document"],
|
||||
"top_n": 2,
|
||||
},
|
||||
headers={"x-forwarded-for": FORWARDED_FOR},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert response.json()["results"] == [
|
||||
{"index": 1, "relevance_score": 0.9},
|
||||
{"index": 0, "relevance_score": 0.1},
|
||||
], response.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
MODEL: Final = "bedrock/converse/global.anthropic.claude-opus-4-8"
|
||||
TOKEN: Final = "synthetic-bedrock-bearer"
|
||||
PROMPT: Final = "How many prime numbers are less than 30? Think it through, then answer with just the number."
|
||||
RESPONSES_PROMPT: Final = "How many prime numbers are less than 30? Answer with just the number."
|
||||
REDACTED_DATA: Final = "RWRhY3RlZC1ieS1CZWRyb2Nr"
|
||||
INPUT_TOKENS: Final = 31
|
||||
OUTPUT_TOKENS: Final = 257
|
||||
RESPONSE: Final = json.dumps(
|
||||
{
|
||||
"output": {
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": [{"reasoningContent": {"redactedContent": REDACTED_DATA}}, {"text": "10"}],
|
||||
}
|
||||
},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {
|
||||
"inputTokens": INPUT_TOKENS,
|
||||
"outputTokens": OUTPUT_TOKENS,
|
||||
"totalTokens": INPUT_TOKENS + OUTPUT_TOKENS,
|
||||
},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_JSON_LIST: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
|
||||
def redacted_thinking_peer(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/model/global.anthropic.claude-opus-4-8/converse"
|
||||
assert request.headers["authorization"] == f"Bearer {TOKEN}"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["messages"] in (
|
||||
[{"role": "user", "content": [{"text": PROMPT}]}],
|
||||
[{"role": "user", "content": [{"text": RESPONSES_PROMPT}]}],
|
||||
), body
|
||||
assert body["additionalModelRequestFields"]["thinking"]["type"] == "adaptive", body
|
||||
return Reply(body=RESPONSE)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.bedrock.hidden_thinking_tokens_are_not_reported_as_text")
|
||||
def test_bedrock_redacted_thinking_is_not_reported_as_zero_reasoning_tokens(gateway: Gateway) -> None:
|
||||
with wire_server(redacted_thinking_peer) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=MODEL, api_key=TOKEN, aws_region_name="us-east-1", aws_bedrock_runtime_endpoint=wire.url
|
||||
)
|
||||
chat: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": PROMPT}],
|
||||
"max_tokens": 4000,
|
||||
"reasoning_effort": "max",
|
||||
},
|
||||
)
|
||||
assert chat.status_code == 200, chat.text
|
||||
chat_body: Final = _JSON_OBJECT.validate_json(chat.content)
|
||||
message: Final = _JSON_OBJECT.validate_python(_JSON_LIST.validate_python(chat_body["choices"])[0]["message"])
|
||||
assert message["content"] == "10", chat.text
|
||||
assert message["thinking_blocks"] == [{"type": "redacted_thinking", "data": REDACTED_DATA}], chat.text
|
||||
usage: Final = _JSON_OBJECT.validate_python(chat_body["usage"])
|
||||
assert usage["completion_tokens"] == OUTPUT_TOKENS, chat.text
|
||||
details: Final = _JSON_OBJECT.validate_python(usage["completion_tokens_details"])
|
||||
assert details == {}, chat.text
|
||||
assert len(wire.drain()) == 1
|
||||
|
||||
responses: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/responses",
|
||||
{"model": model, "input": RESPONSES_PROMPT, "max_output_tokens": 4000, "reasoning": {"effort": "max"}},
|
||||
)
|
||||
assert responses.status_code == 200, responses.text
|
||||
responses_body: Final = _JSON_OBJECT.validate_json(responses.content)
|
||||
output: Final = _JSON_LIST.validate_python(responses_body["output"])
|
||||
reasoning_items: Final = tuple(item for item in output if item["type"] == "reasoning")
|
||||
assert len(reasoning_items) == 1, responses.text
|
||||
assert reasoning_items[0]["encrypted_content"] == json.dumps(
|
||||
[{"type": "redacted_thinking", "data": REDACTED_DATA}], separators=(",", ":")
|
||||
), responses.text
|
||||
responses_usage: Final = _JSON_OBJECT.validate_python(responses_body["usage"])
|
||||
assert responses_usage["output_tokens"] == OUTPUT_TOKENS, responses.text
|
||||
assert _JSON_OBJECT.validate_python(responses_usage["output_tokens_details"])["reasoning_tokens"] == 0, (
|
||||
responses.text
|
||||
)
|
||||
assert len(wire.drain()) == 1
|
||||
62
tests/integration/providers/test_dashscope_chat_wire.py
Normal file
62
tests/integration/providers/test_dashscope_chat_wire.py
Normal file
|
|
@ -0,0 +1,62 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "qwen3.7-plus"
|
||||
_API_KEY: Final = "synthetic-dashscope-key"
|
||||
_PROMPT: Final = "What is 3^3?"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _completion(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "27"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 17, "completion_tokens": 5, "total_tokens": 22},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.dashscope.reasoning_effort_reaches_provider")
|
||||
def test_dashscope_chat_forwards_reasoning_effort_none_to_the_provider(gateway: Gateway) -> None:
|
||||
identity: Final = f"dashscope-reasoning-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/chat/completions"
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"model": _BACKEND,
|
||||
"messages": [{"role": "user", "content": _PROMPT}],
|
||||
"reasoning_effort": "none",
|
||||
}
|
||||
return Reply(body=_completion(identity))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"dashscope/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], "reasoning_effort": "none"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["id"] == identity
|
||||
assert payload["choices"] == [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "27", "provider_specific_fields": {"refusal": None}},
|
||||
"provider_specific_fields": {},
|
||||
}
|
||||
]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
129
tests/integration/providers/test_databricks_chat_wire.py
Normal file
129
tests/integration/providers/test_databricks_chat_wire.py
Normal file
|
|
@ -0,0 +1,129 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, eventually
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "databricks-glm-5-2"
|
||||
_API_KEY: Final = "synthetic-databricks-key"
|
||||
_PROMPT: Final = "Summarise the cached briefing in one sentence."
|
||||
_PROVIDER_USAGE: Final[Mapping[str, JsonValue]] = {
|
||||
"prompt_tokens": 12011,
|
||||
"completion_tokens": 8,
|
||||
"total_tokens": 12019,
|
||||
"cache_read_input_tokens": 12002,
|
||||
"cache_creation_input_tokens": 0,
|
||||
}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
class _PromptTokensDetails(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
cached_tokens: int | None = None
|
||||
|
||||
|
||||
class _Usage(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
prompt_tokens: int
|
||||
completion_tokens: int
|
||||
total_tokens: int
|
||||
prompt_tokens_details: _PromptTokensDetails | None = None
|
||||
|
||||
|
||||
class _Delta(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
content: str | None = None
|
||||
|
||||
|
||||
class _Choice(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
delta: _Delta
|
||||
|
||||
|
||||
class _Chunk(BaseModel):
|
||||
model_config = ConfigDict(extra="ignore")
|
||||
id: str
|
||||
choices: tuple[_Choice, ...]
|
||||
usage: _Usage | None = None
|
||||
|
||||
|
||||
def _frame(identity: str, choices: list[Mapping[str, object]], usage: Mapping[str, JsonValue] | None = None) -> bytes:
|
||||
value: Final = {
|
||||
"id": identity,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": choices,
|
||||
**({} if usage is None else {"usage": usage}),
|
||||
}
|
||||
return b"data: " + json.dumps(value).encode() + b"\n\n"
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.databricks.stream_usage_and_cache_reads_reach_client_and_spend_log")
|
||||
def test_databricks_stream_final_usage_chunk_reaches_client_and_spend_log(gateway: Gateway) -> None:
|
||||
identity: Final = f"databricks-stream-{uuid.uuid4().hex}"
|
||||
frames: Final = (
|
||||
_frame(
|
||||
identity, [{"index": 0, "delta": {"role": "assistant", "content": "The briefing "}, "finish_reason": None}]
|
||||
),
|
||||
_frame(identity, [{"index": 0, "delta": {"content": "is short."}, "finish_reason": None}]),
|
||||
_frame(identity, [{"index": 0, "delta": {}, "finish_reason": "stop"}]),
|
||||
_frame(identity, [], usage=_PROVIDER_USAGE),
|
||||
b"data: [DONE]\n\n",
|
||||
)
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/chat/completions"
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == _BACKEND
|
||||
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
|
||||
assert body["stream"] is True
|
||||
return Reply(content_type="text/event-stream", chunks=frames)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"databricks/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
with gateway.client.stream(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": _PROMPT}],
|
||||
"stream": True,
|
||||
"stream_options": {"include_usage": True},
|
||||
},
|
||||
headers={"Authorization": f"Bearer {gateway.key}"},
|
||||
) as response:
|
||||
assert response.status_code == 200, response.read()
|
||||
lines: Final = tuple(line for line in response.iter_lines() if line.startswith("data: "))
|
||||
assert lines[-1] == "data: [DONE]", lines
|
||||
chunks: Final = tuple(_Chunk.model_validate_json(line.removeprefix("data: ")) for line in lines[:-1])
|
||||
assert {chunk.id for chunk in chunks} == {identity}
|
||||
assert (
|
||||
"".join(choice.delta.content or "" for chunk in chunks for choice in chunk.choices)
|
||||
== "The briefing is short."
|
||||
)
|
||||
usages: Final = tuple(chunk.usage for chunk in chunks if chunk.usage is not None)
|
||||
assert len(usages) == 1, lines
|
||||
assert (
|
||||
usages[0].prompt_tokens,
|
||||
usages[0].completion_tokens,
|
||||
usages[0].total_tokens,
|
||||
usages[0].prompt_tokens_details.cached_tokens if usages[0].prompt_tokens_details is not None else None,
|
||||
) == (12011, 8, 12019, 12002), lines
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
rows: Final = eventually(
|
||||
lambda: read_rows(
|
||||
'SELECT prompt_tokens, completion_tokens, total_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s',
|
||||
(identity,),
|
||||
),
|
||||
lambda values: len(values) == 1,
|
||||
seconds=70,
|
||||
)
|
||||
assert (rows[0]["prompt_tokens"], rows[0]["completion_tokens"], rows[0]["total_tokens"]) == (12011, 8, 12019)
|
||||
92
tests/integration/providers/test_databricks_oauth_wire.py
Normal file
92
tests/integration/providers/test_databricks_oauth_wire.py
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
import base64
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_MODEL: Final = "databricks/synthetic-vendor.chat-model.v1"
|
||||
_CLIENT_ID: Final = "synthetic-databricks-client-id"
|
||||
_CLIENT_SECRET: Final = "synthetic-databricks-client-secret"
|
||||
_ACCESS_TOKEN: Final = "synthetic-databricks-oauth-token"
|
||||
_PROMPT: Final = "Which workspace issued this token?"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _basic_credentials(client_id: str, client_secret: str) -> str:
|
||||
return "Basic " + base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
|
||||
|
||||
|
||||
def _completion(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _MODEL.removeprefix("databricks/"),
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "the workspace origin"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.databricks.oauth_token_url_uses_workspace_origin_for_ai_gateway_api_base")
|
||||
def test_databricks_ai_gateway_api_base_requests_oauth_token_from_workspace_origin(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
identity: Final = f"databricks-oauth-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.target == "/oidc/v1/token":
|
||||
assert request.method == "POST"
|
||||
assert request.headers["authorization"] == _basic_credentials(_CLIENT_ID, _CLIENT_SECRET)
|
||||
assert request.headers["content-type"] == "application/x-www-form-urlencoded"
|
||||
assert parse_qs(request.body.decode()) == {"grant_type": ["client_credentials"], "scope": ["all-apis"]}
|
||||
return Reply(
|
||||
body=json.dumps({"access_token": _ACCESS_TOKEN, "token_type": "Bearer", "expires_in": 3600}).encode()
|
||||
)
|
||||
if request.target == "/ai-gateway/mlflow/v1/chat/completions":
|
||||
assert request.method == "POST"
|
||||
assert request.headers["authorization"] == f"Bearer {_ACCESS_TOKEN}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == _MODEL.removeprefix("databricks/")
|
||||
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
|
||||
return Reply(body=_completion(identity))
|
||||
return Reply(status=401, body=json.dumps({"error": f"unauthenticated path {request.target}"}).encode())
|
||||
|
||||
overrides: Final = {"DATABRICKS_CLIENT_ID": _CLIENT_ID, "DATABRICKS_CLIENT_SECRET": _CLIENT_SECRET}
|
||||
with wire_server(respond) as wire, owned_proxy(gateway, tmp_path, overrides) as candidate:
|
||||
with candidate.scenario() as scenario:
|
||||
model: Final = scenario.model(model=_MODEL, api_base=f"{wire.url}/ai-gateway/mlflow/v1", api_key=None)
|
||||
response: Final = candidate.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["id"] == identity
|
||||
assert payload["choices"] == [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {"content": "the workspace origin", "role": "assistant"},
|
||||
}
|
||||
]
|
||||
assert payload["usage"] == {"prompt_tokens": 9, "completion_tokens": 4, "total_tokens": 13}
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [
|
||||
("POST", "/oidc/v1/token"),
|
||||
("POST", "/ai-gateway/mlflow/v1/chat/completions"),
|
||||
]
|
||||
40
tests/integration/providers/test_deepseek_vision_wire.py
Normal file
40
tests/integration/providers/test_deepseek_vision_wire.py
Normal file
|
|
@ -0,0 +1,40 @@
|
|||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from tests.integration._support.client import JSON_OBJECT, Gateway, object_value
|
||||
|
||||
_VISION_MODEL: Final = "deepseek-v4-flash-vision-exp"
|
||||
_API_KEY: Final = "synthetic-deepseek-key"
|
||||
_VISION_CONTENT: Final[JsonValue] = [
|
||||
{"type": "text", "text": "what is in this image?"},
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/pic.png"}},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.deepseek.vision_image_content_list_reaches_provider")
|
||||
def test_deepseek_vision_forwards_image_url_content_list_instead_of_collapsing_to_text(gateway: Gateway) -> None:
|
||||
with gateway.scenario() as scenario, httpx.Client(base_url=gateway.upstream_url, trust_env=False) as upstream:
|
||||
upstream.get("/__observations").raise_for_status()
|
||||
model: Final = scenario.model(
|
||||
model=f"deepseek/{_VISION_MODEL}",
|
||||
api_key=_API_KEY,
|
||||
model_info={"mode": "chat", "supports_vision": True},
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _VISION_CONTENT}]},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
observations: Final = JSON_OBJECT.validate_json(upstream.get("/__observations").content)["requests"]
|
||||
assert isinstance(observations, list)
|
||||
assert len(observations) == 1, response.text
|
||||
observed: Final = object_value(observations[0])
|
||||
assert observed["path"] == "/v1/chat/completions", response.text
|
||||
assert observed["authorization"] == f"Bearer {_API_KEY}", response.text
|
||||
body: Final = object_value(observed["body"])
|
||||
assert body["model"] == _VISION_MODEL, response.text
|
||||
assert body["messages"] == [{"role": "user", "content": _VISION_CONTENT}], response.text
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_ROUTER_SLUG: Final = "routers/glm-latest"
|
||||
_ROUTER_RESOURCE: Final = "accounts/fireworks/routers/glm-latest"
|
||||
_API_KEY: Final = "synthetic-fireworks-key"
|
||||
_PROMPT: Final = "route me through the router"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _provider_body(request: Request, target: str) -> dict[str, JsonValue]:
|
||||
assert request.method == "POST"
|
||||
assert request.target == target
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
return _JSON_OBJECT.validate_json(request.body)
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_chat_sends_router_resource_name")
|
||||
def test_fireworks_router_slug_chat_sends_router_resource_not_models_path(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
body: Final = _provider_body(request, "/chat/completions")
|
||||
assert body["model"] == _ROUTER_RESOURCE, body
|
||||
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "fw-router-chat",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _ROUTER_RESOURCE,
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": "routed"}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}]},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["choices"] == [
|
||||
{"finish_reason": "stop", "index": 0, "message": {"role": "assistant", "content": "routed"}}
|
||||
]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.fireworks_ai.router_slug_text_completion_sends_router_resource_name")
|
||||
def test_fireworks_router_slug_text_completion_sends_router_resource_not_models_path(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
body: Final = _provider_body(request, "/completions")
|
||||
assert body["model"] == _ROUTER_RESOURCE, body
|
||||
assert body["prompt"] == _PROMPT
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "fw-router-text",
|
||||
"object": "text_completion",
|
||||
"created": 1,
|
||||
"model": _ROUTER_RESOURCE,
|
||||
"choices": [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}],
|
||||
"usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"fireworks_ai/{_ROUTER_SLUG}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request("POST", "/v1/completions", {"model": model, "prompt": _PROMPT})
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["choices"] == [{"index": 0, "text": "routed", "finish_reason": "stop", "logprobs": None}]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/completions")]
|
||||
66
tests/integration/providers/test_openai_chat_wire.py
Normal file
66
tests/integration/providers/test_openai_chat_wire.py
Normal file
|
|
@ -0,0 +1,66 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "gpt-5.4-mini"
|
||||
_API_KEY: Final = "synthetic-openai-key"
|
||||
_PROMPT: Final = "Summarize this conversation in one sentence."
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
def _completion(identity: str, content: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": content}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 19, "completion_tokens": 7, "total_tokens": 26},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.covers("providers.openai_chat_wire.tool_choice_without_tools_is_dropped_before_the_wire")
|
||||
def test_openai_chat_tool_choice_without_tools_is_not_forwarded(gateway: Gateway) -> None:
|
||||
identity: Final = f"openai-toolless-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/chat/completions"
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
assert body["model"] == _BACKEND
|
||||
assert body["messages"] == [{"role": "user", "content": _PROMPT}]
|
||||
assert "tool_choice" not in body, body
|
||||
assert "tools" not in body, body
|
||||
return Reply(body=_completion(identity, "One sentence."))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"openai/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], "tool_choice": "none"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["id"] == identity
|
||||
assert payload["choices"] == [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "One sentence.",
|
||||
"provider_specific_fields": {"refusal": None},
|
||||
},
|
||||
"provider_specific_fields": {},
|
||||
}
|
||||
]
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
75
tests/integration/providers/test_openai_image_edit_wire.py
Normal file
75
tests/integration/providers/test_openai_image_edit_wire.py
Normal file
|
|
@ -0,0 +1,75 @@
|
|||
import json
|
||||
from email.message import Message
|
||||
from email.parser import BytesParser
|
||||
from email.policy import HTTP
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import BaseModel
|
||||
|
||||
_PNG_BYTES: Final = (
|
||||
b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00"
|
||||
b"\x1f\x15\xc4\x89\x00\x00\x00\rIDAT\x08\xd7c\xf8\xcf\xc0\xf0\x1f\x00\x05\x00\x01\xff"
|
||||
b"\x89\x99=\x1d\x00\x00\x00\x00IEND\xaeB`\x82"
|
||||
)
|
||||
_PROMPT: Final = "turn the red circle green"
|
||||
_EDITED_IMAGE_B64: Final = "aW50ZWdyYXRpb24tZWRpdGVkLWltYWdl"
|
||||
|
||||
|
||||
class _Image(BaseModel):
|
||||
b64_json: str
|
||||
|
||||
|
||||
class _ImageResponse(BaseModel):
|
||||
data: tuple[_Image, ...]
|
||||
|
||||
|
||||
def _multipart_parts(request: Request) -> tuple[Message, ...]:
|
||||
envelope: Final = f"content-type: {request.headers['content-type']}\r\n\r\n".encode() + request.body
|
||||
parsed: Final = BytesParser(policy=HTTP).parsebytes(envelope)
|
||||
assert parsed.is_multipart(), request.headers["content-type"]
|
||||
return tuple(parsed.iter_parts())
|
||||
|
||||
|
||||
def _text_fields(parts: tuple[Message, ...]) -> dict[str, str]:
|
||||
return {
|
||||
part.get_param("name", header="content-disposition"): part.get_payload(decode=True).decode()
|
||||
for part in parts
|
||||
if part.get_filename() is None
|
||||
}
|
||||
|
||||
|
||||
def _file_fields(parts: tuple[Message, ...]) -> dict[str, bytes]:
|
||||
return {
|
||||
part.get_param("name", header="content-disposition"): part.get_payload(decode=True)
|
||||
for part in parts
|
||||
if part.get_filename() is not None
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.openai.image_edit_forwards_provider_specific_form_fields")
|
||||
def test_openai_compatible_image_edit_forwards_seed_form_field_to_backend(gateway: Gateway) -> None:
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/v1/images/edits"
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
parts: Final = _multipart_parts(request)
|
||||
assert _text_fields(parts) == {"model": "gpt-image-1", "prompt": _PROMPT, "seed": "42"}
|
||||
assert _file_fields(parts) == {"image[]": _PNG_BYTES}
|
||||
return Reply(body=json.dumps({"created": 1700000000, "data": [{"b64_json": _EDITED_IMAGE_B64}]}).encode())
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/gpt-image-1", api_base=f"{wire.url}/v1", api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request_multipart(
|
||||
"/v1/images/edits",
|
||||
{"model": model, "prompt": _PROMPT, "seed": "42"},
|
||||
{"image": ("red_circle.png", _PNG_BYTES, "image/png")},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _ImageResponse.model_validate_json(response.content)
|
||||
assert [image.b64_json for image in payload.data] == [_EDITED_IMAGE_B64], response.text
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/v1/images/edits")]
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.responses_bridge.max_output_tokens_incomplete_maps_to_length")
|
||||
def test_chat_over_responses_deployment_returns_length_when_output_tokens_run_out(gateway: Gateway) -> None:
|
||||
identity: Final = "responses-incomplete-" + uuid.uuid4().hex
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST" and request.target == "/responses", request.target
|
||||
assert request.headers["authorization"] == "Bearer synthetic-openai-key"
|
||||
body: Final = json.loads(request.body)
|
||||
assert body["model"] == "gpt-5.3-codex"
|
||||
assert body["max_output_tokens"] == 16
|
||||
assert body["reasoning"] == {"effort": "high"}
|
||||
assert body["input"] == [
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": f"explain the plan in detail {identity}"}],
|
||||
}
|
||||
]
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": f"resp_{identity}",
|
||||
"object": "response",
|
||||
"created_at": 1789788253,
|
||||
"status": "incomplete",
|
||||
"incomplete_details": {"reason": "max_output_tokens"},
|
||||
"model": "gpt-5.3-codex",
|
||||
"output": [{"type": "reasoning", "id": f"rs_{identity}", "summary": []}],
|
||||
"usage": {"input_tokens": 12, "output_tokens": 16, "total_tokens": 28},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model="openai/responses/gpt-5.3-codex", api_base=wire.url, api_key="synthetic-openai-key"
|
||||
)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"explain the plan in detail {identity}"}],
|
||||
"reasoning_effort": "high",
|
||||
"max_completion_tokens": 16,
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body: Final = response.json()
|
||||
assert len(wire.drain()) == 1
|
||||
assert [choice["finish_reason"] for choice in body["choices"]] == ["length"], response.text
|
||||
assert body["choices"][0]["message"]["content"] == "", response.text
|
||||
assert body["choices"][0]["message"]["role"] == "assistant", response.text
|
||||
assert body["usage"]["prompt_tokens"] == 12 and body["usage"]["completion_tokens"] == 16, response.text
|
||||
assert body["usage"]["total_tokens"] == 28, response.text
|
||||
86
tests/integration/providers/test_tencent_chat_wire.py
Normal file
86
tests/integration/providers/test_tencent_chat_wire.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
import json
|
||||
import uuid
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_BACKEND: Final = "deepseek-v4-pro"
|
||||
_API_KEY: Final = "synthetic-tencent-key"
|
||||
_PROMPT: Final = "What is 17 + 26? Answer with just the number."
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_REASONING_REQUESTS: Final[tuple[tuple[str, dict[str, JsonValue], dict[str, JsonValue]], ...]] = (
|
||||
("thinking_enabled", {"thinking": {"type": "enabled"}}, {"type": "enabled"}),
|
||||
("reasoning_effort_none", {"reasoning_effort": "none"}, {"type": "disabled"}),
|
||||
)
|
||||
|
||||
|
||||
def _completion(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "43", "reasoning_content": "17 plus 26 is 43."},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64},
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
@pytest.mark.covers("other.provider_wire.tencent.thinking_reaches_provider_in_request_body")
|
||||
@pytest.mark.parametrize(
|
||||
("reasoning_params", "expected_thinking"),
|
||||
tuple(case[1:] for case in _REASONING_REQUESTS),
|
||||
ids=tuple(case[0] for case in _REASONING_REQUESTS),
|
||||
)
|
||||
def test_tencent_thinking_is_sent_in_provider_body_instead_of_failing_the_request(
|
||||
gateway: Gateway, reasoning_params: dict[str, JsonValue], expected_thinking: dict[str, JsonValue]
|
||||
) -> None:
|
||||
identity: Final = f"tencent-thinking-{uuid.uuid4().hex}"
|
||||
|
||||
def respond(request: Request) -> Reply:
|
||||
assert request.method == "POST"
|
||||
assert request.target == "/chat/completions"
|
||||
assert request.headers["authorization"] == f"Bearer {_API_KEY}"
|
||||
assert request.headers["content-type"] == "application/json"
|
||||
assert _JSON_OBJECT.validate_json(request.body) == {
|
||||
"model": _BACKEND,
|
||||
"messages": [{"role": "user", "content": _PROMPT}],
|
||||
"thinking": expected_thinking,
|
||||
}
|
||||
return Reply(body=_completion(identity))
|
||||
|
||||
with wire_server(respond) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"tencent/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "messages": [{"role": "user", "content": _PROMPT}], **reasoning_params},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
payload: Final = _JSON_OBJECT.validate_json(response.content)
|
||||
assert payload["id"] == identity
|
||||
assert payload["choices"] == [
|
||||
{
|
||||
"finish_reason": "stop",
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": "43",
|
||||
"reasoning_content": "17 plus 26 is 43.",
|
||||
"provider_specific_fields": {"refusal": None},
|
||||
},
|
||||
"provider_specific_fields": {},
|
||||
}
|
||||
]
|
||||
assert payload["usage"] == {"prompt_tokens": 23, "completion_tokens": 41, "total_tokens": 64}
|
||||
assert [(request.method, request.target) for request in wire.drain()] == [("POST", "/chat/completions")]
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue