mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'origin/main' into litellm-providers/price-sync-vertex-ai
This commit is contained in:
commit
cc38c971f4
176 changed files with 12490 additions and 2508 deletions
1
.github/e2e-stack/select_tests.py
vendored
1
.github/e2e-stack/select_tests.py
vendored
|
|
@ -11,6 +11,7 @@ UNSUPPORTED: Final = re.compile(
|
|||
r"|^tests/e2e/guardrails/test_presidio_masking_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_otel_v2_langfuse_generation_output_e2e\.py$"
|
||||
r"|^tests/e2e/logging/test_langsmith_batch_serialization_e2e\.py$"
|
||||
r"|^tests/e2e/secret_manager/"
|
||||
)
|
||||
HARNESS: Final = re.compile(
|
||||
r"^tests/e2e/[A-Za-z0-9_.-]+\.(py|ini)$"
|
||||
|
|
|
|||
33
.github/workflows/compat-matrix-image.yml
vendored
Normal file
33
.github/workflows/compat-matrix-image.yml
vendored
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
name: Compat Matrix Image
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
paths:
|
||||
- tests/e2e/claude_code/cron_vm/**
|
||||
- .github/workflows/compat-matrix-image.yml
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
compat-matrix-image:
|
||||
name: compat-matrix-image
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Build the Render cron image
|
||||
run: docker build -f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix:${{ github.sha }} tests/e2e
|
||||
|
||||
- name: Run the pinned binaries as the cron user
|
||||
run: |
|
||||
docker run --rm compat-matrix:${{ github.sha }} bash -c 'set -e; whoami; claude --version; gh --version; uv --version'
|
||||
20
.github/workflows/issue_fixed_comment.yml
vendored
20
.github/workflows/issue_fixed_comment.yml
vendored
|
|
@ -6,8 +6,12 @@ on:
|
|||
workflow_dispatch:
|
||||
inputs:
|
||||
issue_number:
|
||||
description: "Closed issue number to comment on manually."
|
||||
required: true
|
||||
description: "Closed issue number to comment on and close the superseded pull requests of. Ignored by a sweep."
|
||||
required: false
|
||||
sweep:
|
||||
description: "Close every open pull request whose linked issues were all fixed on the default branch. Reads every open pull request, so run it at most once an hour."
|
||||
type: boolean
|
||||
default: false
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/issue_fixed_comment.yml
|
||||
|
|
@ -39,16 +43,17 @@ jobs:
|
|||
with:
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Test the closer lookup, the release placement and the comment
|
||||
- name: Test the closer lookup, the release placement, the comment and the superseded pull request close
|
||||
run: bun test scripts/comment-fixed-issue.test.ts
|
||||
|
||||
comment-fixed-issue:
|
||||
if: github.event_name != 'pull_request' && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Checkout scripts
|
||||
uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
@ -59,13 +64,16 @@ jobs:
|
|||
- name: Setup Bun
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
# Exact version, never latest: the next step holds an issues: write token
|
||||
# Exact version, never latest: the next step holds issues: write and pull-requests: write tokens
|
||||
bun-version: "1.4.0"
|
||||
|
||||
- name: Name the release that carries the fix
|
||||
- name: Name the release that carries the fix and close the pull requests it supersedes
|
||||
shell: bash
|
||||
run: bun run scripts/comment-fixed-issue.ts | tee -a "${GITHUB_STEP_SUMMARY}"
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ISSUE_NUMBER: ${{ github.event.issue.number || github.event.inputs.issue_number }}
|
||||
SWEEP: ${{ github.event.inputs.sweep }}
|
||||
DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
DRY_RUN: ${{ vars.ISSUE_FIXED_COMMENT_ENABLED != 'true' }}
|
||||
CLOSE_PRS_DRY_RUN: ${{ vars.ISSUE_FIXED_CLOSE_PRS_ENABLED != 'true' }}
|
||||
|
|
|
|||
10
.github/workflows/test-rust.yml
vendored
10
.github/workflows/test-rust.yml
vendored
|
|
@ -105,6 +105,16 @@ jobs:
|
|||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install Python dependencies for the bridge tests
|
||||
working-directory: .
|
||||
run: |
|
||||
uv sync --frozen --no-install-project
|
||||
echo "PYTHONPATH=$PWD/.venv/lib/$(ls .venv/lib)/site-packages" >> "$GITHUB_ENV"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
||||
- uses: taiki-e/install-action@d438492cf8a250514fa2d34b30bc3c0dc37c65ff # v2.87.8
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG PROXY_EXTRAS_SOURCE=published
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.69"
|
||||
version = "0.1.70"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.69"
|
||||
version = "0.1.70"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
||||
ARG PGBOUNCER_VERSION=1.25.2
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.100"
|
||||
version = "0.4.101"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.100"
|
||||
version = "0.4.101"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
21
litellm-rust/Cargo.lock
generated
21
litellm-rust/Cargo.lock
generated
|
|
@ -2946,6 +2946,7 @@ name = "litellm-core-utils"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -3075,6 +3076,7 @@ dependencies = [
|
|||
"aws-sdk-secretsmanager",
|
||||
"bytes",
|
||||
"criterion",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
"litellm-auth",
|
||||
"litellm-auth-aws",
|
||||
|
|
@ -3093,6 +3095,7 @@ dependencies = [
|
|||
"litellm-callbacks-legacy-python",
|
||||
"litellm-core",
|
||||
"litellm-core-utils",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
"litellm-http",
|
||||
"litellm-llms",
|
||||
|
|
@ -3100,6 +3103,7 @@ dependencies = [
|
|||
"litellm-secrets-aws",
|
||||
"litellm-secrets-types",
|
||||
"litellm-token-counter",
|
||||
"litellm-tracing",
|
||||
"litellm-types",
|
||||
"pyo3",
|
||||
"pyo3-async-runtimes",
|
||||
|
|
@ -3172,11 +3176,11 @@ dependencies = [
|
|||
"litellm-auth-aws",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3208,6 +3212,7 @@ dependencies = [
|
|||
"base64 0.22.1",
|
||||
"litellm-core-utils",
|
||||
"litellm-secrets-types",
|
||||
"litellm-tracing",
|
||||
"moka",
|
||||
"percent-encoding",
|
||||
"reqwest 0.12.28",
|
||||
|
|
@ -3216,7 +3221,6 @@ dependencies = [
|
|||
"serde_json",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"veil",
|
||||
"wiremock",
|
||||
]
|
||||
|
|
@ -3331,6 +3335,19 @@ dependencies = [
|
|||
"tiktoken-rs",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"fancy-regex 0.19.2",
|
||||
"percent-encoding",
|
||||
"rstest",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-types"
|
||||
version = "0.1.0"
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ license = "MIT"
|
|||
repository = "https://github.com/BerriAI/litellm"
|
||||
|
||||
[workspace.dependencies]
|
||||
litellm-tracing = { path = "crates/tracing" }
|
||||
tracing = "0.1"
|
||||
litellm-core = { path = "crates/core" }
|
||||
litellm-host = { path = "crates/host" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
|
|
|
|||
|
|
@ -7,6 +7,7 @@ repository.workspace = true
|
|||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-types.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,109 +1 @@
|
|||
use fancy_regex::Regex;
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn minimum_custom_key_length() -> usize {
|
||||
std::env::var("MINIMUM_CUSTOM_KEY_LENGTH")
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse().ok())
|
||||
.unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH)
|
||||
}
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
/// Python's `_ENABLE_SECRET_REDACTION` pattern set, compiled once per configuration.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
Self { pattern }
|
||||
}
|
||||
|
||||
/// `None` when `LITELLM_DISABLE_REDACT_SECRETS` turns redaction off.
|
||||
pub fn from_env() -> Option<Self> {
|
||||
let disabled = std::env::var("LITELLM_DISABLE_REDACT_SECRETS")
|
||||
.is_ok_and(|value| value.eq_ignore_ascii_case("true"));
|
||||
(!disabled).then(|| Self::new(minimum_custom_key_length()))
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.pattern.replace_all(value, REDACTED).into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
pub use litellm_tracing::{REDACTED, SecretRedactor};
|
||||
|
|
|
|||
|
|
@ -19,6 +19,9 @@ huggingface = ["litellm-token-counter/huggingface"]
|
|||
tiktoken = ["litellm-token-counter/tiktoken"]
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
litellm-host.workspace = true
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
litellm-cache.workspace = true
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
use crate::logger::run_async;
|
||||
use litellm_cache_response::PartialHits;
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, run_async, to_py};
|
||||
use litellm_host_python::{ExecutionStep, from_py, release_gil, to_py};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
|
|
|
|||
|
|
@ -1,10 +1,11 @@
|
|||
use crate::logger::run_sync_value;
|
||||
use litellm_auth_aws::AwsAuthConfig;
|
||||
use litellm_cache_gcs::{DEFAULT_ENDPOINT, GcsConfig};
|
||||
use litellm_cache_qdrant_semantic::{OpenAiEmbedderConfig, Quantization};
|
||||
use litellm_cache_redis::{RedisNode, RedisTopology};
|
||||
use litellm_cache_redis_semantic::RedisSemanticConfig;
|
||||
use litellm_cache_s3::{S3CacheConfig, S3Endpoint};
|
||||
use litellm_host_python::{release_gil, run_sync_value};
|
||||
use litellm_host_python::release_gil;
|
||||
use litellm_http::ClientVariant;
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
|
|
|
|||
|
|
@ -470,7 +470,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move {
|
||||
service
|
||||
|
|
@ -495,7 +495,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_lookup(&request, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -550,7 +550,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store(&request, response, now()).await },
|
||||
super::cache_error,
|
||||
|
|
@ -619,7 +619,7 @@ impl NativeResponseCache {
|
|||
match self {
|
||||
Self::Exact(_) | Self::QdrantSemantic(_) => {
|
||||
let service = self.clone();
|
||||
litellm_host_python::run_async(
|
||||
crate::logger::run_async(
|
||||
py,
|
||||
async move { service.async_store_batch(entries, now()).await },
|
||||
super::cache_error,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::{collections::VecDeque, time::Duration};
|
||||
|
||||
use litellm_cache::Error;
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep, run_async};
|
||||
use litellm_host_python::{Execution, ExecutionBody, ExecutionStep};
|
||||
use pyo3::{
|
||||
PyTraverseError, PyVisit,
|
||||
exceptions::{PyException, PyRuntimeError},
|
||||
|
|
|
|||
|
|
@ -102,7 +102,7 @@ pub(crate) fn call_config(
|
|||
.without_missing_files(&|path: &Path| path.exists());
|
||||
let resolution = Resolution::from(&settings);
|
||||
for unsupported in unreported(&REPORTED_UNSUPPORTED, resolution.unsupported) {
|
||||
PythonSettings::warn(py, &unsupported.to_string())?;
|
||||
crate::logger::capture(py).scope(|| litellm_tracing::warn!("{unsupported}"));
|
||||
}
|
||||
Ok(resolution.config)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ mod credentials;
|
|||
mod diagnostics;
|
||||
mod errors;
|
||||
mod http;
|
||||
mod logger;
|
||||
mod marshal;
|
||||
mod python_settings;
|
||||
mod routes;
|
||||
|
|
@ -25,6 +26,8 @@ mod _native {
|
|||
#[pymodule_export]
|
||||
use crate::errors::{RustBridgeDeclined, RustUpstreamError};
|
||||
#[pymodule_export]
|
||||
use crate::logger::NativeDiagnosticProcessor;
|
||||
#[pymodule_export]
|
||||
use crate::routes::audio_transcription::{atranscription, transcription};
|
||||
#[pymodule_export]
|
||||
use crate::routes::chat_completions::{
|
||||
|
|
@ -87,6 +90,7 @@ mod tests {
|
|||
"chat_completions",
|
||||
"achat_completions",
|
||||
"ResponsesWebSocketConnection",
|
||||
"NativeDiagnosticProcessor",
|
||||
"TokenCounter",
|
||||
"Tokenizer",
|
||||
"gil_stats",
|
||||
|
|
|
|||
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
46
litellm-rust/crates/python-bridge/src/logger/execution.rs
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
use std::future::Future;
|
||||
|
||||
use pyo3::prelude::*;
|
||||
use serde::Serialize;
|
||||
|
||||
pub(crate) fn run_sync<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Py<PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_async<T, E, F>(
|
||||
py: Python<'_>,
|
||||
future: F,
|
||||
map_error: fn(E) -> PyErr,
|
||||
) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: Serialize + Send + 'static,
|
||||
E: Send + 'static,
|
||||
F: Future<Output = Result<T, E>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async(py, super::capture(py).instrument(future), map_error)
|
||||
}
|
||||
|
||||
pub(crate) fn run_sync_value<T, F>(py: Python<'_>, future: F) -> PyResult<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_sync_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
|
||||
pub(crate) fn run_async_value<T, F>(py: Python<'_>, future: F) -> PyResult<Bound<'_, PyAny>>
|
||||
where
|
||||
T: for<'py> IntoPyObject<'py> + Send + 'static,
|
||||
F: Future<Output = PyResult<T>> + Send + 'static,
|
||||
{
|
||||
litellm_host_python::run_async_value(py, super::capture(py).instrument(future))
|
||||
}
|
||||
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
41
litellm-rust/crates/python-bridge/src/logger/machine.rs
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
use std::sync::OnceLock;
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, Step},
|
||||
route::Route,
|
||||
};
|
||||
use litellm_tracing::Logger;
|
||||
use pyo3::Python;
|
||||
|
||||
pub(crate) struct LoggedMachine<M> {
|
||||
machine: M,
|
||||
logger: OnceLock<Logger>,
|
||||
}
|
||||
|
||||
impl<M> LoggedMachine<M> {
|
||||
pub(crate) fn new(machine: M) -> Self {
|
||||
Self {
|
||||
machine,
|
||||
logger: OnceLock::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<M: Machine> Machine for LoggedMachine<M> {
|
||||
type Route = M::Route;
|
||||
type Complete = M::Complete;
|
||||
|
||||
fn resume(&mut self, result: Option<HostResult<Self::Route>>) -> Step<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.resume(result))))
|
||||
}
|
||||
|
||||
fn interrupt(
|
||||
&mut self,
|
||||
failure: HostFailure<<Self::Route as Route>::Error>,
|
||||
) -> Interrupted<'_, Self> {
|
||||
let logger = self.logger.get_or_init(|| Python::attach(super::capture));
|
||||
Box::pin(logger.instrument(logger.scope(|| self.machine.interrupt(failure))))
|
||||
}
|
||||
}
|
||||
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
166
litellm-rust/crates/python-bridge/src/logger/mod.rs
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
mod execution;
|
||||
mod machine;
|
||||
|
||||
pub(crate) use execution::{run_async, run_async_value, run_sync, run_sync_value};
|
||||
pub(crate) use machine::LoggedMachine;
|
||||
|
||||
use litellm_host_python::Pythonized;
|
||||
use litellm_tracing::{DiagnosticInput, Level, Logger, Metadata, Policy, Processor, Record, Sink};
|
||||
use pyo3::exceptions::PyRuntimeError;
|
||||
use pyo3::prelude::*;
|
||||
|
||||
const MODULE: &str = "litellm.rust_bridge.logger";
|
||||
type NativeDiagnosticOutput = (String, Option<String>, Option<String>, Vec<String>, bool);
|
||||
|
||||
#[pyclass]
|
||||
pub(crate) struct NativeDiagnosticProcessor {
|
||||
inner: Processor,
|
||||
}
|
||||
|
||||
#[pymethods]
|
||||
impl NativeDiagnosticProcessor {
|
||||
#[new]
|
||||
fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
inner: Processor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
fn redact_text(&self, text: &str) -> PyResult<String> {
|
||||
self.inner.redact_text(text).map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_structured_text(&self, key: Option<&str>, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_structured_text(key, text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn redact_client_message(&self, text: &str) -> PyResult<String> {
|
||||
self.inner
|
||||
.redact_client_message(text)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
#[pyo3(signature = (message, exception, stack, leaves, policy))]
|
||||
fn process_diagnostic(
|
||||
&self,
|
||||
message: String,
|
||||
exception: Option<String>,
|
||||
stack: Option<String>,
|
||||
leaves: Vec<(Option<String>, String)>,
|
||||
policy: (bool, i64, i64),
|
||||
) -> PyResult<NativeDiagnosticOutput> {
|
||||
let input = DiagnosticInput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
};
|
||||
let policy = Policy {
|
||||
redact: policy.0,
|
||||
base64_limit: policy.1,
|
||||
text_limit: policy.2,
|
||||
};
|
||||
self.inner
|
||||
.process_diagnostic(&input, policy)
|
||||
.map(|output| {
|
||||
(
|
||||
output.message,
|
||||
output.exception,
|
||||
output.stack,
|
||||
output.leaves,
|
||||
output.changed,
|
||||
)
|
||||
})
|
||||
.map_err(processing_error)
|
||||
}
|
||||
|
||||
fn scrub_access_arguments(&self, arguments: Vec<String>) -> PyResult<Vec<String>> {
|
||||
self.inner
|
||||
.scrub_access_arguments(&arguments)
|
||||
.map_err(processing_error)
|
||||
}
|
||||
}
|
||||
|
||||
fn processing_error(_: fancy_regex::Error) -> PyErr {
|
||||
PyRuntimeError::new_err("diagnostic processing failed")
|
||||
}
|
||||
|
||||
struct PythonSink {
|
||||
correlation: (String, String),
|
||||
}
|
||||
|
||||
fn level(level: &Level) -> u8 {
|
||||
match *level {
|
||||
Level::ERROR => 40,
|
||||
Level::WARN => 30,
|
||||
Level::INFO => 20,
|
||||
Level::DEBUG | Level::TRACE => 10,
|
||||
}
|
||||
}
|
||||
|
||||
fn report<T: Default>(py: Python<'_>, result: PyResult<T>) -> T {
|
||||
match result {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
error.write_unraisable(py, None);
|
||||
T::default()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Sink for PythonSink {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
if !metadata.target().starts_with("litellm_") && !metadata.target().starts_with("_native::")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method1("enabled", (level(metadata.level()),)))
|
||||
.and_then(|enabled| enabled.extract()),
|
||||
)
|
||||
})
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
Python::try_attach(|py| {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE).and_then(|module| {
|
||||
module
|
||||
.call_method1(
|
||||
"emit",
|
||||
(
|
||||
level(record.metadata.level()),
|
||||
&record.message,
|
||||
record.metadata.file().unwrap_or_default(),
|
||||
record.metadata.line().unwrap_or_default(),
|
||||
record.metadata.target(),
|
||||
Pythonized(&record.fields),
|
||||
(&self.correlation.0, &self.correlation.1),
|
||||
),
|
||||
)
|
||||
.map(|_| ())
|
||||
}),
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capture(py: Python<'_>) -> Logger {
|
||||
report(
|
||||
py,
|
||||
py.import(MODULE)
|
||||
.and_then(|module| module.call_method0("context"))
|
||||
.and_then(|value| value.extract())
|
||||
.map(|correlation| Logger::new(PythonSink { correlation })),
|
||||
)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
295
litellm-rust/crates/python-bridge/src/logger/tests.rs
Normal file
|
|
@ -0,0 +1,295 @@
|
|||
use std::{process::Command, task::Poll};
|
||||
|
||||
use litellm_host::{
|
||||
host::HostResult,
|
||||
machine::{HostFailure, Interrupted, Machine, MachineStep, Step},
|
||||
route::Route,
|
||||
};
|
||||
|
||||
use pyo3::{prelude::*, types::PyDict};
|
||||
|
||||
struct DiagnosticMachine;
|
||||
|
||||
impl Route for DiagnosticMachine {
|
||||
type Response = ();
|
||||
type Error = String;
|
||||
type Op = ();
|
||||
type OpResult = ();
|
||||
type Chunk = ();
|
||||
type StreamHead = ();
|
||||
}
|
||||
|
||||
impl Machine for DiagnosticMachine {
|
||||
type Route = Self;
|
||||
type Complete = ();
|
||||
|
||||
fn resume(&mut self, _: Option<HostResult<Self>>) -> Step<'_, Self> {
|
||||
litellm_tracing::warn!("machine started");
|
||||
Box::pin(async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("machine warning");
|
||||
Ok(MachineStep::Complete(()))
|
||||
})
|
||||
}
|
||||
|
||||
fn interrupt(&mut self, _: HostFailure<String>) -> Interrupted<'_, Self> {
|
||||
Box::pin(async {
|
||||
litellm_tracing::warn!("machine interrupted");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn machine_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
let mut machine = super::LoggedMachine::new(DiagnosticMachine);
|
||||
let mut future = Box::pin(async move {
|
||||
machine
|
||||
.resume(None)
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)?;
|
||||
machine
|
||||
.interrupt(HostFailure::Error("stop".into()))
|
||||
.await
|
||||
.map_err(pyo3::exceptions::PyValueError::new_err)
|
||||
});
|
||||
assert!(matches!(
|
||||
litellm_host_python::poll_async_value(py, future.as_mut())?,
|
||||
Poll::Pending
|
||||
));
|
||||
litellm_host_python::run_async_value(py, future)
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn warning(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::warn!(attempt = 3, retry = true, "native warning");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn levels(py: Python<'_>) {
|
||||
super::capture(py).scope(|| {
|
||||
litellm_tracing::trace!("trace");
|
||||
litellm_tracing::debug!("debug");
|
||||
litellm_tracing::info!("info");
|
||||
litellm_tracing::warn!("warn");
|
||||
litellm_tracing::error!("error");
|
||||
litellm_tracing::warn!(target: "unrelated_transport", "private wire data");
|
||||
});
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn asynchronous_warning(py: Python<'_>) -> PyResult<Bound<'_, PyAny>> {
|
||||
super::run_async_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("async warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_warning(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
tokio::task::yield_now().await;
|
||||
litellm_tracing::warn!("sync warning");
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn synchronous_failure(py: Python<'_>) -> PyResult<()> {
|
||||
super::run_sync_value(py, async {
|
||||
litellm_tracing::warn!("failure diagnostic");
|
||||
Err(pyo3::exceptions::PyValueError::new_err("request failed"))
|
||||
})
|
||||
}
|
||||
|
||||
#[pyfunction]
|
||||
fn http_warning(py: Python<'_>) -> PyResult<()> {
|
||||
crate::http::call_config(py, &PyDict::new(py), false).map(|_| ())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn native_events_reach_python_with_levels_context_reentry_and_http_deduplication() {
|
||||
if std::env::var_os("LITELLM_LOGGER_TEST_PROCESS").is_none() {
|
||||
let output = Command::new(std::env::current_exe().unwrap())
|
||||
.args([
|
||||
"--exact",
|
||||
std::thread::current().name().unwrap(),
|
||||
"--nocapture",
|
||||
])
|
||||
.env("LITELLM_LOGGER_TEST_PROCESS", "1")
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}\n{}",
|
||||
String::from_utf8_lossy(&output.stdout),
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
return;
|
||||
}
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let locals = PyDict::new(py);
|
||||
locals
|
||||
.set_item(
|
||||
"repo_root",
|
||||
concat!(env!("CARGO_MANIFEST_DIR"), "/../../.."),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"machine_warning",
|
||||
wrap_pyfunction!(machine_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_failure",
|
||||
wrap_pyfunction!(synchronous_failure, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("levels", wrap_pyfunction!(levels, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("warning", wrap_pyfunction!(warning, py).unwrap())
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"asynchronous_warning",
|
||||
wrap_pyfunction!(asynchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item(
|
||||
"synchronous_warning",
|
||||
wrap_pyfunction!(synchronous_warning, py).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
locals
|
||||
.set_item("http_warning", wrap_pyfunction!(http_warning, py).unwrap())
|
||||
.unwrap();
|
||||
let importable = py
|
||||
.eval(
|
||||
c"__import__('importlib.util', fromlist=['util']).find_spec('dotenv') is not None",
|
||||
Some(&locals),
|
||||
Some(&locals),
|
||||
)
|
||||
.unwrap()
|
||||
.is_truthy()
|
||||
.unwrap();
|
||||
if !importable {
|
||||
eprintln!("SKIP: litellm package dependencies are not importable in this interpreter");
|
||||
return;
|
||||
}
|
||||
py.run(c"
|
||||
import asyncio
|
||||
import logging
|
||||
import sys
|
||||
sys.path.insert(0, repo_root)
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger, session_id_var, trace_id_var
|
||||
|
||||
class Capture(logging.Handler):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.records = []
|
||||
def emit(self, record):
|
||||
self.records.append(record)
|
||||
warning()
|
||||
|
||||
class Broken(logging.Handler):
|
||||
def emit(self, record):
|
||||
raise ValueError('handler failed')
|
||||
|
||||
capture = Capture()
|
||||
old_handlers = verbose_logger.handlers
|
||||
old_level = verbose_logger.level
|
||||
old_correlation = litellm.request_correlation_in_logs
|
||||
old_curve = litellm.ssl_ecdh_curve
|
||||
old_unraisable = sys.unraisablehook
|
||||
failures = []
|
||||
try:
|
||||
verbose_logger.handlers = [capture]
|
||||
litellm.request_correlation_in_logs = True
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert capture.records == []
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
record = capture.records[0]
|
||||
assert record.getMessage() == 'native warning'
|
||||
assert record.levelno == logging.WARNING
|
||||
assert record.rust_fields == {'attempt': 3, 'retry': True}
|
||||
assert record.pathname.endswith('logger/tests.rs')
|
||||
assert record.lineno > 0
|
||||
assert record.rust_target.endswith('logger::tests')
|
||||
verbose_logger.setLevel(logging.ERROR)
|
||||
warning()
|
||||
assert len(capture.records) == 1
|
||||
verbose_logger.setLevel(logging.WARNING)
|
||||
|
||||
async def request(name):
|
||||
session = session_id_var.set(name)
|
||||
trace = trace_id_var.set('trace-' + name)
|
||||
try:
|
||||
await asynchronous_warning()
|
||||
await machine_warning()
|
||||
synchronous_warning()
|
||||
assert session_id_var.get() == name
|
||||
assert trace_id_var.get() == 'trace-' + name
|
||||
finally:
|
||||
trace_id_var.reset(trace)
|
||||
session_id_var.reset(session)
|
||||
|
||||
async def concurrent():
|
||||
await asyncio.gather(request('first'), request('second'))
|
||||
|
||||
asyncio.run(concurrent())
|
||||
assert sorted((r.getMessage(), r.session_id, r.trace_id) for r in capture.records[1:]) == sorted(
|
||||
(message, name, 'trace-' + name)
|
||||
for name in ('first', 'second')
|
||||
for message in ('async warning', 'sync warning', 'machine started', 'machine warning', 'machine interrupted')
|
||||
)
|
||||
|
||||
verbose_logger.setLevel(logging.DEBUG)
|
||||
before_levels = len(capture.records)
|
||||
levels()
|
||||
assert [(r.getMessage(), r.levelno) for r in capture.records[before_levels:]] == [
|
||||
('trace', logging.DEBUG), ('debug', logging.DEBUG), ('info', logging.INFO),
|
||||
('warn', logging.WARNING), ('error', logging.ERROR),
|
||||
]
|
||||
|
||||
before = len(capture.records)
|
||||
litellm.ssl_ecdh_curve = 'logger-test-unsupported-curve'
|
||||
http_warning()
|
||||
http_warning()
|
||||
assert len(capture.records) == before + 1
|
||||
assert 'logger-test-unsupported-curve' in capture.records[-1].getMessage()
|
||||
assert capture.records[-1].pathname.endswith('http.rs')
|
||||
|
||||
verbose_logger.handlers = [Broken()]
|
||||
sys.unraisablehook = failures.append
|
||||
warning()
|
||||
assert len(failures) == 1
|
||||
assert str(failures[0].exc_value) == 'handler failed'
|
||||
try:
|
||||
synchronous_failure()
|
||||
except ValueError as error:
|
||||
assert str(error) == 'request failed'
|
||||
else:
|
||||
raise AssertionError('request failure was lost')
|
||||
assert len(failures) == 2
|
||||
finally:
|
||||
sys.unraisablehook = old_unraisable
|
||||
verbose_logger.handlers = old_handlers
|
||||
verbose_logger.setLevel(old_level)
|
||||
litellm.request_correlation_in_logs = old_correlation
|
||||
litellm.ssl_ecdh_curve = old_curve
|
||||
", Some(&locals), Some(&locals)).unwrap();
|
||||
});
|
||||
}
|
||||
|
|
@ -44,11 +44,6 @@ impl PythonSettings {
|
|||
pub(crate) fn snapshot(self, value: Bound<'_, PyAny>) -> Snapshot<'_> {
|
||||
Snapshot { group: self, value }
|
||||
}
|
||||
|
||||
pub(crate) fn warn(py: Python<'_>, message: &str) -> PyResult<()> {
|
||||
py.import(MODULE)?.getattr("warn")?.call1((message,))?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::audio_transcription::{
|
||||
Error, audio_transcription as run_audio_transcription, types::AudioTranscriptionRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
use crate::logger::{run_async, run_sync};
|
||||
use litellm_core::chat_completions::{
|
||||
Error, chat_completions as run_chat_completions, chat_completions_decline_reason,
|
||||
types::ChatCompletionsRequest,
|
||||
};
|
||||
use litellm_host_python::{from_py_argument, run_async, run_sync};
|
||||
use litellm_host_python::from_py_argument;
|
||||
use litellm_types::utils::ChatCompletionsResponse;
|
||||
use pyo3::prelude::*;
|
||||
use serde_json::{Map, Value};
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ fn run_messages(
|
|||
py,
|
||||
SURFACE,
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
messages_machine(),
|
||||
crate::logger::LoggedMachine::new(messages_machine()),
|
||||
MessagesRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -73,7 +73,7 @@ fn run_ocr(
|
|||
py,
|
||||
if asynchronous { ASYNC_SURFACE } else { SURFACE },
|
||||
PublicCall::capture(&request, &args, &kwargs)?,
|
||||
ocr_machine(client),
|
||||
crate::logger::LoggedMachine::new(ocr_machine(client)),
|
||||
OcrRouteHost::new(request.unbind()),
|
||||
asynchronous,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ impl ResponsesWebSocketConnection {
|
|||
) -> PyResult<Bound<'py, PyAny>> {
|
||||
let headers = marshal_headers(headers)?;
|
||||
let timeout = optional_timeout(timeout_seconds);
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
let inner = RustResponsesWebSocketConnection::connect_url(&url, &headers, timeout)
|
||||
.await
|
||||
.map_err(responses_error_to_pyerr)?;
|
||||
|
|
@ -35,7 +35,7 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn send_text<'py>(&self, py: Python<'py>, text: String) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner
|
||||
.send_text(text)
|
||||
.await
|
||||
|
|
@ -45,14 +45,14 @@ impl ResponsesWebSocketConnection {
|
|||
|
||||
fn recv_text<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.recv_text().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
||||
fn close<'py>(&self, py: Python<'py>) -> PyResult<Bound<'py, PyAny>> {
|
||||
let inner = self.inner.clone();
|
||||
litellm_host_python::run_async_value(py, async move {
|
||||
crate::logger::run_async_value(py, async move {
|
||||
inner.close().await.map_err(responses_error_to_pyerr)
|
||||
})
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,8 @@
|
|||
use crate::logger::run_async;
|
||||
use std::sync::Arc;
|
||||
use std::{num::NonZero, thread::available_parallelism};
|
||||
|
||||
use litellm_host_python::{enter_native, run_async};
|
||||
use litellm_host_python::enter_native;
|
||||
use litellm_token_counter::{
|
||||
CountableRequest, Error, InputTokenCount, TokenCounter as CoreTokenCounter,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ litellm-secrets-types.workspace = true
|
|||
litellm-core-utils.workspace = true
|
||||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
veil.workspace = true
|
||||
aws-sdk-kms = "1.120.0"
|
||||
aws-sdk-secretsmanager = "1.117.0"
|
||||
|
|
|
|||
|
|
@ -215,7 +215,7 @@ impl AwsSecretsManagerV2 {
|
|||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::warn!("secret created but replication failed");
|
||||
litellm_tracing::warn!("secret created but replication failed");
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ reqwest.workspace = true
|
|||
serde_json.workspace = true
|
||||
thiserror.workspace = true
|
||||
veil.workspace = true
|
||||
tracing = "0.1"
|
||||
litellm-tracing.workspace = true
|
||||
percent-encoding = "2.3"
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
|
||||
|
|
|
|||
|
|
@ -92,7 +92,7 @@ impl CyberArkSecretManager {
|
|||
.unwrap_or(true);
|
||||
let mut builder = reqwest::Client::builder();
|
||||
if !verify {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk SSL verification is disabled. This is insecure and should only be used for testing with self-signed certificates."
|
||||
);
|
||||
builder = builder.danger_accept_invalid_certs(true);
|
||||
|
|
@ -259,11 +259,13 @@ impl CyberArkSecretManager {
|
|||
.endpoint
|
||||
.join(&format!("policies/{}/policy/root", self.account));
|
||||
let Ok(policy_url) = policy_url else {
|
||||
tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
litellm_tracing::warn!("Could not build CyberArk policy endpoint");
|
||||
return;
|
||||
};
|
||||
let Ok(authorization) = self.authorization_header(context).await else {
|
||||
tracing::warn!("Could not authenticate while ensuring CyberArk variable exists");
|
||||
litellm_tracing::warn!(
|
||||
"Could not authenticate while ensuring CyberArk variable exists"
|
||||
);
|
||||
return;
|
||||
};
|
||||
let body = format!(
|
||||
|
|
@ -288,19 +290,19 @@ impl CyberArkSecretManager {
|
|||
reqwest::StatusCode::CONFLICT | reqwest::StatusCode::UNPROCESSABLE_ENTITY
|
||||
) =>
|
||||
{
|
||||
tracing::debug!(
|
||||
litellm_tracing::debug!(
|
||||
"CyberArk variable policy already exists or conflicts: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Ok(response) => {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"Could not ensure CyberArk variable exists: {}",
|
||||
response.status()
|
||||
);
|
||||
}
|
||||
Err(error) => {
|
||||
tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
litellm_tracing::warn!("Error ensuring CyberArk variable exists: {error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -324,7 +326,7 @@ impl CyberArkSecretManager {
|
|||
_recovery_window_in_days: Option<u32>,
|
||||
_context: &SecretOperationContext,
|
||||
) -> Result<DeleteOutcome, Error> {
|
||||
tracing::warn!(
|
||||
litellm_tracing::warn!(
|
||||
"CyberArk Conjur does not support direct secret deletion. Secrets must be removed through policy updates."
|
||||
);
|
||||
self.secrets.invalidate(name).await;
|
||||
|
|
|
|||
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
17
litellm-rust/crates/tracing/Cargo.toml
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
[package]
|
||||
name = "litellm-tracing"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
fancy-regex.workspace = true
|
||||
percent-encoding.workspace = true
|
||||
serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
tracing-subscriber = { version = "0.3", default-features = false, features = ["registry", "std"] }
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
26
litellm-rust/crates/tracing/README.md
Normal file
26
litellm-rust/crates/tracing/README.md
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
# Native diagnostic tracing
|
||||
|
||||
`litellm-tracing` connects standard `tracing` events to a host-provided `Sink`. It has no Python dependency and does not install a global subscriber
|
||||
|
||||
Use the exported `debug!`, `info!`, `warn!`, and `error!` macros in native code. A host creates a `Logger` with its sink, uses `scope` for synchronous operations, and wraps futures with `instrument`. Instrument spawned futures explicitly because thread-local subscribers do not automatically follow spawned work
|
||||
|
||||
Bindings implement `litellm_tracing::Sink` to connect events to their host runtime:
|
||||
|
||||
```rust
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
```
|
||||
|
||||
Pass the implementation to `litellm_tracing::Logger::new(sink)`, then call `logger.scope(|| litellm_tracing::info!(attempt = 1, "request started"))`. The sink owns host access, level mapping, correlation capture, and delivery failures. `enabled` runs before event fields are evaluated or formatted. `emit` borrows a record; an adapter that queues delivery must copy the data it needs into an owned value
|
||||
|
||||
Records retain event metadata, the message, and typed event fields. Sink filtering runs for each event so runtime level changes take effect. Logging from inside a sink is suppressed to prevent recursion
|
||||
|
||||
The Python bridge scopes native execution to a sink that uses LiteLLM's existing Python logger. It preserves request correlation, redacts before delivering to handlers, maps Rust trace events to Python debug, and reports handler failures through `sys.unraisablehook`. It accepts LiteLLM targets only, keeping dependency wire diagnostics out of the application logger
|
||||
|
||||
Python consumers continue using `litellm._logging` and its existing loggers, filters, formatters, and context setters. Catalog dispatch selects the processing backend for both Python and native diagnostics. The pure `Processor` takes explicit settings and never emits events
|
||||
|
||||
A future Node bridge can implement the same sink with runtime-specific delivery and expose the same processor through N-API. Node callback scheduling, queue limits, and shutdown belong in that bridge; this crate has no interpreter handles or output queue
|
||||
|
||||
This is diagnostic logging. Request lifecycle hooks and `CustomLogger` dispatch remain separate
|
||||
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
146
litellm-rust/crates/tracing/src/lib.rs
Normal file
|
|
@ -0,0 +1,146 @@
|
|||
use std::{
|
||||
cell::Cell,
|
||||
fmt,
|
||||
future::{Future, poll_fn},
|
||||
pin::pin,
|
||||
};
|
||||
|
||||
use serde_json::{Map, Value};
|
||||
use tracing::{
|
||||
Dispatch, Event, Subscriber,
|
||||
field::{Field, Visit},
|
||||
subscriber::Interest,
|
||||
};
|
||||
use tracing_subscriber::{Layer, Registry, layer::Context, prelude::*};
|
||||
|
||||
mod processing;
|
||||
mod redaction;
|
||||
|
||||
pub use processing::{DiagnosticInput, DiagnosticOutput, Policy, Processor};
|
||||
pub use redaction::{REDACTED, SecretRedactor};
|
||||
pub use tracing::{Level, Metadata, debug, error, info, trace, warn};
|
||||
|
||||
pub trait Sink: Send + Sync + 'static {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool;
|
||||
fn emit(&self, record: &Record);
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Record {
|
||||
pub metadata: &'static Metadata<'static>,
|
||||
pub message: String,
|
||||
pub fields: Map<String, Value>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct Logger {
|
||||
dispatch: Dispatch,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
pub fn new(sink: impl Sink) -> Self {
|
||||
Self {
|
||||
dispatch: Dispatch::new(Registry::default().with(Output(sink))),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn scope<T>(&self, operation: impl FnOnce() -> T) -> T {
|
||||
if EMITTING.get() {
|
||||
return operation();
|
||||
}
|
||||
tracing::dispatcher::with_default(&self.dispatch, operation)
|
||||
}
|
||||
|
||||
pub fn instrument<F: Future>(&self, future: F) -> impl Future<Output = F::Output> + use<F> {
|
||||
let logger = self.clone();
|
||||
async move {
|
||||
let mut future = pin!(future);
|
||||
poll_fn(|context| logger.scope(|| future.as_mut().poll(context))).await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
thread_local! {
|
||||
static EMITTING: Cell<bool> = const { Cell::new(false) };
|
||||
}
|
||||
|
||||
struct Emitting;
|
||||
|
||||
impl Emitting {
|
||||
fn enter() -> Option<Self> {
|
||||
EMITTING.with(|active| (!active.replace(true)).then_some(Self))
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Emitting {
|
||||
fn drop(&mut self) {
|
||||
EMITTING.set(false);
|
||||
}
|
||||
}
|
||||
|
||||
struct Output<S>(S);
|
||||
|
||||
impl<S: Sink, R: Subscriber> Layer<R> for Output<S> {
|
||||
fn register_callsite(&self, _: &'static Metadata<'static>) -> Interest {
|
||||
Interest::sometimes()
|
||||
}
|
||||
|
||||
fn enabled(&self, metadata: &Metadata<'_>, _: Context<'_, R>) -> bool {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return false;
|
||||
};
|
||||
self.0.enabled(metadata)
|
||||
}
|
||||
|
||||
fn on_event(&self, event: &Event<'_>, _: Context<'_, R>) {
|
||||
let Some(_guard) = Emitting::enter() else {
|
||||
return;
|
||||
};
|
||||
let mut record = Record {
|
||||
metadata: event.metadata(),
|
||||
message: String::new(),
|
||||
fields: Map::new(),
|
||||
};
|
||||
event.record(&mut record);
|
||||
self.0.emit(&record);
|
||||
}
|
||||
}
|
||||
|
||||
impl Record {
|
||||
fn field(&mut self, field: &Field, value: Value) {
|
||||
if field.name() == "message" {
|
||||
self.message = match value {
|
||||
Value::String(message) => message,
|
||||
value => value.to_string(),
|
||||
};
|
||||
} else {
|
||||
self.fields.insert(field.name().to_owned(), value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Visit for Record {
|
||||
fn record_debug(&mut self, field: &Field, value: &dyn fmt::Debug) {
|
||||
self.field(field, format!("{value:?}").into());
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &Field, value: &str) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_bool(&mut self, field: &Field, value: bool) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_i64(&mut self, field: &Field, value: i64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_u64(&mut self, field: &Field, value: u64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
|
||||
fn record_f64(&mut self, field: &Field, value: f64) {
|
||||
self.field(field, value.into());
|
||||
}
|
||||
}
|
||||
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
333
litellm-rust/crates/tracing/src/processing.rs
Normal file
|
|
@ -0,0 +1,333 @@
|
|||
use fancy_regex::Result;
|
||||
use percent_encoding::percent_decode_str;
|
||||
|
||||
use crate::{REDACTED, SecretRedactor};
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct Policy {
|
||||
pub redact: bool,
|
||||
pub base64_limit: i64,
|
||||
pub text_limit: i64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct DiagnosticInput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<(Option<String>, String)>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct DiagnosticOutput {
|
||||
pub message: String,
|
||||
pub exception: Option<String>,
|
||||
pub stack: Option<String>,
|
||||
pub leaves: Vec<String>,
|
||||
pub changed: bool,
|
||||
}
|
||||
|
||||
pub struct Processor {
|
||||
redactor: SecretRedactor,
|
||||
}
|
||||
|
||||
impl Processor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
Self {
|
||||
redactor: SecretRedactor::new(minimum_custom_key_length),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact_text(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact(text)
|
||||
}
|
||||
|
||||
pub fn redact_structured_text(&self, key: Option<&str>, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_structured(key, text)
|
||||
}
|
||||
|
||||
pub fn redact_client_message(&self, text: &str) -> Result<String> {
|
||||
self.redactor.try_redact_internal(text)
|
||||
}
|
||||
|
||||
pub fn process_diagnostic(
|
||||
&self,
|
||||
input: &DiagnosticInput,
|
||||
policy: Policy,
|
||||
) -> Result<DiagnosticOutput> {
|
||||
let message = self.process_text(&input.message, policy)?;
|
||||
let exception = input
|
||||
.exception
|
||||
.as_deref()
|
||||
.map(|text| self.process_text(text, policy))
|
||||
.transpose()?;
|
||||
let stack = input
|
||||
.stack
|
||||
.as_deref()
|
||||
.map(|text| {
|
||||
if policy.redact {
|
||||
self.redact_text(text)
|
||||
} else {
|
||||
Ok(text.to_owned())
|
||||
}
|
||||
})
|
||||
.transpose()?;
|
||||
let leaves = input
|
||||
.leaves
|
||||
.iter()
|
||||
.map(|(key, text)| {
|
||||
if policy.redact {
|
||||
self.redact_structured_text(key.as_deref(), text)
|
||||
} else {
|
||||
Ok(text.clone())
|
||||
}
|
||||
})
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let changed = message != input.message
|
||||
|| exception != input.exception
|
||||
|| stack != input.stack
|
||||
|| leaves
|
||||
.iter()
|
||||
.zip(&input.leaves)
|
||||
.any(|(processed, (_, original))| processed != original);
|
||||
Ok(DiagnosticOutput {
|
||||
message,
|
||||
exception,
|
||||
stack,
|
||||
leaves,
|
||||
changed,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn scrub_access_arguments(&self, arguments: &[String]) -> Result<Vec<String>> {
|
||||
arguments
|
||||
.iter()
|
||||
.map(|argument| self.scrub_access_arg(argument))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn process_text(&self, text: &str, policy: Policy) -> Result<String> {
|
||||
let collapsed = if policy.base64_limit > 0 {
|
||||
collapse_base64(text, policy.base64_limit as usize)
|
||||
} else {
|
||||
text.to_owned()
|
||||
};
|
||||
let redacted = if policy.redact {
|
||||
self.redact_text(&collapsed)?
|
||||
} else {
|
||||
collapsed
|
||||
};
|
||||
Ok(
|
||||
if policy.text_limit > 0 && redacted.chars().count() > policy.text_limit as usize {
|
||||
truncate_text(&redacted, policy.text_limit as usize)
|
||||
} else {
|
||||
redacted
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn scrub_access_arg(&self, value: &str) -> Result<String> {
|
||||
let length = value.chars().count();
|
||||
let scanned = if length <= 512 {
|
||||
value
|
||||
} else {
|
||||
let head = &value[..char_offset(value, 512)];
|
||||
if head.contains('?') {
|
||||
&head[..head.rfind(['?', '&']).unwrap_or(0)]
|
||||
} else {
|
||||
head
|
||||
}
|
||||
};
|
||||
let scrubbed = self.redact_text(scanned)?;
|
||||
let (path, query) = scrubbed
|
||||
.split_once('?')
|
||||
.map_or((scrubbed.as_str(), None), |(path, query)| {
|
||||
(path, Some(query))
|
||||
});
|
||||
let safe = if self.hides_encoded_credential(path)? {
|
||||
REDACTED.to_owned()
|
||||
} else if query.is_some() && self.hides_encoded_credential(&scrubbed)? {
|
||||
format!("{path}?{REDACTED}")
|
||||
} else {
|
||||
scrubbed
|
||||
};
|
||||
Ok(if length > 512 {
|
||||
format!(
|
||||
"{safe}... ({} more chars truncated) ...",
|
||||
length - scanned.chars().count()
|
||||
)
|
||||
} else {
|
||||
safe
|
||||
})
|
||||
}
|
||||
|
||||
fn hides_encoded_credential(&self, value: &str) -> Result<bool> {
|
||||
if !value.as_bytes().contains(&b'%') {
|
||||
return Ok(false);
|
||||
}
|
||||
let decoded = percent_decode_str(value).decode_utf8_lossy();
|
||||
Ok(self.redact_text(&decoded)? != decoded)
|
||||
}
|
||||
}
|
||||
|
||||
fn char_offset(text: &str, count: usize) -> usize {
|
||||
text.char_indices()
|
||||
.nth(count)
|
||||
.map_or(text.len(), |(index, _)| index)
|
||||
}
|
||||
|
||||
fn marker(skipped_chars: usize) -> String {
|
||||
format!(
|
||||
"... (litellm_truncated skipped {skipped_chars} chars. Truncation is a stdout logging safeguard. Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.) and at DEBUG level. To increase the truncation limit, set `MAX_STRING_LENGTH_STDOUT_LOG` in your env.) ..."
|
||||
)
|
||||
}
|
||||
|
||||
fn truncate_text(text: &str, limit: usize) -> String {
|
||||
let length = text.chars().count();
|
||||
let kept = limit.saturating_sub(marker(length).len());
|
||||
if kept == 0 {
|
||||
return text[..char_offset(text, limit)].to_owned();
|
||||
}
|
||||
let head = kept / 2;
|
||||
let tail = kept - head;
|
||||
format!(
|
||||
"{}{}{}",
|
||||
&text[..char_offset(text, head)],
|
||||
marker(length - kept),
|
||||
&text[char_offset(text, length - tail)..]
|
||||
)
|
||||
}
|
||||
|
||||
fn base64_byte(byte: u8) -> bool {
|
||||
byte.is_ascii_alphanumeric() || byte == b'+' || byte == b'/'
|
||||
}
|
||||
|
||||
fn looks_like_base64(run: &str) -> bool {
|
||||
let unpadded = run.trim_end_matches('=');
|
||||
let lower_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte));
|
||||
let upper_hex = unpadded
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'A'..=b'F').contains(&byte));
|
||||
let repeated = unpadded.bytes().all(|byte| byte == unpadded.as_bytes()[0]);
|
||||
(!lower_hex && !upper_hex) || repeated
|
||||
}
|
||||
|
||||
fn base64_size(chars: usize) -> String {
|
||||
let bytes = chars as f64 * 3.0 / 4.0;
|
||||
if bytes >= 1024.0 * 1024.0 {
|
||||
return format!("{:.2}MB", bytes / (1024.0 * 1024.0));
|
||||
}
|
||||
if bytes >= 1024.0 {
|
||||
return format!("{:.1}KB", bytes / 1024.0);
|
||||
}
|
||||
format!("{}B", bytes as usize)
|
||||
}
|
||||
|
||||
fn collapse_base64(text: &str, limit: usize) -> String {
|
||||
let bytes = text.as_bytes();
|
||||
let mut position = 0;
|
||||
let mut previous = 0;
|
||||
let mut output = String::new();
|
||||
while position < bytes.len() {
|
||||
if !base64_byte(bytes[position]) || (position > 0 && base64_byte(bytes[position - 1])) {
|
||||
position += 1;
|
||||
continue;
|
||||
}
|
||||
let start = position;
|
||||
while position < bytes.len() && base64_byte(bytes[position]) {
|
||||
position += 1;
|
||||
}
|
||||
let run_end = position;
|
||||
while position < bytes.len() && position - run_end < 2 && bytes[position] == b'=' {
|
||||
position += 1;
|
||||
}
|
||||
let run = &text[start..position];
|
||||
if run_end - start > limit && looks_like_base64(run) {
|
||||
output.push_str(&text[previous..start]);
|
||||
output.push_str(&format!(
|
||||
"[base64_data truncated: {}]",
|
||||
base64_size(run.len())
|
||||
));
|
||||
previous = position;
|
||||
}
|
||||
}
|
||||
output.push_str(&text[previous..]);
|
||||
output
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn redaction_precedes_the_text_bound_and_preserves_unicode_character_limits() {
|
||||
let processor = Processor::new(16);
|
||||
let secret = format!("sk-{}", "q".repeat(48));
|
||||
let text = format!("{}{}{}", "é".repeat(110), secret, "界".repeat(1000));
|
||||
let input = DiagnosticInput {
|
||||
message: text,
|
||||
exception: None,
|
||||
stack: None,
|
||||
leaves: vec![],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 0,
|
||||
text_limit: 500,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.chars().count() <= 500);
|
||||
assert!(!output.message.contains("sk-qq"));
|
||||
assert!(output.changed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn base64_collapse_applies_to_debug_and_exceptions_without_touching_hex() {
|
||||
let processor = Processor::new(16);
|
||||
let input = DiagnosticInput {
|
||||
message: format!("image={} digest={}", "Q".repeat(100), "a1".repeat(50)),
|
||||
exception: Some(format!("upload failed: {}", "Q".repeat(100))),
|
||||
stack: Some("api_key=secret123".to_owned()),
|
||||
leaves: vec![(Some("api_key".to_owned()), "secret123".to_owned())],
|
||||
};
|
||||
let output = processor
|
||||
.process_diagnostic(
|
||||
&input,
|
||||
Policy {
|
||||
redact: true,
|
||||
base64_limit: 20,
|
||||
text_limit: 0,
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
assert!(output.message.contains("[base64_data truncated: 75B]"));
|
||||
assert!(output.message.contains(&"a1".repeat(50)));
|
||||
assert!(
|
||||
output
|
||||
.exception
|
||||
.unwrap()
|
||||
.contains("[base64_data truncated: 75B]")
|
||||
);
|
||||
assert_eq!(output.stack.as_deref(), Some(REDACTED));
|
||||
assert_eq!(output.leaves, vec![REDACTED]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn access_arguments_keep_encoded_paths_and_drop_decoded_credentials() {
|
||||
let processor = Processor::new(16);
|
||||
let arguments = vec![
|
||||
"/v1/models?filter=gpt%2D4o&page=2".to_owned(),
|
||||
"/v1/models?k%65y=sk%2Dabcdefghijklmnopqrstuvwxyz&page=2".to_owned(),
|
||||
];
|
||||
assert_eq!(
|
||||
processor.scrub_access_arguments(&arguments).unwrap(),
|
||||
vec![arguments[0].clone(), "/v1/models?REDACTED".to_owned()]
|
||||
);
|
||||
}
|
||||
}
|
||||
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
140
litellm-rust/crates/tracing/src/redaction.rs
Normal file
|
|
@ -0,0 +1,140 @@
|
|||
use fancy_regex::{NoExpand, Regex};
|
||||
|
||||
pub const REDACTED: &str = "REDACTED";
|
||||
|
||||
#[cfg(test)]
|
||||
const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16;
|
||||
|
||||
fn secret_patterns(minimum_custom_key_length: usize) -> String {
|
||||
let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len());
|
||||
[
|
||||
r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
|
||||
r"\bya29\.[A-Za-z0-9_.~+/-]+",
|
||||
r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#,
|
||||
r"(?:AKIA|ASIA)[0-9A-Z]{16}",
|
||||
r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
|
||||
r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
|
||||
&format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"),
|
||||
r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#,
|
||||
r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#,
|
||||
r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r"x-ak-[A-Za-z0-9\-_]{20,}",
|
||||
r"AIza[0-9A-Za-z\-_]{35}",
|
||||
r#"(?<=[?&])key=[^\s&'"]{8,}"#,
|
||||
r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#,
|
||||
r"dapi[0-9a-f]{32}",
|
||||
r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#,
|
||||
concat!(
|
||||
r"(?:master_key|xai_key|database_url|db_url|connection_string|",
|
||||
r"aws_secret_access_key|aws_session_token|aws_access_key_id|s3_secret_access_key|s3_access_key_id|",
|
||||
r"signing_key|encryption_key|",
|
||||
r"auth_token|access_token|refresh_token|",
|
||||
r"slack_webhook_url|webhook_url|",
|
||||
r"database_connection_string|",
|
||||
r"huggingface_token|jwt_secret)",
|
||||
r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#,
|
||||
),
|
||||
r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
|
||||
r"(?<=[?&])sig=[A-Za-z0-9%+/=]+",
|
||||
r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#,
|
||||
]
|
||||
.join("|")
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct SecretRedactor {
|
||||
pattern: Regex,
|
||||
internal_pattern: Regex,
|
||||
}
|
||||
|
||||
impl SecretRedactor {
|
||||
pub fn new(minimum_custom_key_length: usize) -> Self {
|
||||
let pattern = Regex::new(&format!(
|
||||
"(?i){}",
|
||||
secret_patterns(minimum_custom_key_length)
|
||||
))
|
||||
.expect("secret redaction patterns compile");
|
||||
let internal_pattern = Regex::new(concat!(
|
||||
r#"(?i)/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'"\)\]}>,]+|"#,
|
||||
r#"[A-Za-z]:\\[^\s'"\)\]}>,]+|"#,
|
||||
r"\b(?:10(?:\.\d{1,3}){3}|172\.(?:1[6-9]|2\d|3[01])(?:\.\d{1,3}){2}|",
|
||||
r"192\.168(?:\.\d{1,3}){2}|127(?:\.\d{1,3}){3})\b|",
|
||||
r"\b[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.(?:internal|local|corp|lan|intra|private)\b",
|
||||
))
|
||||
.expect("internal detail patterns compile");
|
||||
Self {
|
||||
pattern,
|
||||
internal_pattern,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn redact(&self, value: &str) -> String {
|
||||
self.try_redact(value)
|
||||
.unwrap_or_else(|_| REDACTED.to_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
self.pattern
|
||||
.try_replacen(value, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
|
||||
pub fn try_redact_structured(
|
||||
&self,
|
||||
key: Option<&str>,
|
||||
value: &str,
|
||||
) -> fancy_regex::Result<String> {
|
||||
let scrubbed = self.try_redact(value)?;
|
||||
if scrubbed != value || key.is_none() {
|
||||
return Ok(scrubbed);
|
||||
}
|
||||
let rendered = format!("'{}': '{value}'", key.unwrap_or_default());
|
||||
Ok(if self.try_redact(&rendered)? != rendered {
|
||||
REDACTED.to_owned()
|
||||
} else {
|
||||
value.to_owned()
|
||||
})
|
||||
}
|
||||
|
||||
pub fn try_redact_internal(&self, value: &str) -> fancy_regex::Result<String> {
|
||||
let without_traceback = value
|
||||
.split_once("Traceback (most recent call last):")
|
||||
.map_or(value, |(prefix, _)| prefix.trim_end());
|
||||
self.internal_pattern
|
||||
.try_replacen(&self.try_redact(without_traceback)?, 0, NoExpand(REDACTED))
|
||||
.map(|value| value.into_owned())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")]
|
||||
#[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")]
|
||||
#[case::short_sk_key_is_kept("sk-abc", "sk-abc")]
|
||||
#[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")]
|
||||
#[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")]
|
||||
#[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")]
|
||||
#[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")]
|
||||
#[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")]
|
||||
#[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")]
|
||||
#[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)]
|
||||
fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) {
|
||||
assert_eq!(
|
||||
SecretRedactor::new(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH).redact(input),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sk_threshold_follows_the_minimum_custom_key_length() {
|
||||
let redactor = SecretRedactor::new(8);
|
||||
assert_eq!(redactor.redact("sk-abcde"), REDACTED);
|
||||
assert_eq!(redactor.redact("sk-abcd"), "sk-abcd");
|
||||
}
|
||||
}
|
||||
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
122
litellm-rust/crates/tracing/tests/logging.rs
Normal file
|
|
@ -0,0 +1,122 @@
|
|||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc,
|
||||
};
|
||||
|
||||
use litellm_tracing::{Level, Logger, Metadata, Record, Sink, info, warn};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
struct Output {
|
||||
enabled: Arc<AtomicBool>,
|
||||
sender: mpsc::Sender<(String, Value, Level, &'static str, Option<u32>)>,
|
||||
}
|
||||
|
||||
impl Sink for Output {
|
||||
fn enabled(&self, _: &Metadata<'_>) -> bool {
|
||||
self.enabled.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn emit(&self, record: &Record) {
|
||||
self.sender
|
||||
.send((
|
||||
record.message.clone(),
|
||||
Value::Object(record.fields.clone()),
|
||||
*record.metadata.level(),
|
||||
record.metadata.target(),
|
||||
record.metadata.line(),
|
||||
))
|
||||
.unwrap();
|
||||
Logger::default().scope(|| warn!("a sink must not recursively emit"));
|
||||
}
|
||||
}
|
||||
|
||||
fn emit() {
|
||||
warn!(
|
||||
attempt = 3_u64,
|
||||
elapsed = 1.5,
|
||||
retry = true,
|
||||
reason = "timeout",
|
||||
"retry {}",
|
||||
3
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_preserve_fields_metadata_and_dynamic_filtering_without_recursion() {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let enabled = Arc::new(AtomicBool::new(false));
|
||||
let logger = Logger::new(Output {
|
||||
enabled: enabled.clone(),
|
||||
sender,
|
||||
});
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
enabled.store(true, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
let (message, fields, level, target, line) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "retry 3");
|
||||
assert_eq!(
|
||||
fields,
|
||||
json!({"attempt": 3, "elapsed": 1.5, "retry": true, "reason": "timeout"})
|
||||
);
|
||||
assert_eq!(level, Level::WARN);
|
||||
assert_eq!(target, module_path!());
|
||||
assert!(line.is_some());
|
||||
enabled.store(false, Ordering::Relaxed);
|
||||
logger.scope(emit);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn concurrent_futures_keep_their_sinks_across_suspension_and_spawn() {
|
||||
let tasks = (0..2)
|
||||
.map(|id| {
|
||||
let (sender, receiver) = mpsc::channel();
|
||||
let logger = Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
});
|
||||
let task = tokio::spawn(logger.instrument(async move {
|
||||
tokio::task::yield_now().await;
|
||||
info!(id, "worker");
|
||||
}));
|
||||
(id, task, receiver)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
for (id, task, receiver) in tasks {
|
||||
task.await.unwrap();
|
||||
let (message, fields, level, _, _) = receiver.try_recv().unwrap();
|
||||
assert_eq!(message, "worker");
|
||||
assert_eq!(fields, json!({"id": id}));
|
||||
assert_eq!(level, Level::INFO);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nested_scopes_restore_the_previous_sink() {
|
||||
let (outer_sender, outer) = mpsc::channel();
|
||||
let (inner_sender, inner) = mpsc::channel();
|
||||
let logger = |sender| {
|
||||
Logger::new(Output {
|
||||
enabled: Arc::new(AtomicBool::new(true)),
|
||||
sender,
|
||||
})
|
||||
};
|
||||
let outside = logger(outer_sender);
|
||||
let inside = logger(inner_sender);
|
||||
outside.scope(|| {
|
||||
info!("before");
|
||||
inside.scope(|| info!("inside"));
|
||||
info!("after");
|
||||
});
|
||||
assert_eq!(
|
||||
outer.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["before", "after"]
|
||||
);
|
||||
assert_eq!(
|
||||
inner.try_iter().map(|event| event.0).collect::<Vec<_>>(),
|
||||
["inside"]
|
||||
);
|
||||
}
|
||||
|
|
@ -1,11 +1,12 @@
|
|||
import ast
|
||||
import contextvars
|
||||
import functools
|
||||
import itertools
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from logging import Formatter
|
||||
from typing import Any, Final, TextIO
|
||||
|
|
@ -22,10 +23,12 @@ from litellm.litellm_core_utils.env_utils import get_env_int
|
|||
from litellm.litellm_core_utils.safe_json_dumps import UNSERIALIZABLE_OBJECT, safe_dumps, safe_json_structure
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
from litellm.litellm_core_utils.secret_redaction import (
|
||||
_python_redact_string,
|
||||
_python_redact_structured_value,
|
||||
redact_internal_details,
|
||||
redact_string,
|
||||
redact_structured_value,
|
||||
)
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
set_verbose = False
|
||||
|
||||
|
|
@ -49,7 +52,7 @@ def _sanitize_correlation_id(value: str) -> str:
|
|||
pass through credential redaction.
|
||||
"""
|
||||
stripped: Final = "".join(ch for ch in value if ch.isprintable())
|
||||
return _redact_string(stripped[:_MAX_CORRELATION_ID_LENGTH])
|
||||
return _redact_string(stripped)[:_MAX_CORRELATION_ID_LENGTH]
|
||||
|
||||
|
||||
def set_session_id(session_id: str) -> "contextvars.Token[str]":
|
||||
|
|
@ -74,12 +77,6 @@ def _redact_string(value: str) -> str:
|
|||
return redact_string(value)
|
||||
|
||||
|
||||
def _redact_structured_value(key: str | None, value: str) -> str:
|
||||
if not _ENABLE_SECRET_REDACTION:
|
||||
return value
|
||||
return redact_structured_value(key, value)
|
||||
|
||||
|
||||
_REDACTED_RECORD_ATTR: Final = "litellm_redacted"
|
||||
_REDACTED_STAMP: Final = object()
|
||||
_UNREDACTED_SCALAR_TYPES: Final = (bool, int, float, type(None))
|
||||
|
|
@ -103,14 +100,6 @@ def _plain_text(value: object) -> str:
|
|||
return UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _redact_extra_value(key: str, value: object) -> object:
|
||||
try:
|
||||
scrubbed: Final = safe_json_structure(value, value_transform=_redact_structured_value, key=key)
|
||||
except Exception:
|
||||
return _redact_string(_plain_text(value))
|
||||
return value if _scrubbing_changed_nothing(scrubbed, value) else scrubbed
|
||||
|
||||
|
||||
def redact_secrets(value: str) -> str:
|
||||
"""Public API: redact known secret/credential patterns from an arbitrary string.
|
||||
|
||||
|
|
@ -150,7 +139,7 @@ def _substituted_color_message(record: logging.LogRecord) -> str | None:
|
|||
return None
|
||||
try:
|
||||
return color_message % record.args
|
||||
except TypeError:
|
||||
except Exception:
|
||||
return color_message
|
||||
|
||||
|
||||
|
|
@ -162,42 +151,7 @@ class SecretRedactionFilter(logging.Filter):
|
|||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
if not _ENABLE_SECRET_REDACTION or _is_redacted(record):
|
||||
return True
|
||||
|
||||
# Runs before args are cleared, and before the extra-field loop below
|
||||
# that redacts the substituted result.
|
||||
substituted_color_message: Final = _substituted_color_message(record)
|
||||
if substituted_color_message is not None:
|
||||
record.color_message = substituted_color_message # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
try:
|
||||
record.msg = _redact_string(record.getMessage())
|
||||
record.args = None
|
||||
except Exception:
|
||||
if isinstance(record.msg, str):
|
||||
record.msg = _redact_string(record.msg)
|
||||
|
||||
# Redact exception tracebacks
|
||||
if record.exc_info and record.exc_info[1] is not None:
|
||||
try:
|
||||
record.exc_text = _redact_string(record.exc_text or self._formatter.formatException(record.exc_info))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if isinstance(record.stack_info, str):
|
||||
record.stack_info = _redact_string(record.stack_info) # rebind-ok: a Filter scrubs records in place
|
||||
|
||||
# Redact extra fields passed via logger.debug("msg", extra={...})
|
||||
record_items: Final[Sequence[tuple[str, object]]] = list(record.__dict__.items())
|
||||
for key, value in record_items:
|
||||
if key in _STANDARD_RECORD_ATTRS:
|
||||
continue
|
||||
if isinstance(value, str):
|
||||
setattr(record, key, _redact_structured_value(key, value))
|
||||
elif not isinstance(value, _UNREDACTED_SCALAR_TYPES):
|
||||
setattr(record, key, _redact_extra_value(key, value))
|
||||
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
return _process_record(record, base64_limit=0, text_limit=0, redact=True)
|
||||
|
||||
|
||||
_secret_filter: Final = SecretRedactionFilter()
|
||||
|
|
@ -211,7 +165,7 @@ _REDACTION_PLACEHOLDER: Final = "REDACTED"
|
|||
def _hides_a_credential(value: str) -> bool:
|
||||
"""Whether *value* only looks clean until it is percent-decoded."""
|
||||
decoded: Final = unquote(value)
|
||||
return _redact_string(decoded) != decoded
|
||||
return _python_redact_string(decoded) != decoded
|
||||
|
||||
|
||||
def _drop_encoded_credential(scrubbed: str) -> str:
|
||||
|
|
@ -239,10 +193,10 @@ def _scrub_access_arg(value: str) -> str:
|
|||
pattern and would then be logged raw.
|
||||
"""
|
||||
if len(value) <= _MAX_SCRUBBED_ACCESS_ARG:
|
||||
return _drop_encoded_credential(_redact_string(value))
|
||||
return _drop_encoded_credential(_python_redact_string(value))
|
||||
head: Final = value[:_MAX_SCRUBBED_ACCESS_ARG]
|
||||
kept: Final = head[: max(head.rfind("?"), head.rfind("&"))] if "?" in head else head
|
||||
scrubbed: Final = _drop_encoded_credential(_redact_string(kept))
|
||||
scrubbed: Final = _drop_encoded_credential(_python_redact_string(kept))
|
||||
return f"{scrubbed}... ({len(value) - len(kept)} more chars truncated) ..."
|
||||
|
||||
|
||||
|
|
@ -258,8 +212,17 @@ class AccessLogRedactionFilter(logging.Filter):
|
|||
if not _ENABLE_SECRET_REDACTION:
|
||||
return True
|
||||
if isinstance(record.args, tuple) and record.args:
|
||||
strings: Final = tuple(arg for arg in record.args if isinstance(arg, str))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.scrub_access_arguments(strings),
|
||||
lambda: tuple(_scrub_access_arg(arg) for arg in strings),
|
||||
)
|
||||
scrubbed: Final = (
|
||||
candidate if len(candidate) == len(strings) else tuple(_scrub_access_arg(arg) for arg in strings)
|
||||
)
|
||||
values: Final = iter(scrubbed)
|
||||
record.args = tuple( # rebind-ok: a Filter scrubs records in place
|
||||
_scrub_access_arg(arg) if isinstance(arg, str) else arg for arg in record.args
|
||||
next(values) if isinstance(arg, str) else arg for arg in record.args
|
||||
)
|
||||
return True
|
||||
# No positional args means everything is in msg, where collapsing is correct.
|
||||
|
|
@ -365,6 +328,185 @@ def _collapse_base64_runs(text: str, limit: int) -> str:
|
|||
return _base64_run_pattern(limit + 1).sub(_replace_base64_run, text)
|
||||
|
||||
|
||||
def _extra_structure(key: str, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
try:
|
||||
return safe_json_structure(value, key=key)
|
||||
except Exception:
|
||||
return _plain_text(value)
|
||||
|
||||
|
||||
def _string_leaves(key: str | None, value: object) -> Iterator[tuple[str | None, str]]:
|
||||
if isinstance(value, str):
|
||||
yield key, value
|
||||
elif isinstance(value, dict):
|
||||
yield from itertools.chain.from_iterable(
|
||||
_string_leaves(child_key, child) for child_key, child in value.items() if isinstance(child_key, str)
|
||||
)
|
||||
elif isinstance(value, (list, tuple)):
|
||||
yield from itertools.chain.from_iterable(_string_leaves(key, child) for child in value)
|
||||
|
||||
|
||||
def _replace_string_leaves(value: object, values: Iterator[str]) -> object:
|
||||
if isinstance(value, str):
|
||||
return next(values)
|
||||
if isinstance(value, dict):
|
||||
return { # mutable-ok: LogRecord extras must keep JSON dict shape for handlers
|
||||
key: _replace_string_leaves(child, values) for key, child in value.items()
|
||||
}
|
||||
if isinstance(value, list):
|
||||
return [ # mutable-ok: LogRecord extras must keep JSON list shape for handlers
|
||||
_replace_string_leaves(child, values) for child in value
|
||||
]
|
||||
if isinstance(value, tuple):
|
||||
return tuple(_replace_string_leaves(child, values) for child in value)
|
||||
return value
|
||||
|
||||
|
||||
def _sort_processed_sets(original: object, processed: object) -> object:
|
||||
if isinstance(original, set) and isinstance(processed, list):
|
||||
return sorted(processed)
|
||||
if isinstance(original, dict) and isinstance(processed, dict):
|
||||
return { # mutable-ok: sorting nested sets must preserve the surrounding JSON dict
|
||||
key: _sort_processed_sets(original.get(key), value) for key, value in processed.items()
|
||||
}
|
||||
if isinstance(original, list) and isinstance(processed, list):
|
||||
return [ # mutable-ok: sorting nested sets must preserve the surrounding JSON list
|
||||
_sort_processed_sets(before, after) for before, after in zip(original, processed)
|
||||
]
|
||||
if isinstance(original, tuple) and isinstance(processed, tuple):
|
||||
return tuple(_sort_processed_sets(before, after) for before, after in zip(original, processed))
|
||||
return processed
|
||||
|
||||
|
||||
def _python_process_diagnostic(
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
redact: bool,
|
||||
base64_limit: int,
|
||||
text_limit: int,
|
||||
) -> tuple[str, str | None, str | None, tuple[str, ...], bool]:
|
||||
def process_text(text: str) -> str:
|
||||
collapsed: Final = _collapse_base64_runs(text, base64_limit) if base64_limit > 0 else text
|
||||
scrubbed: Final = _python_redact_string(collapsed) if redact else collapsed
|
||||
return _truncate_for_stdout_log(scrubbed, text_limit) if 0 < text_limit < len(scrubbed) else scrubbed
|
||||
|
||||
processed_message: Final = process_text(message)
|
||||
processed_exception: Final = process_text(exception) if exception is not None else None
|
||||
processed_stack: Final = _python_redact_string(stack) if redact and stack is not None else stack
|
||||
processed_leaves: Final = tuple(
|
||||
_python_redact_structured_value(key, text) if redact else text for key, text in leaves
|
||||
)
|
||||
changed: Final = (
|
||||
processed_message != message
|
||||
or processed_exception != exception
|
||||
or processed_stack != stack
|
||||
or any(processed != original for processed, (_, original) in zip(processed_leaves, leaves))
|
||||
)
|
||||
return processed_message, processed_exception, processed_stack, processed_leaves, changed
|
||||
|
||||
|
||||
def _render_message(record: logging.LogRecord) -> str:
|
||||
try:
|
||||
return record.getMessage()
|
||||
except Exception:
|
||||
return record.msg if isinstance(record.msg, str) else UNSERIALIZABLE_OBJECT
|
||||
|
||||
|
||||
def _render_exception(record: logging.LogRecord) -> str | None:
|
||||
if not isinstance(record.exc_info, tuple) or len(record.exc_info) < 2 or record.exc_info[1] is None:
|
||||
return None
|
||||
try:
|
||||
return record.exc_text or SecretRedactionFilter._formatter.formatException(record.exc_info)
|
||||
except Exception:
|
||||
return "REDACTED"
|
||||
|
||||
|
||||
def _process_record(record: logging.LogRecord, *, base64_limit: int, text_limit: int, redact: bool) -> bool:
|
||||
if _is_redacted(record):
|
||||
return True
|
||||
message: Final = _render_message(record)
|
||||
exception: Final = _render_exception(record)
|
||||
stack: Final = record.stack_info if isinstance(record.stack_info, str) else None
|
||||
substituted_color: Final = _substituted_color_message(record)
|
||||
extras: Final = (
|
||||
tuple(
|
||||
(
|
||||
key,
|
||||
value,
|
||||
_extra_structure(
|
||||
key, substituted_color if key == "color_message" and substituted_color is not None else value
|
||||
),
|
||||
)
|
||||
for key, value in record.__dict__.items()
|
||||
if key not in _STANDARD_RECORD_ATTRS
|
||||
and key != _REDACTED_RECORD_ATTR
|
||||
and not isinstance(value, _UNREDACTED_SCALAR_TYPES)
|
||||
)
|
||||
if redact
|
||||
else ()
|
||||
)
|
||||
extra_leaves: Final = tuple(
|
||||
itertools.chain.from_iterable(_string_leaves(key, prepared) for key, _, prepared in extras)
|
||||
)
|
||||
raw_template: Final = record.msg if redact and isinstance(record.msg, str) and record.args else None
|
||||
color_template: Final = record.__dict__.get("color_message")
|
||||
raw_color: Final = color_template if redact and isinstance(color_template, str) and record.args else None
|
||||
leaves: Final = (
|
||||
extra_leaves
|
||||
+ (((None, raw_template),) if raw_template is not None else ())
|
||||
+ (((None, raw_color),) if raw_color is not None else ())
|
||||
)
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic(message, exception, stack, leaves, (redact, base64_limit, text_limit)),
|
||||
lambda: _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit),
|
||||
)
|
||||
processed_message, processed_exception, processed_stack, processed_leaves, _ = (
|
||||
candidate
|
||||
if len(candidate[3]) == len(leaves)
|
||||
else _python_process_diagnostic(message, exception, stack, leaves, redact, base64_limit, text_limit)
|
||||
)
|
||||
raw_template_changed: Final = raw_template is not None and processed_leaves[len(extra_leaves)] != raw_template
|
||||
safe_message: Final = "REDACTED" if raw_template_changed and processed_message == message else processed_message
|
||||
if redact or safe_message != message:
|
||||
record.msg = safe_message # rebind-ok: the Filter interface mutates the record
|
||||
record.args = None # rebind-ok: the rendered message replaces interpolation inputs
|
||||
if processed_exception is not None:
|
||||
record.exc_text = processed_exception # rebind-ok: the Filter interface mutates the record
|
||||
if processed_stack is not None:
|
||||
record.stack_info = processed_stack # rebind-ok: the Filter interface mutates the record
|
||||
processed_values: Final = iter(processed_leaves[: len(extra_leaves)])
|
||||
for key, original, prepared in extras:
|
||||
replacement: Final = _sort_processed_sets(original, _replace_string_leaves(prepared, processed_values))
|
||||
if not _scrubbing_changed_nothing(replacement, original):
|
||||
setattr(record, key, replacement)
|
||||
raw_color_changed: Final = (
|
||||
raw_color is not None and processed_leaves[len(extra_leaves) + int(raw_template is not None)] != raw_color
|
||||
)
|
||||
if raw_color_changed and getattr(record, "color_message", None) == substituted_color:
|
||||
setattr(record, "color_message", "REDACTED")
|
||||
setattr(record, _REDACTED_RECORD_ATTR, _REDACTED_STAMP)
|
||||
return True
|
||||
|
||||
|
||||
def _redact_json_record(value: object) -> object:
|
||||
prepared: Final = safe_json_structure(value)
|
||||
leaves: Final = tuple(_string_leaves(None, prepared))
|
||||
candidate: Final = diagnostics.run(
|
||||
lambda native: native.process_diagnostic("", None, None, leaves, (True, 0, 0))[3],
|
||||
lambda: tuple(_python_redact_structured_value(key, text) for key, text in leaves),
|
||||
)
|
||||
replacements: Final = (
|
||||
candidate
|
||||
if len(candidate) == len(leaves)
|
||||
else tuple(_python_redact_structured_value(key, text) for key, text in leaves)
|
||||
)
|
||||
return _sort_processed_sets(value, _replace_string_leaves(prepared, iter(replacements)))
|
||||
|
||||
|
||||
class StdoutLogTruncationFilter(logging.Filter):
|
||||
"""Bounds how much of an oversized log line reaches stdout.
|
||||
|
||||
|
|
@ -412,7 +554,17 @@ class StdoutLogTruncationFilter(logging.Filter):
|
|||
return True
|
||||
|
||||
|
||||
_stdout_truncation_filter: Final = StdoutLogTruncationFilter()
|
||||
class DiagnosticProcessingFilter(StdoutLogTruncationFilter):
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
return _process_record(
|
||||
record,
|
||||
base64_limit=_get_max_base64_length_stdout_log(),
|
||||
text_limit=_get_max_string_length_stdout_log() if record.levelno >= logging.INFO else 0,
|
||||
redact=_ENABLE_SECRET_REDACTION,
|
||||
)
|
||||
|
||||
|
||||
_diagnostic_filter: Final = DiagnosticProcessingFilter()
|
||||
|
||||
|
||||
class CorrelationContextFilter(logging.Filter):
|
||||
|
|
@ -633,7 +785,9 @@ class JsonFormatter(Formatter):
|
|||
if record.exc_info:
|
||||
json_record["stacktrace"] = record.exc_text or self.formatException(record.exc_info)
|
||||
|
||||
return safe_dumps(json_record, value_transform=None if _is_redacted(record) else _redact_structured_value)
|
||||
return safe_dumps(
|
||||
json_record if _is_redacted(record) or not _ENABLE_SECRET_REDACTION else _redact_json_record(json_record)
|
||||
)
|
||||
|
||||
|
||||
class CorrelationPlainFormatter(logging.Formatter):
|
||||
|
|
@ -663,7 +817,7 @@ def _setup_json_exception_handlers(formatter):
|
|||
# Create a handler with JSON formatting for exceptions
|
||||
error_handler: Final = logging.StreamHandler()
|
||||
error_handler.setFormatter(formatter)
|
||||
error_handler.addFilter(_stdout_truncation_filter)
|
||||
error_handler.addFilter(_diagnostic_filter)
|
||||
error_handler.addFilter(_secret_filter)
|
||||
error_handler.addFilter(_correlation_filter)
|
||||
|
||||
|
|
@ -734,10 +888,10 @@ verbose_logger.addHandler(handler)
|
|||
|
||||
# Filters attached to the logger, not the handler, survive callers swapping in their own
|
||||
# handlers (JSON mode, uvicorn log config, a host app's root handler).
|
||||
verbose_router_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_logger.addFilter(_stdout_truncation_filter)
|
||||
verbose_router_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_logger.addFilter(_diagnostic_filter)
|
||||
verbose_proxy_stdout_logger.addFilter(_diagnostic_filter)
|
||||
verbose_logger.addFilter(_diagnostic_filter)
|
||||
|
||||
|
||||
def _suppress_loggers():
|
||||
|
|
|
|||
|
|
@ -2375,6 +2375,11 @@ def default_image_cost_calculator(
|
|||
model_name_without_custom_llm_provider = model.replace(f"{custom_llm_provider}/", "")
|
||||
base_model_name = f"{custom_llm_provider}/{size_str}/{model_name_without_custom_llm_provider}"
|
||||
model_name_with_quality: Final = f"{quality}/{base_model_name}" if quality else base_model_name
|
||||
provider_first_model_name_with_quality: Final = (
|
||||
f"{custom_llm_provider}/{quality}/{size_str}/{model_name_without_custom_llm_provider or model}"
|
||||
if quality and custom_llm_provider
|
||||
else None
|
||||
)
|
||||
|
||||
# gpt-image-1 models use low, medium, high quality. If user did not specify quality, use medium fot gpt-image-1 model family
|
||||
model_name_with_v2_quality: Final = f"{ImageGenerationRequestQuality.HIGH.value}/{base_model_name}"
|
||||
|
|
@ -2386,6 +2391,7 @@ def default_image_cost_calculator(
|
|||
|
||||
models_to_check: Final = (
|
||||
model_name_with_quality,
|
||||
provider_first_model_name_with_quality,
|
||||
base_model_name,
|
||||
model_name_with_v2_quality,
|
||||
model_with_quality_without_provider,
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import re
|
|||
from typing import Final
|
||||
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge import diagnostics
|
||||
|
||||
REDACTED: Final = "REDACTED"
|
||||
|
||||
|
|
@ -87,9 +88,13 @@ def _build_secret_patterns() -> "re.Pattern[str]":
|
|||
_SECRET_RE: Final = _build_secret_patterns()
|
||||
|
||||
|
||||
def _python_redact_string(value: str) -> str:
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
|
||||
|
||||
def redact_string(value: str) -> str:
|
||||
"""Scrub known secret/credential patterns from *value* and return the result."""
|
||||
return _SECRET_RE.sub(REDACTED, value)
|
||||
return diagnostics.run(lambda native: native.redact_text(value), lambda: _python_redact_string(value))
|
||||
|
||||
|
||||
_UNIX_SYSTEM_PATH: Final = r"/(?:etc|var|opt|usr|home|root|private|Users|tmp|mnt|srv)/[^\s'\"\)\]}>,]+"
|
||||
|
|
@ -105,15 +110,21 @@ _INTERNAL_DETAIL_RE: Final = re.compile(
|
|||
_TRACEBACK_MARKER: Final = "Traceback (most recent call last):"
|
||||
|
||||
|
||||
def redact_internal_details(value: str) -> str:
|
||||
def _python_redact_internal_details(value: str) -> str:
|
||||
"""Drop an embedded traceback and scrub filesystem paths and internal hostnames,
|
||||
on top of redact_string(). For client-facing messages only: server logs keep this detail."""
|
||||
marker_index: Final = value.find(_TRACEBACK_MARKER)
|
||||
without_traceback: Final = value[:marker_index].rstrip() if marker_index != -1 else value
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, redact_string(without_traceback))
|
||||
return _INTERNAL_DETAIL_RE.sub(REDACTED, _python_redact_string(without_traceback))
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
def redact_internal_details(value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_client_message(value), lambda: _python_redact_internal_details(value)
|
||||
)
|
||||
|
||||
|
||||
def _python_redact_structured_value(key: str | None, value: str) -> str:
|
||||
"""Scrub *value* as it appeared under *key* inside a structured record.
|
||||
|
||||
redact_string() replaces a whole ``key: value`` span with REDACTED, which is
|
||||
|
|
@ -122,8 +133,15 @@ def redact_structured_value(key: str | None, value: str) -> str:
|
|||
repr would, so the key-name patterns still fire, but collapses only the value
|
||||
so the caller's structure survives.
|
||||
"""
|
||||
scrubbed: Final = redact_string(value)
|
||||
scrubbed: Final = _python_redact_string(value)
|
||||
if scrubbed != value or key is None:
|
||||
return scrubbed
|
||||
rendered: Final = f"'{key}': '{value}'"
|
||||
return REDACTED if redact_string(rendered) != rendered else value
|
||||
return REDACTED if _python_redact_string(rendered) != rendered else value
|
||||
|
||||
|
||||
def redact_structured_value(key: str | None, value: str) -> str:
|
||||
return diagnostics.run(
|
||||
lambda native: native.redact_structured_text(key, value),
|
||||
lambda: _python_redact_structured_value(key, value),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -629,17 +629,34 @@ class AmazonConverseConfig(BaseConfig):
|
|||
"""
|
||||
return self._is_deepseek_r1_model(model=model, base_model=base_model)
|
||||
|
||||
@classmethod
|
||||
def _supports_sampling_params(cls, model: str) -> bool:
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
||||
base_model: Final = BedrockModelInfo.get_base_model(model)
|
||||
if base_model.startswith("anthropic"):
|
||||
return True
|
||||
candidates: Final = (model, *(f"{prefix}{base_model}" for prefix in ("global.", "us.", "eu.")))
|
||||
for candidate in candidates:
|
||||
if (
|
||||
flag := AnthropicModelInfo._get_model_capability( # pyright: ignore[reportPrivateUsage] # Shared API
|
||||
candidate, "supports_sampling_params"
|
||||
)
|
||||
) is not None:
|
||||
return flag
|
||||
return True
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[str]:
|
||||
from litellm.utils import supports_function_calling
|
||||
|
||||
supports_sampling: Final = self._supports_sampling_params(model)
|
||||
supported_params: Final = [
|
||||
"max_tokens",
|
||||
"max_completion_tokens",
|
||||
"stream",
|
||||
"stream_options",
|
||||
"stop",
|
||||
"temperature",
|
||||
"top_p",
|
||||
*(("temperature", "top_p") if supports_sampling else ()),
|
||||
"extra_headers",
|
||||
"response_format",
|
||||
"requestMetadata",
|
||||
|
|
@ -1019,14 +1036,26 @@ class AmazonConverseConfig(BaseConfig):
|
|||
value = [value]
|
||||
optional_params["stopSequences"] = value
|
||||
if param == "temperature" or param == "top_p":
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
if base_model.startswith("anthropic"):
|
||||
AnthropicConfig._apply_sampling_param(
|
||||
optional_params=optional_params,
|
||||
model=model,
|
||||
param=param,
|
||||
value=value,
|
||||
drop_params=drop_params,
|
||||
output_key="topP" if param == "top_p" else param,
|
||||
)
|
||||
elif not self._supports_sampling_params(model):
|
||||
if not (litellm.drop_params or drop_params):
|
||||
raise litellm.utils.UnsupportedParamsError(
|
||||
message=(
|
||||
f"{model} does not support {param}={value}. "
|
||||
"To drop unsupported params, set `litellm.drop_params = True`."
|
||||
),
|
||||
status_code=400,
|
||||
)
|
||||
else:
|
||||
optional_params["topP" if param == "top_p" else param] = value
|
||||
if param == "tools" and isinstance(value, list):
|
||||
self._apply_tool_call_transformation(
|
||||
tools=cast(list[OpenAIChatCompletionToolParam], value),
|
||||
|
|
|
|||
|
|
@ -231,6 +231,12 @@ async def make_call(
|
|||
sync_stream=False,
|
||||
)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
elif bedrock_invoke_provider == "moonshot":
|
||||
decoder = AmazonOpenAICompatibleStreamDecoder(
|
||||
model=model,
|
||||
sync_stream=False,
|
||||
)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
else:
|
||||
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
||||
completion_stream = decoder.aiter_bytes(response.aiter_bytes(chunk_size=stream_chunk_size))
|
||||
|
|
@ -329,6 +335,12 @@ def make_sync_call(
|
|||
sync_stream=True,
|
||||
)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
elif bedrock_invoke_provider == "moonshot":
|
||||
decoder = AmazonOpenAICompatibleStreamDecoder(
|
||||
model=model,
|
||||
sync_stream=True,
|
||||
)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
else:
|
||||
decoder = AWSEventStreamDecoder(model=model, json_mode=json_mode)
|
||||
completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size))
|
||||
|
|
@ -795,6 +807,24 @@ class AmazonDeepSeekR1StreamDecoder(AWSEventStreamDecoder):
|
|||
return self.deepseek_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
|
||||
|
||||
class AmazonOpenAICompatibleStreamDecoder(AWSEventStreamDecoder):
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
sync_stream: bool,
|
||||
) -> None:
|
||||
super().__init__(model=model)
|
||||
from litellm.llms.openai.chat.gpt_transformation import OpenAIChatCompletionStreamingHandler
|
||||
|
||||
self.openai_model_response_iterator = OpenAIChatCompletionStreamingHandler(
|
||||
streaming_response=None,
|
||||
sync_stream=sync_stream,
|
||||
)
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict[str, object]) -> ModelResponseStream:
|
||||
return self.openai_model_response_iterator.chunk_parser(chunk=chunk_data)
|
||||
|
||||
|
||||
class MockResponseIterator: # for returning ai21 streaming responses
|
||||
def __init__(self, model_response, json_mode: bool | None = False):
|
||||
self.model_response = model_response
|
||||
|
|
|
|||
|
|
@ -770,9 +770,7 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
aws_decoder: Final = AmazonAnthropicClaudeMessagesStreamDecoder(
|
||||
model=model,
|
||||
)
|
||||
completion_stream: Final = aws_decoder.aiter_bytes(
|
||||
httpx_response.aiter_bytes(chunk_size=aws_decoder.DEFAULT_CHUNK_SIZE)
|
||||
)
|
||||
completion_stream: Final = aws_decoder.aiter_bytes(httpx_response.aiter_bytes())
|
||||
# Convert decoded Bedrock events to Server-Sent Events expected by Anthropic clients.
|
||||
return self.bedrock_sse_wrapper(
|
||||
completion_stream=completion_stream,
|
||||
|
|
@ -919,16 +917,6 @@ class AmazonAnthropicClaudeMessagesConfig(
|
|||
|
||||
|
||||
class AmazonAnthropicClaudeMessagesStreamDecoder(AWSEventStreamDecoder):
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
) -> None:
|
||||
"""
|
||||
Iterator to return Bedrock invoke response in anthropic /messages format
|
||||
"""
|
||||
super().__init__(model=model)
|
||||
self.DEFAULT_CHUNK_SIZE = 1024
|
||||
|
||||
def _chunk_parser(self, chunk_data: dict) -> GChunk | ModelResponseStream | dict:
|
||||
"""
|
||||
Parse the chunk data into anthropic /messages format
|
||||
|
|
|
|||
|
|
@ -287,7 +287,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
Check if the model is Gemini 3 or newer.
|
||||
"""
|
||||
model_name = model.split("/")[-1].lower()
|
||||
if not model_name:
|
||||
is_vertex_fine_tuned_model: Final = model_name.isdigit() or (
|
||||
model.startswith("gemini/") and not model_name.startswith("gemini-")
|
||||
)
|
||||
if not model_name or is_vertex_fine_tuned_model or model_name.startswith("gemma-"):
|
||||
return False
|
||||
# Pre-Gemini 3 models: gemini-1.x, gemini-2.x, gemini-pro, gemini-flash, gemini-exp
|
||||
if re.match(r"^gemini-(?:[12](?:\.\d+)?|exp|(?:pro|flash)(?!-(?:lite-)?latest$))(?:-|$)", model_name):
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -42,6 +42,7 @@ from litellm.proxy._types import (
|
|||
SpecialMCPServerName,
|
||||
SpecialMCPServerNames,
|
||||
UserAPIKeyAuth,
|
||||
hash_token,
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.agent_endpoints.auth.agent_access_groups import (
|
||||
|
|
@ -182,6 +183,20 @@ def _is_litellm_auth_admission_error(exc: Exception) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def _explicit_credential_matches_envelope(
|
||||
explicit_auth: UserAPIKeyAuth,
|
||||
presented_token: str,
|
||||
identity: EnvelopeIdentity,
|
||||
) -> bool:
|
||||
"""Match the stored key hash or user ID, including token-only mapped JWT keys."""
|
||||
match identity.subject_type:
|
||||
case "key_hash":
|
||||
return identity.subject in (hash_token(presented_token), explicit_auth.token)
|
||||
case "user_id":
|
||||
return explicit_auth.user_id is not None and explicit_auth.user_id == identity.subject
|
||||
return assert_never(identity.subject_type)
|
||||
|
||||
|
||||
def _has_client_supplied_mcp_auth(
|
||||
mcp_auth_header: str | None,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
|
||||
|
|
@ -475,6 +490,31 @@ class MCPRequestHandler:
|
|||
# Only OAuth metadata routes registered under /.well-known/ are public.
|
||||
if request_route.startswith("/.well-known/"):
|
||||
validated_user_api_key_auth = UserAPIKeyAuth()
|
||||
elif (
|
||||
has_explicit_litellm_key
|
||||
and oauth2_headers
|
||||
and is_bridge_envelope_shaped(oauth2_headers["Authorization"])
|
||||
and (
|
||||
dual_bridge_target := MCPRequestHandler._single_dcr_bridge_delegate_target(
|
||||
path=request_route,
|
||||
mcp_servers=mcp_servers,
|
||||
client_ip=IPAddressUtils.get_mcp_client_ip(request),
|
||||
)
|
||||
)
|
||||
is not None
|
||||
):
|
||||
(
|
||||
validated_user_api_key_auth,
|
||||
mcp_server_auth_headers,
|
||||
) = await MCPRequestHandler._admit_dcr_bridge_dual_credential(
|
||||
server=dual_bridge_target.server,
|
||||
requested_name=dual_bridge_target.requested_name,
|
||||
authorization_value=oauth2_headers["Authorization"],
|
||||
litellm_api_key=litellm_api_key,
|
||||
mcp_server_auth_headers=mcp_server_auth_headers,
|
||||
request=request,
|
||||
route=request_route,
|
||||
)
|
||||
elif has_explicit_litellm_key:
|
||||
# An explicit x-litellm-api-key is always a LiteLLM credential, even
|
||||
# for a delegated server, so validate it: identity / spend / rate
|
||||
|
|
@ -782,6 +822,45 @@ class MCPRequestHandler:
|
|||
higher-priority alias slot, pairing the admitted identity with an attacker's upstream
|
||||
credential; the alias-keyed injection overwrites any such caller value.
|
||||
"""
|
||||
result: Final = await MCPRequestHandler._open_dcr_bridge_envelope(
|
||||
server=server,
|
||||
requested_name=requested_name,
|
||||
authorization_value=authorization_value,
|
||||
request=request,
|
||||
route=route,
|
||||
)
|
||||
header_key: Final = server.alias or server.server_name
|
||||
if header_key is None:
|
||||
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
|
||||
admitted: Final = await MCPRequestHandler._reload_admitted_principal(result.identity)
|
||||
await MCPRequestHandler._enforce_admitted_live_policy(admitted=admitted, request=request, route=route)
|
||||
injected: Final = { # mutable-ok: mcp_server_auth_headers contract requires concrete dicts
|
||||
header_key: { # mutable-ok: concrete dict header payload
|
||||
"Authorization": result.upstream_authorization.get_secret_value()
|
||||
}
|
||||
}
|
||||
new_headers: Final = { # mutable-ok: merged header map must stay a concrete dict
|
||||
**(mcp_server_auth_headers or {}), # mutable-ok: empty-dict fallback for the merge
|
||||
**injected,
|
||||
}
|
||||
return admitted, new_headers
|
||||
|
||||
@staticmethod
|
||||
async def _open_dcr_bridge_envelope(
|
||||
server: MCPServer,
|
||||
requested_name: str,
|
||||
authorization_value: str,
|
||||
request: Request,
|
||||
route: str,
|
||||
) -> BridgeEnvelopeAdmitted:
|
||||
"""Open a bridge envelope after the pre-DB gates, or fail closed with the scope's challenge.
|
||||
|
||||
Shared by the envelope-only arm (:meth:`_admit_dcr_bridge_delegate`) and the dual-credential
|
||||
arm (:meth:`_admit_dcr_bridge_dual_credential`): both require master_key, run the same
|
||||
proxy-wide pre-DB checks the standard pipeline applies before any key lookup, and resolve
|
||||
the envelope's crypto. Returns only the ``BridgeEnvelopeAdmitted`` result; an invalid,
|
||||
expired, tampered, or non-envelope value raises the requested scope's ``invalid_token``
|
||||
challenge instead."""
|
||||
from litellm.proxy.proxy_server import master_key
|
||||
|
||||
if not master_key:
|
||||
|
|
@ -793,20 +872,67 @@ class MCPRequestHandler:
|
|||
result: Final = resolve_bridge_envelope(authorization_value, keys, datetime.now(timezone.utc), server.server_id)
|
||||
match result:
|
||||
case BridgeEnvelopeAdmitted():
|
||||
header_key: Final = server.alias or server.server_name
|
||||
if header_key is None:
|
||||
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
|
||||
admitted: Final = await MCPRequestHandler._reload_admitted_principal(result.identity)
|
||||
await MCPRequestHandler._enforce_admitted_live_policy(admitted=admitted, request=request, route=route)
|
||||
injected: Final = {header_key: {"Authorization": result.upstream_authorization.get_secret_value()}}
|
||||
new_headers: Final = {**(mcp_server_auth_headers or {}), **injected}
|
||||
return admitted, new_headers
|
||||
return result
|
||||
case BridgeEnvelopeInvalid() | NotBridgeEnvelope():
|
||||
raise MCPRequestHandler._dcr_bridge_invalid_token_challenge(
|
||||
requested_name=requested_name, request=request
|
||||
)
|
||||
case _:
|
||||
assert_never(result)
|
||||
return assert_never(result)
|
||||
|
||||
@staticmethod
|
||||
async def _admit_dcr_bridge_dual_credential(
|
||||
server: MCPServer,
|
||||
requested_name: str,
|
||||
authorization_value: str,
|
||||
litellm_api_key: str,
|
||||
mcp_server_auth_headers: dict[str, dict[str, str]] | None,
|
||||
request: Request,
|
||||
route: str,
|
||||
) -> tuple[UserAPIKeyAuth, dict[str, dict[str, str]] | None]:
|
||||
"""Admit a request carrying BOTH an explicit litellm credential and a bridge envelope.
|
||||
|
||||
MCP clients send ``x-litellm-api-key`` on every request, including the ``tools/list`` that
|
||||
follows the ``/{server}/token`` mint, so the envelope arrives alongside the key rather than
|
||||
alone. The explicit credential is validated first (its own pipeline, so a bad key keeps the
|
||||
normal 401/403), then the envelope is opened and its sealed identity must match the explicit
|
||||
credential's principal — a mismatch is a 403, never a fallback onto either credential alone.
|
||||
On a match the explicit credential's ``UserAPIKeyAuth`` is the admission context (key
|
||||
permissions, budgets, rate limits) and the sealed upstream token is injected under the
|
||||
server's per-server auth-header key, while the leak-defense chokepoint strips the envelope
|
||||
``Authorization`` itself from egress."""
|
||||
presented_token: Final = _get_bearer_token_or_received_api_key(litellm_api_key)
|
||||
explicit_auth: Final = await user_api_key_auth(api_key=f"Bearer {presented_token}", request=request)
|
||||
result: Final = await MCPRequestHandler._open_dcr_bridge_envelope(
|
||||
server=server,
|
||||
requested_name=requested_name,
|
||||
authorization_value=authorization_value,
|
||||
request=request,
|
||||
route=route,
|
||||
)
|
||||
if not _explicit_credential_matches_envelope(
|
||||
explicit_auth=explicit_auth,
|
||||
presented_token=presented_token,
|
||||
identity=result.identity,
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail={ # mutable-ok: HTTPException detail payload requires a concrete dict
|
||||
"error": "oauth_principal_mismatch"
|
||||
},
|
||||
)
|
||||
header_key: Final = server.alias or server.server_name
|
||||
if header_key is None:
|
||||
raise HTTPException(status_code=500, detail="Server misconfigured: MCP server has no routable name")
|
||||
injected: Final = { # mutable-ok: mcp_server_auth_headers contract requires concrete dicts
|
||||
header_key: { # mutable-ok: concrete dict header payload
|
||||
"Authorization": result.upstream_authorization.get_secret_value()
|
||||
}
|
||||
}
|
||||
new_headers: Final = { # mutable-ok: merged header map must stay a concrete dict
|
||||
**(mcp_server_auth_headers or {}), # mutable-ok: empty-dict fallback for the merge
|
||||
**injected,
|
||||
}
|
||||
return explicit_auth, new_headers
|
||||
|
||||
@staticmethod
|
||||
async def _admit_dcr_bridge_authorization(
|
||||
|
|
@ -1091,9 +1217,15 @@ class MCPRequestHandler:
|
|||
project, org, and budget state are NOT re-checked here; the caller runs the admitted
|
||||
identity through ``_enforce_admitted_live_policy`` for those.
|
||||
"""
|
||||
from litellm.proxy._experimental.mcp_server.bridge_token_flow import (
|
||||
master_key_admin_auth, # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
)
|
||||
from litellm.proxy.auth.auth_checks import get_key_object
|
||||
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
|
||||
|
||||
admin: Final = master_key_admin_auth(key_hash)
|
||||
if admin is not None:
|
||||
return admin
|
||||
if prisma_client is None:
|
||||
raise HTTPException(status_code=500, detail="Server misconfigured: no database connection")
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -209,6 +209,31 @@ async def _resolve_active_litellm_key(request: Request) -> "_ResolvedKey | _KeyR
|
|||
return await _reload_active_key_by_hash(hash_token(token))
|
||||
|
||||
|
||||
def master_key_admin_auth(key_hash: str) -> "UserAPIKeyAuth | None":
|
||||
from litellm.constants import ( # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
)
|
||||
from litellm.proxy._types import ( # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
LitellmUserRoles,
|
||||
UserAPIKeyAuth,
|
||||
hash_token,
|
||||
)
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
litellm_proxy_admin_name,
|
||||
master_key,
|
||||
)
|
||||
|
||||
if not master_key or not secrets.compare_digest(key_hash, hash_token(master_key)):
|
||||
return None
|
||||
auth: Final = UserAPIKeyAuth(
|
||||
api_key=LITELLM_PROXY_MASTER_KEY_ALIAS,
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN,
|
||||
user_id=litellm_proxy_admin_name,
|
||||
)
|
||||
auth.via_virtual_key = True
|
||||
return auth
|
||||
|
||||
|
||||
async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResolutionFailure":
|
||||
"""Reload the live key record for ``key_hash`` (cache first, then DB) and gate it on active state,
|
||||
returning the resolved key or a precise failure. Shared by the token request's presented-key
|
||||
|
|
@ -233,6 +258,8 @@ async def _reload_active_key_by_hash(key_hash: str) -> "_ResolvedKey | _KeyResol
|
|||
user_api_key_cache,
|
||||
)
|
||||
|
||||
if (admin := master_key_admin_auth(key_hash)) is not None:
|
||||
return _ResolvedKey(key_hash=key_hash, key=admin)
|
||||
if prisma_client is None:
|
||||
return "unresolvable"
|
||||
try:
|
||||
|
|
@ -475,7 +502,7 @@ async def _resolve_jwt_auth(
|
|||
proxy_logging_obj=proxy_logging_obj,
|
||||
)
|
||||
if isinstance(mapped, UserAPIKeyAuth):
|
||||
return None if await _key_owner_scim_deactivated(mapped) or not _active_key_user_id(mapped) else mapped
|
||||
return None if await _key_owner_scim_deactivated(mapped) or not _key_is_active(mapped) else mapped
|
||||
if mapped is not None:
|
||||
return None
|
||||
if write_route is None:
|
||||
|
|
@ -593,6 +620,7 @@ def _bridge_grant_from_token_response(token_response: object) -> "UpstreamTokenG
|
|||
|
||||
_BridgeMintError = Literal[
|
||||
"no_identity",
|
||||
"jwt_client_policy_unsupported",
|
||||
"invalid_refresh",
|
||||
"identity_unavailable",
|
||||
"identity_faulted",
|
||||
|
|
@ -633,6 +661,13 @@ def _bridge_mint_error_response(error: _BridgeMintError) -> JSONResponse:
|
|||
"this server issues a gateway-bound credential; complete the interactive sign-in, or "
|
||||
"send a litellm credential (x-litellm-api-key or Authorization) on the token request",
|
||||
)
|
||||
case "jwt_client_policy_unsupported":
|
||||
status, code, desc = (
|
||||
400,
|
||||
"invalid_request",
|
||||
"JWT bridge minting is not supported with a claim-based MCP client allowlist; "
|
||||
"the bridge credential cannot preserve the signed client identity",
|
||||
)
|
||||
case "invalid_refresh":
|
||||
status, code, desc = (
|
||||
400,
|
||||
|
|
@ -736,11 +771,18 @@ async def _prepare_bridge_mint(
|
|||
|
||||
Two identity sources, one envelope. The interactive DCR client authenticates via SSO at the bridged
|
||||
authorize, so its identity arrives as ``bridge_identity`` (the user recovered from the gateway
|
||||
authorization code) and mints a user subject. The scripted two-header client presents a litellm key
|
||||
on the token request instead, so its identity is the active key's hash and mints a key_hash subject.
|
||||
A missing or invalid presented key keeps its resolution origin so the mapper statuses it truthfully;
|
||||
neither source present is ``no_identity``. The refresh_token grant has its own phase-1
|
||||
authorization code) and mints a user subject. The scripted two-header client presents a litellm
|
||||
credential (a virtual key or a JWT) on the token request instead: a key mints a key_hash subject,
|
||||
while a JWT resolves through the same auth path as admission and mints a key_hash subject when it
|
||||
maps to a virtual key. An unmapped JWT is rejected because a user subject cannot preserve its
|
||||
JWT-specific authorization restrictions. A JWT client-claim allowlist also prevents JWT minting:
|
||||
the envelope cannot retain the signed client identity for subsequent allowlist checks. A missing or invalid
|
||||
presented key keeps its resolution origin so the mapper statuses it truthfully; neither source
|
||||
present is ``no_identity``. The refresh_token grant has its own phase-1
|
||||
(:func:`_prepare_bridge_refresh`), which recovers identity from the presented refresh envelope."""
|
||||
from litellm.proxy._experimental.mcp_server.client_allowlist import ( # noqa: PLC0415 # keep mint policy dependencies local
|
||||
load_mcp_client_allowlist,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import ( # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
envelope_keys_from_master_key,
|
||||
)
|
||||
|
|
@ -748,7 +790,10 @@ async def _prepare_bridge_mint(
|
|||
key_hash_identity,
|
||||
user_identity,
|
||||
)
|
||||
from litellm.proxy._types import UserAPIKeyAuth # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.auth.handle_jwt import JWTHandler # noqa: PLC0415 # proxy import cycle
|
||||
from litellm.proxy.proxy_server import ( # noqa: PLC0415 # inline import avoids a module-load circular import
|
||||
general_settings,
|
||||
master_key,
|
||||
)
|
||||
|
||||
|
|
@ -758,6 +803,16 @@ async def _prepare_bridge_mint(
|
|||
if bridge_identity is not None:
|
||||
identity = user_identity(server_id=mcp_server.server_id, user_id=bridge_identity.litellm_user_id)
|
||||
return _BridgeMintReady(identity=identity, keys=keys)
|
||||
presented_token: Final = _litellm_key_from_request(request)
|
||||
if presented_token is not None and JWTHandler.is_jwt(presented_token):
|
||||
client_allowlist: Final = load_mcp_client_allowlist(general_settings)
|
||||
if client_allowlist is not None and client_allowlist.jwt_field is not None:
|
||||
return "jwt_client_policy_unsupported"
|
||||
resolved_jwt: Final = await _resolve_jwt_auth(request, presented_token, None)
|
||||
if isinstance(resolved_jwt, UserAPIKeyAuth) and resolved_jwt.token:
|
||||
identity = key_hash_identity(server_id=mcp_server.server_id, key_hash=resolved_jwt.token)
|
||||
return _BridgeMintReady(identity=identity, keys=keys)
|
||||
return "no_identity"
|
||||
resolved: Final = await _resolve_active_litellm_key(request)
|
||||
if not isinstance(resolved, _ResolvedKey):
|
||||
return _key_resolution_failure_to_mint_error(resolved)
|
||||
|
|
|
|||
|
|
@ -937,6 +937,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
# proxy admin, or team admin naming their own team via team_id
|
||||
"/auto_router/test_routing",
|
||||
"/auto_router/validate_complexity_router_config",
|
||||
"/auto_router/availability",
|
||||
# Per-session auto-router read - the endpoint scopes the row to the caller's own key hash
|
||||
"/auto_router/session",
|
||||
"/cost/predict-cache",
|
||||
|
|
|
|||
|
|
@ -58,6 +58,8 @@ from litellm.router_utils.auto_router_model_naming import (
|
|||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
SHADOW_EVAL_TURN_VALVE,
|
||||
AutoRouterAvailabilityRequest,
|
||||
AutoRouterAvailabilityResponse,
|
||||
AutoRouterBenchmarkGroup,
|
||||
AutoRouterBenchmarksResponse,
|
||||
AutoRouterBenchmarkTotals,
|
||||
|
|
@ -391,6 +393,54 @@ async def validate_complexity_router_config(
|
|||
return ComplexityRouterConfigValidationResponse(valid=error is None, error=error)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/auto_router/availability",
|
||||
tags=["model management"], # mutable-ok: FastAPI requires a list
|
||||
response_model=AutoRouterAvailabilityResponse,
|
||||
)
|
||||
async def get_auto_router_availability(
|
||||
data: AutoRouterAvailabilityRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
from litellm.proxy.management_helpers.auto_router_availability import auto_router_availability
|
||||
from litellm.proxy.proxy_server import (
|
||||
_license_check, # pyright: ignore[reportPrivateUsage] # same entitlement owner as the model write gate
|
||||
heuristic_v1_tuning_baselines,
|
||||
llm_router,
|
||||
proxy_config,
|
||||
)
|
||||
|
||||
member_team: Final = await _authorize_router_dry_run(user_api_key_dict, data.team_id)
|
||||
rows: Final = proxy_config.auto_router_db_catalog
|
||||
if rows is None or llm_router is None:
|
||||
raise HTTPException(status_code=503, detail="Auto-router availability is unavailable")
|
||||
saved: Final = next((row for row in rows if row.model_id == data.saved_model_id), None)
|
||||
if data.saved_model_id is not None:
|
||||
if saved is None:
|
||||
raise HTTPException(status_code=404, detail="Saved auto router is unavailable")
|
||||
if user_api_key_dict.user_role != LitellmUserRoles.PROXY_ADMIN and (
|
||||
saved.team_id != data.team_id or (member_team is not None and saved.created_by != user_api_key_dict.user_id)
|
||||
):
|
||||
raise HTTPException(status_code=403, detail="Cannot check another user's auto router")
|
||||
existing: Final = saved.deployment if saved is not None else None
|
||||
others: Final = tuple(row.deployment for row in rows if row is not saved) + tuple(llm_router.config_deployments())
|
||||
candidate: Final = MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{"model": "auto_router/complexity_router", "complexity_router_config": data.complexity_router_config}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": data.saved_model_id or "availability-new-router", "db_model": True}),
|
||||
}
|
||||
)
|
||||
return auto_router_availability(
|
||||
others=others,
|
||||
existing=existing,
|
||||
candidate=candidate,
|
||||
baselines=heuristic_v1_tuning_baselines,
|
||||
limit=_license_check.auto_router_capability_limit(),
|
||||
)
|
||||
|
||||
|
||||
async def _resolve_saved_routing_test(
|
||||
data: AutoRouterRoutingTestRequest,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
|
|
|
|||
|
|
@ -1782,10 +1782,11 @@ async def delete_team_models(
|
|||
# Under MODEL_RECONCILE_LOCK, for the same reason as delete_model: the rows are
|
||||
# gone, but a reconcile holding a pre-delete snapshot would upsert these ids back
|
||||
# onto this pod. The lock orders the eviction after any in-flight reconcile.
|
||||
if llm_router is not None:
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK
|
||||
from litellm.proxy.proxy_server import MODEL_RECONCILE_LOCK, proxy_config
|
||||
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset(deleted_model_ids))
|
||||
if llm_router is not None:
|
||||
for model_id in deleted_model_ids:
|
||||
llm_router.delete_deployment(id=model_id)
|
||||
|
||||
|
|
@ -2194,6 +2195,7 @@ async def delete_model(
|
|||
llm_router,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
proxy_config,
|
||||
proxy_logging_obj,
|
||||
store_model_in_db,
|
||||
user_api_key_cache,
|
||||
|
|
@ -2245,8 +2247,9 @@ async def delete_model(
|
|||
# this pod serving a model the database no longer has, until the next
|
||||
# reconcile. Taking the lock orders this eviction after any such in-flight
|
||||
# reconcile's re-add, so the eviction is the last word.
|
||||
if llm_router is not None:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
async with MODEL_RECONCILE_LOCK:
|
||||
proxy_config.remove_auto_router_catalog_entries(frozenset({model_info.id}))
|
||||
if llm_router is not None:
|
||||
llm_router.delete_deployment(id=model_info.id)
|
||||
|
||||
# Runs after the row delete so the sibling check sees post-delete state.
|
||||
|
|
|
|||
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
148
litellm/proxy/management_helpers/auto_router_availability.py
Normal file
|
|
@ -0,0 +1,148 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from copy import deepcopy
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, Json, TypeAdapter, ValidationError
|
||||
|
||||
from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper
|
||||
from litellm.router_utils.auto_router_model_naming import (
|
||||
GATED_AUTO_ROUTER_CAPABILITIES,
|
||||
capability_limit_violation,
|
||||
classify_strategy_router_model,
|
||||
count_capability_routers,
|
||||
gated_capability_of,
|
||||
)
|
||||
from litellm.router_utils.auto_router_tuning_baseline import (
|
||||
is_mutable_tuned_candidate,
|
||||
mutable_tuned_identities,
|
||||
tuning_quota_violation,
|
||||
)
|
||||
from litellm.types.management_endpoints.auto_router_endpoints import (
|
||||
AutoRouterAllowance,
|
||||
AutoRouterAvailabilityResponse,
|
||||
)
|
||||
|
||||
|
||||
class _CatalogModelInfo(BaseModel):
|
||||
team_id: str | None = None
|
||||
|
||||
|
||||
class _CatalogSource(BaseModel):
|
||||
model_id: str
|
||||
created_by: str | None = None
|
||||
litellm_params: Json[dict[str, object]] | dict[str, object]
|
||||
model_info: Json[_CatalogModelInfo] | _CatalogModelInfo | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AutoRouterCatalogEntry:
|
||||
model_id: str
|
||||
team_id: str | None
|
||||
created_by: str | None
|
||||
deployment: Mapping[str, object]
|
||||
|
||||
|
||||
def _catalog_field(value: object, key: str) -> object:
|
||||
if not isinstance(value, str):
|
||||
return deepcopy(value)
|
||||
return decrypt_value_helper(value, key=key, exception_type="debug", return_original_value=True)
|
||||
|
||||
|
||||
def build_auto_router_catalog(rows: Sequence[object]) -> tuple[AutoRouterCatalogEntry, ...] | None:
|
||||
try:
|
||||
sources: Final = TypeAdapter(tuple[_CatalogSource, ...]).validate_python(rows, from_attributes=True)
|
||||
except ValidationError:
|
||||
return None
|
||||
return tuple(
|
||||
AutoRouterCatalogEntry(
|
||||
model_id=row.model_id,
|
||||
team_id=row.model_info.team_id if row.model_info is not None else None,
|
||||
created_by=row.created_by,
|
||||
deployment=MappingProxyType(
|
||||
{
|
||||
"litellm_params": MappingProxyType(
|
||||
{
|
||||
"model": model,
|
||||
"complexity_router_config": _catalog_field(
|
||||
row.litellm_params.get("complexity_router_config"), "complexity_router_config"
|
||||
),
|
||||
}
|
||||
),
|
||||
"model_info": MappingProxyType({"id": row.model_id, "db_model": True}),
|
||||
}
|
||||
),
|
||||
)
|
||||
for row in sources
|
||||
if isinstance(model := _catalog_field(row.litellm_params.get("model"), "model"), str)
|
||||
and classify_strategy_router_model(model) == "complexity"
|
||||
)
|
||||
|
||||
|
||||
def auto_router_availability(
|
||||
*,
|
||||
others: Sequence[Mapping[str, object]],
|
||||
existing: Mapping[str, object] | None,
|
||||
candidate: Mapping[str, object],
|
||||
baselines: Mapping[str, str] | None,
|
||||
limit: int | None,
|
||||
) -> AutoRouterAvailabilityResponse:
|
||||
existing_params: Final = None if existing is None else existing.get("litellm_params")
|
||||
candidate_params: Final = candidate.get("litellm_params")
|
||||
owned: Final = gated_capability_of(existing_params) if isinstance(existing_params, Mapping) else None
|
||||
claimed: Final = gated_capability_of(candidate_params) if isinstance(candidate_params, Mapping) else None
|
||||
counts: Final = tuple(
|
||||
(capability, count_capability_routers(others, capability=capability))
|
||||
for capability in GATED_AUTO_ROUTER_CAPABILITIES
|
||||
)
|
||||
tuned_count: Final = len(mutable_tuned_identities(others, baselines)) if baselines is not None else 0
|
||||
allowances: Final = tuple(
|
||||
AutoRouterAllowance(
|
||||
key=capability.key,
|
||||
limit=limit,
|
||||
remaining=None if limit is None else max(0, limit - held),
|
||||
used_by_this_router=owned is capability,
|
||||
)
|
||||
for capability, held in counts
|
||||
)
|
||||
capability_error: Final = next(
|
||||
(
|
||||
capability_limit_violation(capability=capability, held=held + 1, limit=limit)
|
||||
for capability, held in counts
|
||||
if capability is claimed
|
||||
),
|
||||
None,
|
||||
)
|
||||
tuning_error: Final = (
|
||||
tuning_quota_violation(candidate=candidate, others=others, baselines=baselines, limit=limit)
|
||||
if baselines is not None
|
||||
else None
|
||||
)
|
||||
capability_labels: Final = {
|
||||
"heuristic_v2": "Heuristic v2",
|
||||
"capability": "Capability",
|
||||
"llm_v2": "Fuse v2",
|
||||
"tier_or_classifier_prompt": "Custom tiers or classifier instructions",
|
||||
}
|
||||
return AutoRouterAvailabilityResponse(
|
||||
allowances=(
|
||||
*allowances,
|
||||
AutoRouterAllowance(
|
||||
key="heuristic_tuning",
|
||||
limit=limit,
|
||||
remaining=None if limit is None or baselines is None else max(0, limit - tuned_count),
|
||||
available=limit is None or baselines is not None,
|
||||
used_by_this_router=bool(
|
||||
existing is not None and baselines is not None and is_mutable_tuned_candidate(existing, baselines)
|
||||
),
|
||||
),
|
||||
),
|
||||
error=(
|
||||
f"{capability_labels[claimed.key]} has no available allowance. Choose another option or free an existing allowance."
|
||||
if capability_error is not None and claimed is not None
|
||||
else "These scoring rules need an available Rule-based tuning allowance. Check the weights, thresholds, keywords, and custom dimensions in Advanced settings. Model choices do not use this allowance."
|
||||
if tuning_error is not None
|
||||
else None
|
||||
),
|
||||
)
|
||||
|
|
@ -132,6 +132,7 @@ from litellm.proxy.common_utils.callback_utils import (
|
|||
strip_callback_config,
|
||||
)
|
||||
from litellm.proxy.common_utils.realtime_utils import _realtime_request_body
|
||||
from litellm.proxy.management_helpers.auto_router_availability import AutoRouterCatalogEntry, build_auto_router_catalog
|
||||
from litellm.router_utils.access_windows import access_windows_config_error
|
||||
from litellm.router_utils.add_retry_fallback_headers import (
|
||||
get_fallback_errors_from_headers,
|
||||
|
|
@ -5068,6 +5069,7 @@ class ProxyConfig:
|
|||
|
||||
def __init__(self) -> None:
|
||||
self.config: Mapping[str, object] = MappingProxyType({})
|
||||
self.auto_router_db_catalog: tuple[AutoRouterCatalogEntry, ...] | None = None
|
||||
self._last_semantic_filter_config: dict[str, object] | None = None
|
||||
self._last_websearch_interception_config: dict[str, object] | None = None
|
||||
self._last_hashicorp_vault_config: dict[str, object] | None = None
|
||||
|
|
@ -7691,6 +7693,12 @@ class ProxyConfig:
|
|||
def _should_load_db_object(self, object_type: str | SupportedDBObjectType) -> bool:
|
||||
return should_load_db_object(object_type=object_type)
|
||||
|
||||
def remove_auto_router_catalog_entries(self, model_ids: frozenset[str]) -> None:
|
||||
if self.auto_router_db_catalog is not None:
|
||||
self.auto_router_db_catalog = tuple(
|
||||
row for row in self.auto_router_db_catalog if row.model_id not in model_ids
|
||||
)
|
||||
|
||||
async def _get_models_from_db(self, prisma_client: PrismaClient) -> Sequence[_ProxyModelRow] | None:
|
||||
"""
|
||||
Fetch all model deployments from the DB.
|
||||
|
|
@ -7711,6 +7719,7 @@ class ProxyConfig:
|
|||
new_models: Final[Sequence[_ProxyModelRow]] = await ModelRepository(
|
||||
WriterPinnedClient(prisma_client.db)
|
||||
).table.find_many()
|
||||
self.auto_router_db_catalog = build_auto_router_catalog(new_models)
|
||||
return new_models
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -1,18 +1,18 @@
|
|||
{
|
||||
"1m_context": {
|
||||
"label": "1M Context",
|
||||
"description": "Routes across models with 1M-token context windows: Luna for simple queries, Terra for medium, Sol for complex, Opus 5 at high thinking for reasoning.",
|
||||
"description": "Routes across models with 1M-token context windows: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, Opus 5.5 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"REASONING": [
|
||||
{
|
||||
"model_name": "claude-opus-5",
|
||||
"model_name": "claude-opus-5-5",
|
||||
"litellm_params": { "reasoning_effort": "high" }
|
||||
}
|
||||
]
|
||||
|
|
@ -28,12 +28,12 @@
|
|||
},
|
||||
"anthropic_family": {
|
||||
"label": "Anthropic Family",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"description": "Routes across the Claude model family: Haiku for simple queries, Sonnet for medium, Opus 5.5 for complex, Fable 5.1 at high thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["claude-haiku-4-5"],
|
||||
"MEDIUM": ["claude-sonnet-5"],
|
||||
"COMPLEX": ["claude-opus-5"],
|
||||
"COMPLEX": ["claude-opus-5-5"],
|
||||
"REASONING": ["claude-fable-5-1"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
@ -55,12 +55,12 @@
|
|||
},
|
||||
"gemini_family": {
|
||||
"label": "Gemini Family",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 2.5 for simple queries, Flash Lite 3.1 for medium, Flash 3.7 for complex, Pro 3.1 for reasoning-heavy requests.",
|
||||
"description": "Routes across the Gemini model family: Flash Lite 3.5 for simple queries, Flash 3.8 for medium and complex queries, Pro 3.1 for reasoning-heavy requests.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gemini-2.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.1-flash-lite"],
|
||||
"COMPLEX": ["gemini-3.7-flash"],
|
||||
"SIMPLE": ["gemini-3.5-flash-lite"],
|
||||
"MEDIUM": ["gemini-3.8-flash"],
|
||||
"COMPLEX": ["gemini-3.8-flash"],
|
||||
"REASONING": ["gemini-3.1-pro-preview"]
|
||||
},
|
||||
"classifier_type": "heuristic",
|
||||
|
|
@ -74,18 +74,18 @@
|
|||
},
|
||||
"lite": {
|
||||
"label": "Lite",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.2 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"description": "Cost-optimized routing across providers: DeepSeek V4 Flash for simple queries, Muse Spark 1.3 at xhigh for medium, Kimi K3 at max for complex, Claude Opus 5.5 for reasoning. An LLM classifier with the agentic rubric assigns tiers.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["deepseek-v4-flash"],
|
||||
"MEDIUM": ["muse-spark-1.2"],
|
||||
"MEDIUM": ["muse-spark-1.3"],
|
||||
"COMPLEX": ["kimi-k3"],
|
||||
"REASONING": ["claude-opus-5"]
|
||||
"REASONING": ["claude-opus-5-5"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
"MEDIUM": [
|
||||
{
|
||||
"model_name": "muse-spark-1.2",
|
||||
"model_name": "muse-spark-1.3",
|
||||
"litellm_params": { "reasoning_effort": "xhigh" }
|
||||
}
|
||||
],
|
||||
|
|
@ -113,12 +113,12 @@
|
|||
},
|
||||
"openai_family": {
|
||||
"label": "OpenAI Family",
|
||||
"description": "Routes across the GPT model family: Luna for simple queries, Terra for medium, Sol for complex, Astra at xhigh thinking for reasoning.",
|
||||
"description": "Routes across the GPT model family: GPT-6 Luna for simple queries, GPT-5.6 Terra for medium, GPT-6 Sol for complex, GPT-6 Astra at xhigh thinking for reasoning.",
|
||||
"complexity_router_config": {
|
||||
"tiers": {
|
||||
"SIMPLE": ["gpt-5.6-luna"],
|
||||
"SIMPLE": ["gpt-6-luna"],
|
||||
"MEDIUM": ["gpt-5.6-terra"],
|
||||
"COMPLEX": ["gpt-5.6-sol"],
|
||||
"COMPLEX": ["gpt-6-sol"],
|
||||
"REASONING": ["gpt-6-astra"]
|
||||
},
|
||||
"tier_model_configs": {
|
||||
|
|
|
|||
|
|
@ -1004,6 +1004,8 @@ def _stamp_deployment_attribution(
|
|||
return attribution
|
||||
metadata.setdefault("model_info", attribution["model_info"])
|
||||
metadata.setdefault("deployment", attribution["deployment"])
|
||||
if isinstance(model_group, str):
|
||||
metadata.setdefault("model_group", model_group)
|
||||
return attribution
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
{
|
||||
"version": "2026-09-17-v1",
|
||||
"version": "2026-09-22-v1",
|
||||
"models": [
|
||||
{
|
||||
"id": "gpt-6-astra-v1",
|
||||
|
|
@ -8,6 +8,20 @@
|
|||
"text": "OpenAI model for demanding end-to-end work, including reasoning, coding, research, and document tasks",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-astra"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-sol-v1",
|
||||
"label": "GPT-6 Sol",
|
||||
"model": "gpt-6-sol",
|
||||
"text": "OpenAI model for complex coding and agentic workflows, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-sol"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-6-luna-v1",
|
||||
"label": "GPT-6 Luna",
|
||||
"model": "gpt-6-luna",
|
||||
"text": "OpenAI model for efficient, high-volume workloads, supporting reasoning and tool calling through the Responses API",
|
||||
"sources": ["https://developers.openai.com/api/docs/models/gpt-6-luna"]
|
||||
},
|
||||
{
|
||||
"id": "gpt-5.6-sol-v1",
|
||||
"label": "GPT-5.6 Sol",
|
||||
|
|
@ -63,6 +77,13 @@
|
|||
"model": "claude-fable-5-1",
|
||||
"text": "Anthropic model for demanding reasoning, long-running agentic coding, and multistep research, with always-on adaptive thinking",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/fable-5-1/overview"]
|
||||
},
|
||||
{
|
||||
"id": "claude-opus-5-5-v1",
|
||||
"label": "Claude Opus 5.5",
|
||||
"model": "claude-opus-5-5",
|
||||
"text": "Anthropic model for complex reasoning and agentic work, supporting adaptive thinking and tool use",
|
||||
"sources": ["https://platform.claude.com/docs/en/models/opus-5-5/overview"]
|
||||
}
|
||||
],
|
||||
"harnesses": [
|
||||
|
|
|
|||
|
|
@ -10,12 +10,10 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm.router_strategy.complexity_router.config import ComplexityRouterConfig
|
||||
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v2"
|
||||
# v2 hashes combine models and scoring rules; a new snapshot is required to separate them.
|
||||
TUNING_BASELINE_PARAM_NAME: Final = "auto_router_tuning_baseline_v3"
|
||||
|
||||
HEURISTIC_V1_TUNING_FIELDS: Final = (
|
||||
"tiers",
|
||||
"tier_model_configs",
|
||||
"classifier_type",
|
||||
"tier_boundaries",
|
||||
"reasoning_override_min_score",
|
||||
"token_thresholds",
|
||||
|
|
@ -49,8 +47,10 @@ def tuning_fingerprint(complexity_router_config: object) -> str | None:
|
|||
validated: Final = ComplexityRouterConfig.model_validate(raw)
|
||||
except ValidationError:
|
||||
return None
|
||||
supplied: Final = ((_TUNING_FIELD_SET - frozenset(("tier_model_configs",))) & frozenset(raw)) | (
|
||||
frozenset(("tier_model_configs",)) if validated.tier_model_configs else frozenset()
|
||||
# The UI always writes this built-in marker. Freeze its spelling so future defaults cannot change recorded hashes.
|
||||
default_escalation: Final = validated.escalation_keywords in (None, ["LITELLM ESCALATE"])
|
||||
supplied: Final = (_TUNING_FIELD_SET & frozenset(raw)) - (
|
||||
frozenset(("escalation_keywords",)) if default_escalation else frozenset()
|
||||
)
|
||||
payload: Final = validated.model_dump(
|
||||
mode="json",
|
||||
|
|
@ -148,9 +148,9 @@ def tuning_limit_violation(*, held: int, limit: int | None) -> str | None:
|
|||
if limit is None or held <= limit:
|
||||
return None
|
||||
return (
|
||||
f"At most {limit} auto-router(s) with changed heuristic scorer settings or tier models can be modified "
|
||||
f"At most {limit} auto-router(s) with changed heuristic scoring rules can be modified "
|
||||
"without an auto-router license. Keep this router on its recorded settings, or revert the other changed "
|
||||
"router to its baseline, or remove one of them."
|
||||
"router to its baseline, or remove one of them. Selecting models does not use this allowance."
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,22 @@ class RustUpstreamError(Exception): ...
|
|||
class ForkedAfterNativeRuntimeStarted(RuntimeError): ...
|
||||
class ProcessReservedForForking(RuntimeError): ...
|
||||
|
||||
@final
|
||||
class NativeDiagnosticProcessor:
|
||||
def __new__(cls, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: Sequence[tuple[str | None, str]],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, list[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: Sequence[str]) -> list[str]: ...
|
||||
|
||||
def ocr(
|
||||
request: LiteLLMOcrRequest,
|
||||
args: tuple[object, ...],
|
||||
|
|
@ -335,6 +351,7 @@ def reserve_process_for_forking() -> None: ...
|
|||
__all__ = [
|
||||
"ForkedAfterNativeRuntimeStarted",
|
||||
"HuggingFaceEncoding",
|
||||
"NativeDiagnosticProcessor",
|
||||
"ProcessReservedForForking",
|
||||
"ResponsesWebSocketConnection",
|
||||
"RustBridgeDeclined",
|
||||
|
|
|
|||
|
|
@ -86,11 +86,25 @@ class SecretManagerRule:
|
|||
return isinstance(context, SecretManagerContext) and (self.systems is None or context.system in self.systems)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerContext:
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LoggerRule:
|
||||
rollout: Rollout
|
||||
|
||||
def matches(self, context: Context) -> bool:
|
||||
return isinstance(context, LoggerContext)
|
||||
|
||||
|
||||
Context: TypeAlias = RouteContext | CacheContext | SecretManagerContext | LoggerContext
|
||||
Rule: TypeAlias = RouteRule | CacheRule | SecretManagerRule | LoggerRule
|
||||
Rules: TypeAlias = tuple[Rule, ...]
|
||||
|
||||
RULES: Final[Rules] = (
|
||||
LoggerRule(Rollout.RUST_OPT_IN),
|
||||
RouteRule(Route.OCR, Rollout.RUST_REQUIRED, providers=frozenset({"aws_textract"})),
|
||||
RouteRule(Route.OCR, Rollout.RUST_OPT_OUT),
|
||||
RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY),
|
||||
|
|
|
|||
59
litellm/rust_bridge/diagnostics.py
Normal file
59
litellm/rust_bridge/diagnostics.py
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Sequence
|
||||
from functools import lru_cache
|
||||
from typing import Final, Protocol, TypeVar, cast
|
||||
|
||||
from litellm.rust_bridge.bindings import NativeBinding
|
||||
|
||||
ResultT: Final = TypeVar("ResultT")
|
||||
|
||||
|
||||
class NativeDiagnosticProcessor(Protocol):
|
||||
def redact_text(self, text: str) -> str: ...
|
||||
def redact_structured_text(self, key: str | None, text: str) -> str: ...
|
||||
def redact_client_message(self, text: str) -> str: ...
|
||||
def process_diagnostic(
|
||||
self,
|
||||
message: str,
|
||||
exception: str | None,
|
||||
stack: str | None,
|
||||
leaves: tuple[tuple[str | None, str], ...],
|
||||
policy: tuple[bool, int, int],
|
||||
) -> tuple[str, str | None, str | None, Sequence[str], bool]: ...
|
||||
def scrub_access_arguments(self, arguments: tuple[str, ...]) -> Sequence[str]: ...
|
||||
|
||||
|
||||
class NativeDiagnosticFactory(Protocol):
|
||||
def __call__(self, minimum_custom_key_length: int) -> NativeDiagnosticProcessor: ...
|
||||
|
||||
|
||||
def _as_factory(value: object) -> NativeDiagnosticFactory | None:
|
||||
if not isinstance(value, type):
|
||||
return None
|
||||
return cast(NativeDiagnosticFactory, value) # cast-ok: PyO3 factory must be a type
|
||||
|
||||
|
||||
PROCESSOR: Final = NativeBinding("NativeDiagnosticProcessor", validate=_as_factory)
|
||||
|
||||
|
||||
@lru_cache(maxsize=4)
|
||||
def _construct(factory: NativeDiagnosticFactory, minimum_custom_key_length: int) -> NativeDiagnosticProcessor:
|
||||
return factory(minimum_custom_key_length)
|
||||
|
||||
|
||||
def run(native: Callable[[NativeDiagnosticProcessor], ResultT], python: Callable[[], ResultT]) -> ResultT:
|
||||
from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH
|
||||
from litellm.rust_bridge.catalog import LoggerContext, decision
|
||||
from litellm.rust_bridge.configuration import Decision
|
||||
|
||||
selected: Final = decision(LoggerContext())
|
||||
if selected is Decision.PYTHON:
|
||||
return python()
|
||||
factory: Final = PROCESSOR.load()
|
||||
if factory is None:
|
||||
return python()
|
||||
try:
|
||||
return native(_construct(factory, MINIMUM_CUSTOM_KEY_LENGTH))
|
||||
except Exception:
|
||||
return python()
|
||||
63
litellm/rust_bridge/logger.py
Normal file
63
litellm/rust_bridge/logger.py
Normal file
|
|
@ -0,0 +1,63 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm._logging import (
|
||||
CorrelationContextFilter,
|
||||
DiagnosticProcessingFilter,
|
||||
session_id_var,
|
||||
set_session_id,
|
||||
set_trace_id,
|
||||
trace_id_var,
|
||||
verbose_logger,
|
||||
)
|
||||
|
||||
_REDACTION: Final = DiagnosticProcessingFilter()
|
||||
_CORRELATION: Final = CorrelationContextFilter()
|
||||
|
||||
|
||||
def context() -> tuple[str, str]:
|
||||
return session_id_var.get(), trace_id_var.get()
|
||||
|
||||
|
||||
def enabled(level: int) -> bool:
|
||||
return verbose_logger.isEnabledFor(level)
|
||||
|
||||
|
||||
def emit(
|
||||
level: int,
|
||||
message: str,
|
||||
pathname: str,
|
||||
lineno: int,
|
||||
target: str,
|
||||
fields: Mapping[str, JsonValue],
|
||||
correlation: tuple[str, str],
|
||||
) -> None:
|
||||
if not enabled(level):
|
||||
return
|
||||
session_token: Final = set_session_id(correlation[0])
|
||||
trace_token: Final = set_trace_id(correlation[1])
|
||||
try:
|
||||
record: Final = verbose_logger.makeRecord(
|
||||
verbose_logger.name,
|
||||
level,
|
||||
pathname,
|
||||
lineno,
|
||||
message,
|
||||
(),
|
||||
None,
|
||||
func=target,
|
||||
extra={
|
||||
"rust_target": target,
|
||||
"rust_fields": dict(fields),
|
||||
}, # mutable-ok: LogRecord requires JSON dict extras
|
||||
)
|
||||
_REDACTION.filter(record)
|
||||
_CORRELATION.filter(record)
|
||||
verbose_logger.handle(record)
|
||||
finally:
|
||||
trace_id_var.reset(trace_token)
|
||||
session_id_var.reset(session_token)
|
||||
|
|
@ -58,12 +58,6 @@ class SecretManagerBinding:
|
|||
settings_object: object
|
||||
|
||||
|
||||
def warn(message: str) -> None:
|
||||
from litellm._logging import verbose_logger
|
||||
|
||||
verbose_logger.warning("%s", message)
|
||||
|
||||
|
||||
def secret_manager() -> SecretManager:
|
||||
from litellm.secret_managers.main import (
|
||||
_should_read_secret_from_secret_manager, # pyright: ignore[reportPrivateUsage] # canonical resolver is private
|
||||
|
|
|
|||
|
|
@ -44,6 +44,25 @@ class ComplexityRouterConfigValidationResponse(BaseModel):
|
|||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterAvailabilityRequest(BaseModel):
|
||||
team_id: str | None = None
|
||||
saved_model_id: str | None = None
|
||||
complexity_router_config: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class AutoRouterAllowance(BaseModel):
|
||||
key: str
|
||||
limit: int | None
|
||||
remaining: int | None
|
||||
used_by_this_router: bool = False
|
||||
available: bool = True
|
||||
|
||||
|
||||
class AutoRouterAvailabilityResponse(BaseModel):
|
||||
allowances: tuple[AutoRouterAllowance, ...]
|
||||
error: str | None = None
|
||||
|
||||
|
||||
class AutoRouterRoutingTestRequest(BaseModel):
|
||||
"""A single request to classify against a complexity-router config that need not be saved yet.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:e624c5d5e42382ce7165ddafcbbf8e6769a24cbd02ea6114b880b05ae5ba2a8d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm"
|
||||
version = "1.103.0"
|
||||
version = "1.104.0"
|
||||
description = "Library to easily interface with LLM API providers"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10, <3.15"
|
||||
|
|
@ -75,8 +75,8 @@ proxy = [
|
|||
"mcp>=2.2.0,<3",
|
||||
"httpx2>=2.5.0,<3",
|
||||
"pydantic>=2.12.0,<3",
|
||||
"litellm-proxy-extras==0.4.100",
|
||||
"litellm-enterprise==0.1.69",
|
||||
"litellm-proxy-extras==0.4.101",
|
||||
"litellm-enterprise==0.1.70",
|
||||
"RestrictedPython>=8.5,<9.0",
|
||||
"rich>=13.9.4,<14.0",
|
||||
"InquirerPy>=0.3.4,<1.0",
|
||||
|
|
@ -355,7 +355,7 @@ litellm-enterprise = { workspace = true }
|
|||
members = ["enterprise", "litellm-proxy-extras"]
|
||||
|
||||
[tool.commitizen]
|
||||
version = "1.103.0"
|
||||
version = "1.104.0"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -3,62 +3,140 @@ import { describe, expect, test } from "bun:test";
|
|||
import type { Comment, GitHubApi } from "./auto-close-duplicates";
|
||||
import {
|
||||
FIXED_MARKER,
|
||||
OPEN_PULL_REQUESTS_QUERY,
|
||||
SUPERSEDED_MARKER,
|
||||
closeVerdict,
|
||||
closerOf,
|
||||
commentFixedIssue,
|
||||
describeIssue,
|
||||
describeSweep,
|
||||
fixOf,
|
||||
fixedBody,
|
||||
handleFixedIssue,
|
||||
nextMinor,
|
||||
parseVersion,
|
||||
placement,
|
||||
readConfig,
|
||||
releaseCandidate,
|
||||
supersededBody,
|
||||
sweep,
|
||||
type ClosedIssue,
|
||||
type FixedConfig,
|
||||
type IssueClosure,
|
||||
type LinkedIssue,
|
||||
type LinkedPullRequest,
|
||||
type PullRequestsPage,
|
||||
} from "./comment-fixed-issue";
|
||||
|
||||
const MERGE_COMMIT = "68c4c82ac977b48b2b81ee8d633d5771307c6162";
|
||||
const ISSUE = 41750;
|
||||
|
||||
const mergedPr = {
|
||||
__typename: "PullRequest" as const,
|
||||
number: 41767,
|
||||
merged: true,
|
||||
baseRefName: "main",
|
||||
repository: { nameWithOwner: "BerriAI/litellm" },
|
||||
mergeCommit: { oid: MERGE_COMMIT },
|
||||
};
|
||||
|
||||
type Closer = ClosedIssue["timelineItems"]["nodes"][number]["closer"];
|
||||
const commitCloser = { __typename: "Commit" as const, oid: MERGE_COMMIT, repository: { nameWithOwner: "BerriAI/litellm" } };
|
||||
|
||||
const closedBy = (closer: Closer, state: ClosedIssue["state"] = "CLOSED"): ClosedIssue => ({
|
||||
type Closer = IssueClosure["timelineItems"]["nodes"][number]["closer"];
|
||||
|
||||
const closure = (closer: Closer, state: IssueClosure["state"] = "CLOSED"): IssueClosure => ({
|
||||
state,
|
||||
timelineItems: { nodes: [{ closer }] },
|
||||
});
|
||||
|
||||
const page = (pages: readonly (readonly LinkedPullRequest[])[], index: number): PullRequestsPage => ({
|
||||
pageInfo: { hasNextPage: index + 1 < pages.length, endCursor: String(index + 1) },
|
||||
nodes: pages[index] ?? [],
|
||||
});
|
||||
|
||||
const cursorIndex = (after: string | null): number => (after === null ? 0 : Number(after));
|
||||
|
||||
const closedBy = (closer: Closer, state?: IssueClosure["state"], linked: readonly LinkedPullRequest[] = []): ClosedIssue => ({
|
||||
...closure(closer, state),
|
||||
closedByPullRequestsReferences: page([linked], 0),
|
||||
});
|
||||
|
||||
const reopenedAt = (createdAt: string): LinkedPullRequest["reopens"] => ({ nodes: [{ createdAt }] });
|
||||
|
||||
const linkedIssue = (number: number, closer: Closer = mergedPr, state?: IssueClosure["state"]): LinkedIssue => ({
|
||||
number,
|
||||
repository: { nameWithOwner: "BerriAI/litellm" },
|
||||
...closure(closer, state),
|
||||
});
|
||||
|
||||
const links = (...issues: readonly LinkedIssue[]): LinkedPullRequest["closingIssuesReferences"] => ({ totalCount: issues.length, nodes: issues });
|
||||
|
||||
const openPr = (number: number, overrides: Partial<LinkedPullRequest> = {}): LinkedPullRequest => ({
|
||||
number,
|
||||
state: "OPEN",
|
||||
baseRefName: "main",
|
||||
repository: { nameWithOwner: "BerriAI/litellm" },
|
||||
closingIssuesReferences: links(linkedIssue(ISSUE)),
|
||||
reopens: { nodes: [] },
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const pyproject = (version: string): string =>
|
||||
`[project]\nname = "litellm"\nversion = "${version}"\n\n[tool.commitizen]\nversion = "${version}"\n`;
|
||||
|
||||
const config: FixedConfig = { repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false };
|
||||
const config: FixedConfig = { repo: "BerriAI/litellm", defaultBranch: "main", commentDryRun: false, closeDryRun: false };
|
||||
|
||||
const prFix = (issue = ISSUE, number = 41767) => ({ issue, source: { kind: "pull_request" as const, number, oid: MERGE_COMMIT } });
|
||||
const commitFix = (issue = ISSUE) => ({ issue, source: { kind: "commit" as const, oid: MERGE_COMMIT } });
|
||||
const oneFixBody = supersededBody([prFix()], "main");
|
||||
|
||||
const noPause = async (): Promise<void> => {};
|
||||
|
||||
const supersededComment: Comment = {
|
||||
id: 2,
|
||||
body: `${SUPERSEDED_MARKER}\n#41750 was fixed by #41767 on main, so this pull request is closed. Reopen it if something was missed.`,
|
||||
created_at: "2026-09-18T00:00:00Z",
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
};
|
||||
|
||||
interface World {
|
||||
readonly issue?: ClosedIssue | null;
|
||||
readonly comments?: readonly Comment[];
|
||||
readonly pullRequestComments?: Readonly<Record<number, readonly Comment[]>>;
|
||||
readonly openPullRequests?: readonly (readonly LinkedPullRequest[])[];
|
||||
readonly linkedPullRequests?: readonly (readonly LinkedPullRequest[])[];
|
||||
readonly version?: string;
|
||||
// Which existing rc.1 tags contain the merge commit; a tag absent from the map does not exist
|
||||
readonly tags?: Readonly<Record<string, boolean>>;
|
||||
readonly reachable?: Readonly<Record<string, readonly string[]>>;
|
||||
}
|
||||
|
||||
function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes: string[] } {
|
||||
const writes: string[] = [];
|
||||
const tags = world.tags ?? {};
|
||||
const reachable = world.reachable ?? { main: [MERGE_COMMIT] };
|
||||
const pages = world.openPullRequests ?? [];
|
||||
const api: GitHubApi = {
|
||||
request: async <T>(method: string, path: string, body?: object): Promise<T> => {
|
||||
if (method === "POST" && path === "/graphql") {
|
||||
return { data: { repository: { issue: world.issue === undefined ? closedBy(mergedPr) : world.issue } } } as T;
|
||||
const { query, variables } = body as { query: string; variables: { after: string | null } };
|
||||
if (query === OPEN_PULL_REQUESTS_QUERY) {
|
||||
return { data: { repository: { pullRequests: page(pages, cursorIndex(variables.after)) } } } as T;
|
||||
}
|
||||
const issue = world.issue === undefined ? closedBy(mergedPr) : world.issue;
|
||||
if (issue === null || world.linkedPullRequests === undefined) {
|
||||
return { data: { repository: { issue } } } as T;
|
||||
}
|
||||
const closedByPullRequestsReferences = page(world.linkedPullRequests, cursorIndex(variables.after));
|
||||
return { data: { repository: { issue: { ...issue, closedByPullRequestsReferences } } } } as T;
|
||||
}
|
||||
if (method !== "GET") {
|
||||
writes.push(`${method} ${path} ${JSON.stringify(body)}`);
|
||||
return {} as T;
|
||||
}
|
||||
if (path.startsWith("/repos/BerriAI/litellm/issues/41750/comments")) {
|
||||
return (world.comments ?? []) as T;
|
||||
const comments = /^\/repos\/BerriAI\/litellm\/issues\/(\d+)\/comments/.exec(path);
|
||||
if (comments !== null) {
|
||||
const number = Number(comments[1]);
|
||||
return ((number === ISSUE ? world.comments : world.pullRequestComments?.[number]) ?? []) as T;
|
||||
}
|
||||
if (path === `/repos/BerriAI/litellm/contents/pyproject.toml?ref=${MERGE_COMMIT}`) {
|
||||
return { content: btoa(pyproject(world.version ?? "1.103.0")).replace(/(.{60})/g, "$1\n") } as T;
|
||||
|
|
@ -68,6 +146,10 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes:
|
|||
return (matching[1] in tags ? [{ ref: `refs/tags/${matching[1]}` }] : []) as T;
|
||||
}
|
||||
const compare = /^\/repos\/BerriAI\/litellm\/compare\/(.+)\.\.\.(.+)$/.exec(path);
|
||||
const branch = compare === null ? undefined : reachable[compare[1] ?? ""];
|
||||
if (compare !== null && branch !== undefined) {
|
||||
return { status: branch.includes(compare[2] ?? "") ? "behind" : "diverged" } as T;
|
||||
}
|
||||
if (compare !== null && compare[2] === MERGE_COMMIT) {
|
||||
return { status: tags[compare[1]] ? "behind" : "ahead" } as T;
|
||||
}
|
||||
|
|
@ -79,23 +161,28 @@ function fakeApi(world: World = {}): { readonly api: GitHubApi; readonly writes:
|
|||
|
||||
describe("closerOf", () => {
|
||||
test("a pull request merged into the default branch is the fix", () => {
|
||||
expect(closerOf(closedBy(mergedPr), "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT });
|
||||
expect(closerOf(closedBy(mergedPr), "BerriAI/litellm", "main")).toEqual({ kind: "pull_request", number: 41767, mergeCommit: MERGE_COMMIT });
|
||||
});
|
||||
|
||||
test("an issue closed by hand, by a commit, or by an unmerged pull request gets no comment", () => {
|
||||
expect(closerOf(closedBy(null), "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
|
||||
expect(closerOf(closedBy({ __typename: "Commit", oid: MERGE_COMMIT }), "main").kind).toBe("skip");
|
||||
expect(closerOf(closedBy({ ...mergedPr, merged: false }), "main").kind).toBe("skip");
|
||||
expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "main").kind).toBe("skip");
|
||||
expect(closerOf(closedBy(null), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
|
||||
expect(closerOf(closedBy(commitCloser), "BerriAI/litellm", "main").kind).toBe("skip");
|
||||
expect(closerOf(closedBy({ ...mergedPr, merged: false }), "BerriAI/litellm", "main").kind).toBe("skip");
|
||||
expect(closerOf(closedBy({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm", "main").kind).toBe("skip");
|
||||
});
|
||||
|
||||
test("a pull request merged into a release branch is not a fix on main", () => {
|
||||
const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "main");
|
||||
const verdict = closerOf(closedBy({ ...mergedPr, baseRefName: "release/1.102.0rc2" }), "BerriAI/litellm", "main");
|
||||
expect(verdict).toEqual({ kind: "skip", reason: "#41767 merged into release/1.102.0rc2, not main" });
|
||||
});
|
||||
|
||||
test("an issue reopened after the close event is left alone", () => {
|
||||
expect(closerOf(closedBy(mergedPr, "OPEN"), "main")).toEqual({ kind: "skip", reason: "the issue is open again" });
|
||||
expect(closerOf(closedBy(mergedPr, "OPEN"), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "the issue is open again" });
|
||||
});
|
||||
|
||||
test("a pull request merged in a fork closes the issue on GitHub but is no fix here", () => {
|
||||
const forkPr = { ...mergedPr, number: 9, repository: { nameWithOwner: "someone/litellm" } };
|
||||
expect(closerOf(closedBy(forkPr), "BerriAI/litellm", "main")).toEqual({ kind: "skip", reason: "closed by someone/litellm#9, a pull request in another repository" });
|
||||
});
|
||||
});
|
||||
|
||||
|
|
@ -173,44 +260,327 @@ describe("fixedBody", () => {
|
|||
});
|
||||
});
|
||||
|
||||
describe("commentFixedIssue", () => {
|
||||
describe("fixOf", () => {
|
||||
test("a merged pull request or a commit is a fix, whatever branch it was merged into", () => {
|
||||
expect(fixOf(closure(mergedPr), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT });
|
||||
expect(fixOf(closure({ ...mergedPr, baseRefName: "litellm_internal_staging" }), "BerriAI/litellm")).toEqual({ kind: "pull_request", number: 41767, oid: MERGE_COMMIT });
|
||||
expect(fixOf(closure(commitCloser), "BerriAI/litellm")).toEqual({ kind: "commit", oid: MERGE_COMMIT });
|
||||
});
|
||||
|
||||
test("a hand close, a fork's pull request, an unmerged pull request, or a reopened issue is no fix", () => {
|
||||
expect(fixOf(closure(null), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by hand" });
|
||||
expect(fixOf(closure({ ...mergedPr, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" });
|
||||
expect(fixOf(closure({ ...commitCloser, repository: { nameWithOwner: "someone/litellm" } }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed from someone/litellm" });
|
||||
expect(fixOf(closure({ ...mergedPr, merged: false }), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "was closed by #41767, which is not merged" });
|
||||
expect(fixOf(closure({ ...mergedPr, mergeCommit: null }), "BerriAI/litellm").kind).toBe("skip");
|
||||
expect(fixOf(closure(mergedPr, "OPEN"), "BerriAI/litellm")).toEqual({ kind: "skip", reason: "is open again" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("closeVerdict", () => {
|
||||
test("an open pull request whose only linked issue was fixed by a merged pull request is a candidate", () => {
|
||||
expect(closeVerdict(openPr(41760), config)).toEqual({ kind: "candidate", fixes: [prFix()] });
|
||||
});
|
||||
|
||||
test("every linked issue has to be fixed, and each fix is named", () => {
|
||||
const both = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, commitCloser)) });
|
||||
expect(closeVerdict(both, config)).toEqual({ kind: "candidate", fixes: [prFix(), commitFix(41751)] });
|
||||
});
|
||||
|
||||
test("a pull request that still links an open issue keeps its work", () => {
|
||||
const stillOpen = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) });
|
||||
expect(closeVerdict(stillOpen, config)).toEqual({ kind: "skip", reason: "still linked to open #41751" });
|
||||
});
|
||||
|
||||
test("a linked issue closed by hand or by an unmerged pull request is not a fix that supersedes the pull request", () => {
|
||||
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) }), config)).toEqual({
|
||||
kind: "skip",
|
||||
reason: `#${ISSUE} was closed by hand`,
|
||||
});
|
||||
const unmerged = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, { ...mergedPr, merged: false })) });
|
||||
expect(closeVerdict(unmerged, config)).toEqual({ kind: "skip", reason: `#${ISSUE} was closed by #41767, which is not merged` });
|
||||
});
|
||||
|
||||
test("a pull request linking an issue in another repository, or more issues than the query reads, is left alone", () => {
|
||||
const foreign = { ...linkedIssue(41751), repository: { nameWithOwner: "mlflow/mlflow" } };
|
||||
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE), foreign) }), config)).toEqual({
|
||||
kind: "skip",
|
||||
reason: "links mlflow/mlflow#41751",
|
||||
});
|
||||
const truncated = openPr(41760, { closingIssuesReferences: { totalCount: 11, nodes: [linkedIssue(ISSUE)] } });
|
||||
expect(closeVerdict(truncated, config)).toEqual({ kind: "skip", reason: "links 11 issues, more than the 10 this workflow reads" });
|
||||
});
|
||||
|
||||
test("a pull request against a release line is a backport and stays open, one against a retired development branch does not", () => {
|
||||
for (const base of ["release/v1.102.0-rc.2", "stable/v1.83.14", "v_1_83_3_stable_patch"]) {
|
||||
expect(closeVerdict(openPr(41760, { baseRefName: base }), config)).toEqual({ kind: "skip", reason: `targets the release line ${base}` });
|
||||
}
|
||||
expect(closeVerdict(openPr(41760, { baseRefName: "litellm_internal_staging" }), config).kind).toBe("candidate");
|
||||
});
|
||||
|
||||
test("a pull request in a fork, one that is not open, or one linking no issue is left alone", () => {
|
||||
expect(closeVerdict(openPr(1, { repository: { nameWithOwner: "someone/litellm" } }), config)).toEqual({ kind: "skip", reason: "lives in someone/litellm" });
|
||||
expect(closeVerdict(openPr(41767, { state: "MERGED" }), config)).toEqual({ kind: "skip", reason: "is merged" });
|
||||
expect(closeVerdict(openPr(41760, { closingIssuesReferences: links() }), config)).toEqual({ kind: "skip", reason: "links no issue" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("supersededBody", () => {
|
||||
test("names each issue with the pull request or commit that fixed it, invites a reopen, and carries the marker the rerun looks for", () => {
|
||||
expect(oneFixBody.startsWith(SUPERSEDED_MARKER)).toBe(true);
|
||||
expect(oneFixBody).toContain("#41750 was fixed by #41767 on main");
|
||||
expect(oneFixBody).toContain("Reopen it");
|
||||
expect(supersededBody([prFix(), commitFix(41751)], "main")).toContain("#41750 was fixed by #41767 and #41751 by commit 68c4c82ac9 on main");
|
||||
});
|
||||
|
||||
test("stays within the 25-word comment rule for one and two fixes", () => {
|
||||
for (const fixes of [[prFix()], [commitFix()], [prFix(), commitFix(41751)]]) {
|
||||
const words = supersededBody(fixes, "main").replace(SUPERSEDED_MARKER, "").trim().split(/\s+/);
|
||||
expect(words.length).toBeGreaterThanOrEqual(15);
|
||||
expect(words.length).toBeLessThanOrEqual(25);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleFixedIssue", () => {
|
||||
test("a real run posts one comment naming the pull request and the release", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await commentFixedIssue(api, config);
|
||||
expect(verdict).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" });
|
||||
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(comment).toMatchObject({ kind: "commented", pullRequest: 41767, tag: "v1.103.0-rc.1" });
|
||||
expect(pullRequests).toEqual([]);
|
||||
expect(writes).toHaveLength(1);
|
||||
expect(writes[0]).toContain("POST /repos/BerriAI/litellm/issues/41750/comments");
|
||||
expect(writes[0]).toContain("Fixed by #41767. This ships in v1.103.0-rc.1 and up");
|
||||
});
|
||||
|
||||
test("a dry run renders the comment and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi();
|
||||
const verdict = await commentFixedIssue(api, { ...config, dryRun: true });
|
||||
expect(verdict.kind).toBe("commented");
|
||||
expect(writes).toEqual([]);
|
||||
test("every other open pull request linked to the fixed issue is commented on and then closed, a pause before every write", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760), openPr(41761)]) });
|
||||
let pauses = 0;
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, async () => {
|
||||
pauses += 1;
|
||||
});
|
||||
expect(pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed", "closed"]);
|
||||
expect(writes).toEqual([
|
||||
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41750/comments"),
|
||||
`POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`,
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
|
||||
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"),
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
|
||||
]);
|
||||
expect(pauses).toBe(4);
|
||||
});
|
||||
|
||||
test("an issue that already carries the comment is not commented twice", async () => {
|
||||
test("the fixed-in comment and the closing are gated separately: a close dry run still posts the comment and closes nothing", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
|
||||
let pauses = 0;
|
||||
const outcome = await handleFixedIssue(api, { ...config, closeDryRun: true }, ISSUE, async () => {
|
||||
pauses += 1;
|
||||
});
|
||||
expect(outcome.comment.kind).toBe("commented");
|
||||
expect(outcome.pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
|
||||
expect(writes).toHaveLength(1);
|
||||
expect(writes[0]).toContain("/issues/41750/comments");
|
||||
expect(pauses).toBe(0);
|
||||
});
|
||||
|
||||
test("a comment dry run still closes the pull requests for real", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
|
||||
const outcome = await handleFixedIssue(api, { ...config, commentDryRun: true }, ISSUE, noPause);
|
||||
expect(outcome.comment.kind).toBe("commented");
|
||||
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
|
||||
});
|
||||
|
||||
test("an issue that already carries the fixed-in comment is not commented twice, and its linked pull requests still get closed", async () => {
|
||||
const existing: Comment = {
|
||||
id: 1,
|
||||
body: `${FIXED_MARKER}\nFixed by #41767. This ships in v1.103.0-rc.1 and up.`,
|
||||
created_at: "2026-09-18T00:00:00Z",
|
||||
user: { type: "Bot", login: "github-actions[bot]" },
|
||||
};
|
||||
const { api, writes } = fakeApi({ comments: [existing] });
|
||||
expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" });
|
||||
const { api, writes } = fakeApi({ comments: [existing], issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]) });
|
||||
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(outcome.comment).toEqual({ kind: "skip", reason: "already carries a fixed-in comment" });
|
||||
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
|
||||
});
|
||||
|
||||
test("a fix merged into the retired development branch or pushed as a commit counts once it is on the default branch", async () => {
|
||||
const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" };
|
||||
const staging = openPr(41760, { baseRefName: "litellm_internal_staging", closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) });
|
||||
const byCommit = openPr(41761, { closingIssuesReferences: links(linkedIssue(41751, commitCloser)) });
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [staging, byCommit]) });
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([
|
||||
{ kind: "closed", number: 41760, body: oneFixBody },
|
||||
{ kind: "closed", number: 41761, body: supersededBody([commitFix(41751)], "main") },
|
||||
]);
|
||||
expect(writes).toHaveLength(5);
|
||||
expect(writes[3]).toContain("#41751 was fixed by commit 68c4c82ac9 on main");
|
||||
});
|
||||
|
||||
test("a fix whose commit never reached the default branch supersedes nothing", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41760)]), reachable: { main: [] } });
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([{ kind: "skip", number: 41760, reason: "#41750 was fixed by #41767, which is not on main" }]);
|
||||
expect(writes).toHaveLength(1);
|
||||
expect(writes[0]).toContain("/issues/41750/comments");
|
||||
});
|
||||
|
||||
test("the default branch comes from the config for the containment check and the comment alike", async () => {
|
||||
const stagingConfig = { ...config, defaultBranch: "litellm_internal_staging" };
|
||||
const closer = { ...mergedPr, baseRefName: "litellm_internal_staging" };
|
||||
const { api, writes } = fakeApi({
|
||||
issue: closedBy(closer, "CLOSED", [openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, closer)) })]),
|
||||
reachable: { litellm_internal_staging: [MERGE_COMMIT] },
|
||||
});
|
||||
const { pullRequests } = await handleFixedIssue(api, stagingConfig, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([prFix()], "litellm_internal_staging") }]);
|
||||
expect(writes[1]).toContain("on litellm_internal_staging, so this pull request is closed");
|
||||
});
|
||||
|
||||
test("the closer sits in the linked list as merged and gets neither a line nor a write", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [openPr(41767, { state: "MERGED" }), openPr(41760)]) });
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
|
||||
expect(writes.map((write) => write.split(" ")[1])).toEqual([
|
||||
"/repos/BerriAI/litellm/issues/41750/comments",
|
||||
"/repos/BerriAI/litellm/issues/41760/comments",
|
||||
"/repos/BerriAI/litellm/pulls/41760",
|
||||
]);
|
||||
});
|
||||
|
||||
test("a pull request this workflow closed once and its author reopened stays open", async () => {
|
||||
const reopened = openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") });
|
||||
const { api, writes } = fakeApi({
|
||||
issue: closedBy(mergedPr, "CLOSED", [reopened, openPr(41761)]),
|
||||
pullRequestComments: { 41760: [supersededComment] },
|
||||
});
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests[0]).toEqual({ kind: "skip", number: 41760, reason: "was closed by this workflow once and reopened" });
|
||||
expect(pullRequests[1]?.kind).toBe("closed");
|
||||
expect(writes.filter((write) => write.includes("41760"))).toEqual([]);
|
||||
});
|
||||
|
||||
test("a pull request whose comment landed but whose close failed is closed on the next run without a second comment", async () => {
|
||||
const reopenedBeforeTheComment = openPr(41761, { reopens: reopenedAt("2026-09-17T00:00:00Z") });
|
||||
const { api, writes } = fakeApi({
|
||||
issue: closedBy(mergedPr, "CLOSED", [openPr(41760), reopenedBeforeTheComment]),
|
||||
pullRequestComments: { 41760: [supersededComment], 41761: [supersededComment] },
|
||||
});
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([
|
||||
{ kind: "closed", number: 41760, body: supersededComment.body },
|
||||
{ kind: "closed", number: 41761, body: supersededComment.body },
|
||||
]);
|
||||
expect(writes.filter((write) => write.includes("/4176"))).toEqual([
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
|
||||
]);
|
||||
});
|
||||
|
||||
test("a superseded marker pasted by anyone but the workflow neither keeps a pull request open nor replaces its comment", async () => {
|
||||
const forged: Comment = { ...supersededComment, id: 3, user: { type: "User", login: "someone" } };
|
||||
const { api, writes } = fakeApi({
|
||||
issue: closedBy(mergedPr, "CLOSED", [openPr(41760, { reopens: reopenedAt("2026-09-19T00:00:00Z") })]),
|
||||
pullRequestComments: { 41760: [forged] },
|
||||
});
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
|
||||
expect(writes.filter((write) => write.includes("/41760"))).toEqual([
|
||||
`POST /repos/BerriAI/litellm/issues/41760/comments ${JSON.stringify({ body: oneFixBody })}`,
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
|
||||
]);
|
||||
});
|
||||
|
||||
test("every page of linked pull requests is read, not just the first", async () => {
|
||||
const { api } = fakeApi({ linkedPullRequests: [[openPr(41760)], [openPr(41761)], [openPr(41762)]] });
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([
|
||||
{ kind: "closed", number: 41760, body: oneFixBody },
|
||||
{ kind: "closed", number: 41761, body: oneFixBody },
|
||||
{ kind: "closed", number: 41762, body: oneFixBody },
|
||||
]);
|
||||
});
|
||||
|
||||
test("a linked pull request from a fork or one still tied to another open issue is reported, not closed", async () => {
|
||||
const fork = openPr(1, { repository: { nameWithOwner: "someone/litellm" } });
|
||||
const busy = openPr(41762, { closingIssuesReferences: links(linkedIssue(ISSUE), linkedIssue(41751, null, "OPEN")) });
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "CLOSED", [fork, busy]) });
|
||||
const { pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(pullRequests).toEqual([
|
||||
{ kind: "skip", number: 1, reason: "lives in someone/litellm" },
|
||||
{ kind: "skip", number: 41762, reason: "still linked to open #41751" },
|
||||
]);
|
||||
expect(writes).toHaveLength(1);
|
||||
});
|
||||
|
||||
test("a hand-closed issue gets no comment and leaves its linked pull requests open with the reason on each", async () => {
|
||||
const byHand = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, null)) });
|
||||
const { api, writes } = fakeApi({ issue: closedBy(null, "CLOSED", [byHand]) });
|
||||
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(outcome.comment).toEqual({ kind: "skip", reason: "closed by hand, not by a pull request" });
|
||||
expect(outcome.pullRequests).toEqual([{ kind: "skip", number: 41760, reason: `#${ISSUE} was closed by hand` }]);
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a hand-closed issue never reaches the release lookup or the API writes", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(null) });
|
||||
expect((await commentFixedIssue(api, config)).kind).toBe("skip");
|
||||
test("an issue closed by a commit on the default branch gets no comment but still closes its linked pull requests", async () => {
|
||||
const byCommit = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, commitCloser)) });
|
||||
const { api, writes } = fakeApi({ issue: closedBy(commitCloser, "CLOSED", [byCommit]) });
|
||||
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(comment).toEqual({ kind: "skip", reason: "closed by commit 68c4c82ac9, not by a pull request" });
|
||||
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: supersededBody([commitFix()], "main") }]);
|
||||
expect(writes).toEqual([expect.stringContaining("/issues/41760/comments"), 'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}']);
|
||||
expect(writes[0]).toContain("#41750 was fixed by commit 68c4c82ac9 on main");
|
||||
});
|
||||
|
||||
test("an issue closed from the retired development branch gets no comment but still closes its linked pull requests once the fix is on the default branch", async () => {
|
||||
const stagingPr = { ...mergedPr, baseRefName: "litellm_internal_staging" };
|
||||
const staging = openPr(41760, { closingIssuesReferences: links(linkedIssue(ISSUE, stagingPr)) });
|
||||
const { api, writes } = fakeApi({ issue: closedBy(stagingPr, "CLOSED", [staging]) });
|
||||
const { comment, pullRequests } = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(comment).toEqual({ kind: "skip", reason: "#41767 merged into litellm_internal_staging, not main" });
|
||||
expect(pullRequests).toEqual([{ kind: "closed", number: 41760, body: oneFixBody }]);
|
||||
expect(writes.map((write) => write.split(" ")[1])).toEqual(["/repos/BerriAI/litellm/issues/41760/comments", "/repos/BerriAI/litellm/pulls/41760"]);
|
||||
});
|
||||
|
||||
test("an issue that is open again gets no comment and its linked pull requests are neither read nor touched", async () => {
|
||||
const { api, writes } = fakeApi({ issue: closedBy(mergedPr, "OPEN", [openPr(41760)]) });
|
||||
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(outcome).toEqual({ comment: { kind: "skip", reason: "the issue is open again" }, pullRequests: [] });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
|
||||
test("a number that is not an issue in the repository is a skip", async () => {
|
||||
const { api, writes } = fakeApi({ issue: null });
|
||||
expect(await commentFixedIssue(api, config)).toEqual({ kind: "skip", reason: "not an issue in this repository" });
|
||||
const outcome = await handleFixedIssue(api, config, ISSUE, noPause);
|
||||
expect(outcome.comment).toEqual({ kind: "skip", reason: "not an issue in this repository" });
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("sweep", () => {
|
||||
test("walks every page of open pull requests and closes the ones whose linked issues were all fixed", async () => {
|
||||
const unlinked = openPr(41700, { closingIssuesReferences: links() });
|
||||
const busy = openPr(41701, { closingIssuesReferences: links(linkedIssue(41751, null, "OPEN")) });
|
||||
const { api, writes } = fakeApi({ openPullRequests: [[unlinked, openPr(41760)], [busy, openPr(41761)]] });
|
||||
const outcome = await sweep(api, { ...config, commentDryRun: true }, noPause);
|
||||
expect(outcome.considered).toBe(4);
|
||||
expect(outcome.pullRequests).toEqual([
|
||||
{ kind: "closed", number: 41760, body: oneFixBody },
|
||||
{ kind: "skip", number: 41701, reason: "still linked to open #41751" },
|
||||
{ kind: "closed", number: 41761, body: oneFixBody },
|
||||
]);
|
||||
expect(writes).toEqual([
|
||||
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41760/comments"),
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41760 {"state":"closed"}',
|
||||
expect.stringContaining("POST /repos/BerriAI/litellm/issues/41761/comments"),
|
||||
'PATCH /repos/BerriAI/litellm/pulls/41761 {"state":"closed"}',
|
||||
]);
|
||||
});
|
||||
|
||||
test("a sweep dry run lists what it would close and writes nothing", async () => {
|
||||
const { api, writes } = fakeApi({ openPullRequests: [[openPr(41760)]] });
|
||||
const outcome = await sweep(api, { ...config, closeDryRun: true }, noPause);
|
||||
expect(outcome.pullRequests.map((pullRequest) => pullRequest.kind)).toEqual(["closed"]);
|
||||
expect(writes).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
|
@ -218,10 +588,24 @@ describe("commentFixedIssue", () => {
|
|||
describe("readConfig", () => {
|
||||
const env = { GITHUB_TOKEN: "t", GITHUB_REPOSITORY: "BerriAI/litellm", ISSUE_NUMBER: "41750", DEFAULT_BRANCH: "main" };
|
||||
|
||||
test("reads the four inputs and treats anything but the literal true as a real run", () => {
|
||||
expect(readConfig(env)).toEqual({ token: "t", repo: "BerriAI/litellm", issueNumber: 41750, defaultBranch: "main", dryRun: false });
|
||||
expect(readConfig({ ...env, DRY_RUN: "true" }).dryRun).toBe(true);
|
||||
expect(readConfig({ ...env, DRY_RUN: "false" }).dryRun).toBe(false);
|
||||
test("reads the inputs and treats anything but the literal true as a real run for each gate", () => {
|
||||
expect(readConfig(env)).toEqual({
|
||||
token: "t",
|
||||
repo: "BerriAI/litellm",
|
||||
defaultBranch: "main",
|
||||
commentDryRun: false,
|
||||
closeDryRun: false,
|
||||
run: { kind: "issue", number: 41750 },
|
||||
});
|
||||
expect(readConfig({ ...env, DRY_RUN: "true" })).toMatchObject({ commentDryRun: true, closeDryRun: false });
|
||||
expect(readConfig({ ...env, CLOSE_PRS_DRY_RUN: "true" })).toMatchObject({ commentDryRun: false, closeDryRun: true });
|
||||
expect(readConfig({ ...env, DRY_RUN: "false", CLOSE_PRS_DRY_RUN: "false" })).toMatchObject({ commentDryRun: false, closeDryRun: false });
|
||||
});
|
||||
|
||||
test("a sweep needs no issue number and anything but the literal true is an issue run", () => {
|
||||
expect(readConfig({ ...env, ISSUE_NUMBER: undefined, SWEEP: "true" }).run).toEqual({ kind: "sweep" });
|
||||
expect(readConfig({ ...env, SWEEP: "false" }).run).toEqual({ kind: "issue", number: 41750 });
|
||||
expect(() => readConfig({ ...env, ISSUE_NUMBER: "", SWEEP: "false" })).toThrow("dispatch with an issue_number or with sweep ticked");
|
||||
});
|
||||
|
||||
test("refuses a missing token, repo, branch or a bad issue number", () => {
|
||||
|
|
@ -232,3 +616,31 @@ describe("readConfig", () => {
|
|||
expect(() => readConfig({ ...env, ISSUE_NUMBER: "abc" })).toThrow("ISSUE_NUMBER");
|
||||
});
|
||||
});
|
||||
|
||||
describe("step summary", () => {
|
||||
const closed = { kind: "closed" as const, number: 41760, body: oneFixBody };
|
||||
const left = { kind: "skip" as const, number: 41762, reason: "still linked to open #41751" };
|
||||
const commented = { kind: "commented" as const, pullRequest: 41767, tag: "v1.103.0-rc.1", body: fixedBody(41767, { tag: "v1.103.0-rc.1", shipped: false }) };
|
||||
|
||||
test("an issue run names the comment, each close, and each pull request left open", () => {
|
||||
const summary = describeIssue(config, ISSUE, { comment: commented, pullRequests: [closed, left] });
|
||||
expect(summary).toContain("#41750: commented, fixed by #41767 in v1.103.0-rc.1");
|
||||
expect(summary).toContain("#41760: closed with: #41750 was fixed by #41767 on main");
|
||||
expect(summary).toContain("#41762: left open, still linked to open #41751");
|
||||
expect(summary).not.toContain("DRY RUN");
|
||||
});
|
||||
|
||||
test("a close dry run names the repo variable that turns closing on", () => {
|
||||
const summary = describeIssue({ ...config, closeDryRun: true }, ISSUE, { comment: commented, pullRequests: [closed] });
|
||||
expect(summary).toContain("ISSUE_FIXED_CLOSE_PRS_ENABLED");
|
||||
expect(summary).toContain("#41760: DRY RUN, would close with: #41750 was fixed by #41767 on main");
|
||||
});
|
||||
|
||||
test("a sweep summary counts what it saw and lists only the closes", () => {
|
||||
const summary = describeSweep(config, { considered: 3522, pullRequests: [closed, left] });
|
||||
expect(summary).toContain("Swept 3522 open pull requests, 2 linked to an issue, 1 closed");
|
||||
expect(summary).toContain("#41760: closed with:");
|
||||
expect(summary).not.toContain("#41762");
|
||||
expect(describeSweep({ ...config, closeDryRun: true }, { considered: 3522, pullRequests: [closed] })).toContain("1 would be closed, DRY RUN, set the ISSUE_FIXED_CLOSE_PRS_ENABLED");
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -6,35 +6,66 @@ declare const process: { readonly env: Readonly<Record<string, string | undefine
|
|||
|
||||
export interface FixedConfig {
|
||||
readonly repo: string;
|
||||
readonly issueNumber: number;
|
||||
readonly defaultBranch: string;
|
||||
readonly dryRun: boolean;
|
||||
readonly commentDryRun: boolean;
|
||||
readonly closeDryRun: boolean;
|
||||
}
|
||||
|
||||
export type Run = { readonly kind: "issue"; readonly number: number } | { readonly kind: "sweep" };
|
||||
|
||||
interface PullRequestCloser {
|
||||
readonly __typename: "PullRequest";
|
||||
readonly number: number;
|
||||
readonly merged: boolean;
|
||||
readonly baseRefName: string;
|
||||
readonly repository: { readonly nameWithOwner: string };
|
||||
readonly mergeCommit: { readonly oid: string } | null;
|
||||
}
|
||||
|
||||
interface CommitCloser {
|
||||
readonly __typename: "Commit";
|
||||
readonly oid: string;
|
||||
readonly repository: { readonly nameWithOwner: string };
|
||||
}
|
||||
|
||||
export interface ClosedIssue {
|
||||
export interface IssueClosure {
|
||||
readonly state: "OPEN" | "CLOSED";
|
||||
readonly timelineItems: {
|
||||
readonly nodes: readonly { readonly closer: PullRequestCloser | CommitCloser | null }[];
|
||||
};
|
||||
}
|
||||
|
||||
export interface LinkedIssue extends IssueClosure {
|
||||
readonly number: number;
|
||||
readonly repository: { readonly nameWithOwner: string };
|
||||
}
|
||||
|
||||
export interface LinkedPullRequest {
|
||||
readonly number: number;
|
||||
readonly state: "OPEN" | "CLOSED" | "MERGED";
|
||||
readonly baseRefName: string;
|
||||
readonly repository: { readonly nameWithOwner: string };
|
||||
readonly closingIssuesReferences: { readonly totalCount: number; readonly nodes: readonly LinkedIssue[] };
|
||||
readonly reopens: { readonly nodes: readonly { readonly createdAt: string }[] };
|
||||
}
|
||||
|
||||
export interface PullRequestsPage {
|
||||
readonly pageInfo: { readonly hasNextPage: boolean; readonly endCursor: string | null };
|
||||
readonly nodes: readonly LinkedPullRequest[];
|
||||
}
|
||||
|
||||
export interface ClosedIssue extends IssueClosure {
|
||||
readonly closedByPullRequestsReferences: PullRequestsPage;
|
||||
}
|
||||
|
||||
interface TimelineResponse {
|
||||
readonly data?: { readonly repository?: { readonly issue: ClosedIssue | null } };
|
||||
}
|
||||
|
||||
interface OpenPullRequestsResponse {
|
||||
readonly data?: { readonly repository?: { readonly pullRequests: PullRequestsPage } };
|
||||
}
|
||||
|
||||
interface MatchingRef {
|
||||
readonly ref: string;
|
||||
}
|
||||
|
|
@ -59,33 +90,100 @@ export type FixedVerdict =
|
|||
| { readonly kind: "commented"; readonly pullRequest: number; readonly tag: string; readonly body: string }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export const FIXED_MARKER = "<!-- litellm:fixed-in -->";
|
||||
const MAX_MINOR_BUMPS = 3;
|
||||
export type FixSource =
|
||||
| { readonly kind: "pull_request"; readonly number: number; readonly oid: string }
|
||||
| { readonly kind: "commit"; readonly oid: string };
|
||||
|
||||
export const CLOSER_QUERY = `query($owner: String!, $name: String!, $number: Int!) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
issue(number: $number) {
|
||||
state
|
||||
timelineItems(last: 1, itemTypes: [CLOSED_EVENT]) {
|
||||
nodes {
|
||||
... on ClosedEvent {
|
||||
closer {
|
||||
__typename
|
||||
... on PullRequest { number merged baseRefName mergeCommit { oid } }
|
||||
... on Commit { oid }
|
||||
}
|
||||
}
|
||||
export type FixVerdict = FixSource | { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export interface Fix {
|
||||
readonly issue: number;
|
||||
readonly source: FixSource;
|
||||
}
|
||||
|
||||
export type CloseVerdict =
|
||||
| { readonly kind: "candidate"; readonly fixes: readonly Fix[] }
|
||||
| { readonly kind: "skip"; readonly reason: string };
|
||||
|
||||
export type CloseOutcome =
|
||||
| { readonly kind: "closed"; readonly number: number; readonly body: string }
|
||||
| { readonly kind: "skip"; readonly number: number; readonly reason: string };
|
||||
|
||||
export interface IssueOutcome {
|
||||
readonly comment: FixedVerdict;
|
||||
readonly pullRequests: readonly CloseOutcome[];
|
||||
}
|
||||
|
||||
export interface SweepOutcome {
|
||||
readonly considered: number;
|
||||
readonly pullRequests: readonly CloseOutcome[];
|
||||
}
|
||||
|
||||
export const FIXED_MARKER = "<!-- litellm:fixed-in -->";
|
||||
export const SUPERSEDED_MARKER = "<!-- litellm:superseded -->";
|
||||
const MAX_MINOR_BUMPS = 3;
|
||||
const MAX_LINKED_PULL_REQUESTS = 50;
|
||||
const MAX_LINKED_ISSUES = 10;
|
||||
const SWEEP_PAGE_SIZE = 100;
|
||||
const CLOSE_PAUSE_MS = 1000;
|
||||
const WORKFLOW_LOGIN = "github-actions[bot]";
|
||||
const OPEN_AGAIN = "the issue is open again";
|
||||
const isReleaseLine = (base: string): boolean => base.startsWith("release/") || base.includes("stable");
|
||||
|
||||
const CLOSURE_FRAGMENT = `fragment Closure on Issue {
|
||||
state
|
||||
timelineItems(last: 1, itemTypes: [CLOSED_EVENT]) {
|
||||
nodes {
|
||||
... on ClosedEvent {
|
||||
closer {
|
||||
__typename
|
||||
... on PullRequest { number merged baseRefName repository { nameWithOwner } mergeCommit { oid } }
|
||||
... on Commit { oid repository { nameWithOwner } }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`;
|
||||
|
||||
const LINKED_PULL_REQUEST_FRAGMENT = `fragment Linked on PullRequest {
|
||||
number
|
||||
state
|
||||
baseRefName
|
||||
repository { nameWithOwner }
|
||||
closingIssuesReferences(first: ${MAX_LINKED_ISSUES}) { totalCount nodes { number repository { nameWithOwner } ...Closure } }
|
||||
reopens: timelineItems(last: 1, itemTypes: [REOPENED_EVENT]) { nodes { ... on ReopenedEvent { createdAt } } }
|
||||
}`;
|
||||
|
||||
export const CLOSER_QUERY = `query($owner: String!, $name: String!, $number: Int!, $after: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
issue(number: $number) {
|
||||
...Closure
|
||||
closedByPullRequestsReferences(first: ${MAX_LINKED_PULL_REQUESTS}, after: $after) {
|
||||
pageInfo { hasNextPage endCursor }
|
||||
nodes { ...Linked }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
${CLOSURE_FRAGMENT}
|
||||
${LINKED_PULL_REQUEST_FRAGMENT}`;
|
||||
|
||||
export const OPEN_PULL_REQUESTS_QUERY = `query($owner: String!, $name: String!, $after: String) {
|
||||
repository(owner: $owner, name: $name) {
|
||||
pullRequests(states: OPEN, first: ${SWEEP_PAGE_SIZE}, after: $after) {
|
||||
pageInfo { hasNextPage endCursor }
|
||||
nodes { ...Linked }
|
||||
}
|
||||
}
|
||||
}
|
||||
${CLOSURE_FRAGMENT}
|
||||
${LINKED_PULL_REQUEST_FRAGMENT}`;
|
||||
|
||||
const skip = (reason: string): { readonly kind: "skip"; readonly reason: string } => ({ kind: "skip", reason });
|
||||
|
||||
export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer {
|
||||
export function closerOf(issue: IssueClosure, repo: string, defaultBranch: string): Closer {
|
||||
if (issue.state !== "CLOSED") {
|
||||
return skip("the issue is open again");
|
||||
return skip(OPEN_AGAIN);
|
||||
}
|
||||
const closer = issue.timelineItems.nodes[0]?.closer ?? null;
|
||||
if (closer === null) {
|
||||
|
|
@ -94,6 +192,9 @@ export function closerOf(issue: ClosedIssue, defaultBranch: string): Closer {
|
|||
if (closer.__typename === "Commit") {
|
||||
return skip(`closed by commit ${closer.oid.slice(0, 10)}, not by a pull request`);
|
||||
}
|
||||
if (closer.repository.nameWithOwner !== repo) {
|
||||
return skip(`closed by ${closer.repository.nameWithOwner}#${closer.number}, a pull request in another repository`);
|
||||
}
|
||||
if (!closer.merged || closer.mergeCommit === null) {
|
||||
return skip(`closed by #${closer.number}, which is not merged`);
|
||||
}
|
||||
|
|
@ -121,8 +222,8 @@ async function tagExists(api: GitHubApi, repo: string, tag: string): Promise<boo
|
|||
return refs.some((ref) => ref.ref === `refs/tags/${tag}`);
|
||||
}
|
||||
|
||||
async function tagContains(api: GitHubApi, repo: string, tag: string, sha: string): Promise<boolean> {
|
||||
const comparison = await api.request<Comparison>("GET", `/repos/${repo}/compare/${tag}...${sha}`);
|
||||
async function refContains(api: GitHubApi, repo: string, ref: string, sha: string): Promise<boolean> {
|
||||
const comparison = await api.request<Comparison>("GET", `/repos/${repo}/compare/${ref}...${sha}`);
|
||||
return comparison.status === "behind" || comparison.status === "identical";
|
||||
}
|
||||
|
||||
|
|
@ -139,7 +240,7 @@ async function firstReleaseWith(
|
|||
if (!(await tagExists(api, repo, tag))) {
|
||||
return { kind: "release", tag, shipped: false };
|
||||
}
|
||||
if (await tagContains(api, repo, tag, sha)) {
|
||||
if (await refContains(api, repo, tag, sha)) {
|
||||
return { kind: "release", tag, shipped: true };
|
||||
}
|
||||
if (bumpsLeft === 0) {
|
||||
|
|
@ -164,21 +265,13 @@ export function fixedBody(pullRequest: number, release: { readonly tag: string;
|
|||
return `${FIXED_MARKER}\nFixed by #${pullRequest}. ${availability}`;
|
||||
}
|
||||
|
||||
export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Promise<FixedVerdict> {
|
||||
const [owner, name] = config.repo.split("/");
|
||||
const response = await api.request<TimelineResponse>("POST", "/graphql", {
|
||||
query: CLOSER_QUERY,
|
||||
variables: { owner, name, number: config.issueNumber },
|
||||
});
|
||||
const issue = response.data?.repository?.issue ?? null;
|
||||
if (issue === null) {
|
||||
return skip("not an issue in this repository");
|
||||
}
|
||||
const closer = closerOf(issue, config.defaultBranch);
|
||||
if (closer.kind === "skip") {
|
||||
return closer;
|
||||
}
|
||||
const issuePath = `/repos/${config.repo}/issues/${config.issueNumber}`;
|
||||
export async function commentFixedIssue(
|
||||
api: GitHubApi,
|
||||
config: FixedConfig,
|
||||
issueNumber: number,
|
||||
closer: { readonly number: number; readonly mergeCommit: string },
|
||||
): Promise<FixedVerdict> {
|
||||
const issuePath = `/repos/${config.repo}/issues/${issueNumber}`;
|
||||
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
|
||||
if (comments.some((comment) => comment.body.includes(FIXED_MARKER))) {
|
||||
return skip("already carries a fixed-in comment");
|
||||
|
|
@ -188,37 +281,259 @@ export async function commentFixedIssue(api: GitHubApi, config: FixedConfig): Pr
|
|||
return release;
|
||||
}
|
||||
const body = fixedBody(closer.number, release);
|
||||
if (!config.dryRun) {
|
||||
if (!config.commentDryRun) {
|
||||
await api.request("POST", `${issuePath}/comments`, { body });
|
||||
}
|
||||
return { kind: "commented", pullRequest: closer.number, tag: release.tag, body };
|
||||
}
|
||||
|
||||
export function readConfig(env: Readonly<Record<string, string | undefined>>): FixedConfig & { readonly token: string } {
|
||||
export function fixOf(issue: IssueClosure, repo: string): FixVerdict {
|
||||
if (issue.state !== "CLOSED") {
|
||||
return skip("is open again");
|
||||
}
|
||||
const closer = issue.timelineItems.nodes[0]?.closer ?? null;
|
||||
if (closer === null) {
|
||||
return skip("was closed by hand");
|
||||
}
|
||||
if (closer.repository.nameWithOwner !== repo) {
|
||||
return skip(`was closed from ${closer.repository.nameWithOwner}`);
|
||||
}
|
||||
if (closer.__typename === "Commit") {
|
||||
return { kind: "commit", oid: closer.oid };
|
||||
}
|
||||
if (!closer.merged || closer.mergeCommit === null) {
|
||||
return skip(`was closed by #${closer.number}, which is not merged`);
|
||||
}
|
||||
return { kind: "pull_request", number: closer.number, oid: closer.mergeCommit.oid };
|
||||
}
|
||||
|
||||
export function closeVerdict(pullRequest: LinkedPullRequest, config: FixedConfig): CloseVerdict {
|
||||
if (pullRequest.repository.nameWithOwner !== config.repo) {
|
||||
return skip(`lives in ${pullRequest.repository.nameWithOwner}`);
|
||||
}
|
||||
if (pullRequest.state !== "OPEN") {
|
||||
return skip(`is ${pullRequest.state.toLowerCase()}`);
|
||||
}
|
||||
if (isReleaseLine(pullRequest.baseRefName)) {
|
||||
return skip(`targets the release line ${pullRequest.baseRefName}`);
|
||||
}
|
||||
const { totalCount, nodes: linked } = pullRequest.closingIssuesReferences;
|
||||
if (linked.length === 0) {
|
||||
return skip("links no issue");
|
||||
}
|
||||
if (totalCount > linked.length) {
|
||||
return skip(`links ${totalCount} issues, more than the ${MAX_LINKED_ISSUES} this workflow reads`);
|
||||
}
|
||||
const foreign = linked.find((issue) => issue.repository.nameWithOwner !== config.repo);
|
||||
if (foreign !== undefined) {
|
||||
return skip(`links ${foreign.repository.nameWithOwner}#${foreign.number}`);
|
||||
}
|
||||
const stillOpen = linked.find((issue) => issue.state === "OPEN");
|
||||
if (stillOpen !== undefined) {
|
||||
return skip(`still linked to open #${stillOpen.number}`);
|
||||
}
|
||||
const verdicts = linked.map((issue) => ({ issue: issue.number, fix: fixOf(issue, config.repo) }));
|
||||
for (const { issue, fix } of verdicts) {
|
||||
if (fix.kind === "skip") {
|
||||
return skip(`#${issue} ${fix.reason}`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
kind: "candidate",
|
||||
fixes: verdicts.flatMap(({ issue, fix }) => (fix.kind === "skip" ? [] : [{ issue, source: fix }])),
|
||||
};
|
||||
}
|
||||
|
||||
function describeSource(source: FixSource): string {
|
||||
return source.kind === "pull_request" ? `#${source.number}` : `commit ${source.oid.slice(0, 10)}`;
|
||||
}
|
||||
|
||||
async function fixOffDefaultBranch(api: GitHubApi, config: FixedConfig, fixes: readonly Fix[]): Promise<Fix | undefined> {
|
||||
const onBranch = await Promise.all(fixes.map((fix) => refContains(api, config.repo, config.defaultBranch, fix.source.oid)));
|
||||
return fixes.find((_, index) => !onBranch[index]);
|
||||
}
|
||||
|
||||
export function supersededBody(fixes: readonly Fix[], defaultBranch: string): string {
|
||||
const pairs = fixes
|
||||
.map((fix, index) => `#${fix.issue} ${index === 0 ? "was fixed by" : "by"} ${describeSource(fix.source)}`)
|
||||
.join(" and ");
|
||||
return `${SUPERSEDED_MARKER}\n${pairs} on ${defaultBranch}, so this pull request is closed. Reopen it if something was missed.`;
|
||||
}
|
||||
|
||||
function reopenedAfter(pullRequest: LinkedPullRequest, comment: Comment): boolean {
|
||||
const reopen = pullRequest.reopens.nodes[0];
|
||||
return reopen !== undefined && Date.parse(reopen.createdAt) > Date.parse(comment.created_at);
|
||||
}
|
||||
|
||||
async function closePullRequest(
|
||||
api: GitHubApi,
|
||||
config: FixedConfig,
|
||||
pullRequest: LinkedPullRequest,
|
||||
pause: () => Promise<void>,
|
||||
): Promise<CloseOutcome> {
|
||||
const verdict = closeVerdict(pullRequest, config);
|
||||
if (verdict.kind === "skip") {
|
||||
return { kind: "skip", number: pullRequest.number, reason: verdict.reason };
|
||||
}
|
||||
const offBranch = await fixOffDefaultBranch(api, config, verdict.fixes);
|
||||
if (offBranch !== undefined) {
|
||||
const fix = describeSource(offBranch.source);
|
||||
return { kind: "skip", number: pullRequest.number, reason: `#${offBranch.issue} was fixed by ${fix}, which is not on ${config.defaultBranch}` };
|
||||
}
|
||||
const issuePath = `/repos/${config.repo}/issues/${pullRequest.number}`;
|
||||
const comments = await listAll<Comment>(api, `${issuePath}/comments`);
|
||||
const marker = comments.find((comment) => comment.user.login === WORKFLOW_LOGIN && comment.body.includes(SUPERSEDED_MARKER));
|
||||
if (marker !== undefined && reopenedAfter(pullRequest, marker)) {
|
||||
return { kind: "skip", number: pullRequest.number, reason: "was closed by this workflow once and reopened" };
|
||||
}
|
||||
const body = marker?.body ?? supersededBody(verdict.fixes, config.defaultBranch);
|
||||
if (config.closeDryRun) {
|
||||
return { kind: "closed", number: pullRequest.number, body };
|
||||
}
|
||||
if (marker === undefined) {
|
||||
await pause();
|
||||
await api.request("POST", `${issuePath}/comments`, { body });
|
||||
}
|
||||
await pause();
|
||||
await api.request("PATCH", `/repos/${config.repo}/pulls/${pullRequest.number}`, { state: "closed" });
|
||||
return { kind: "closed", number: pullRequest.number, body };
|
||||
}
|
||||
|
||||
export function closePullRequests(
|
||||
api: GitHubApi,
|
||||
config: FixedConfig,
|
||||
candidates: readonly LinkedPullRequest[],
|
||||
pause: () => Promise<void>,
|
||||
): Promise<readonly CloseOutcome[]> {
|
||||
return candidates.reduce<Promise<readonly CloseOutcome[]>>(
|
||||
async (previous, candidate) => [...(await previous), await closePullRequest(api, config, candidate, pause)],
|
||||
Promise.resolve([]),
|
||||
);
|
||||
}
|
||||
|
||||
type NextPage = (after: string | null) => Promise<PullRequestsPage>;
|
||||
|
||||
async function collectPages(page: PullRequestsPage, nextPage: NextPage): Promise<readonly LinkedPullRequest[]> {
|
||||
if (!page.pageInfo.hasNextPage) {
|
||||
return page.nodes;
|
||||
}
|
||||
return [...page.nodes, ...(await collectPages(await nextPage(page.pageInfo.endCursor), nextPage))];
|
||||
}
|
||||
|
||||
async function closedIssue(api: GitHubApi, config: FixedConfig, issueNumber: number, after: string | null): Promise<ClosedIssue | null> {
|
||||
const [owner, name] = config.repo.split("/");
|
||||
const response = await api.request<TimelineResponse>("POST", "/graphql", {
|
||||
query: CLOSER_QUERY,
|
||||
variables: { owner, name, number: issueNumber, after },
|
||||
});
|
||||
return response.data?.repository?.issue ?? null;
|
||||
}
|
||||
|
||||
export async function handleFixedIssue(
|
||||
api: GitHubApi,
|
||||
config: FixedConfig,
|
||||
issueNumber: number,
|
||||
pause: () => Promise<void>,
|
||||
): Promise<IssueOutcome> {
|
||||
const issue = await closedIssue(api, config, issueNumber, null);
|
||||
if (issue === null) {
|
||||
return { comment: skip("not an issue in this repository"), pullRequests: [] };
|
||||
}
|
||||
if (issue.state !== "CLOSED") {
|
||||
return { comment: skip(OPEN_AGAIN), pullRequests: [] };
|
||||
}
|
||||
const closer = closerOf(issue, config.repo, config.defaultBranch);
|
||||
const comment = closer.kind === "skip" ? closer : await commentFixedIssue(api, config, issueNumber, closer);
|
||||
const nextPage: NextPage = async (after) => {
|
||||
const more = await closedIssue(api, config, issueNumber, after);
|
||||
if (more === null) {
|
||||
throw new Error(`#${issueNumber} came back without data while reading its linked pull requests after cursor ${after}`);
|
||||
}
|
||||
return more.closedByPullRequestsReferences;
|
||||
};
|
||||
const linked = await collectPages(issue.closedByPullRequestsReferences, nextPage);
|
||||
const open = linked.filter((pullRequest) => pullRequest.state === "OPEN");
|
||||
const pullRequests = await closePullRequests(api, config, open, pause);
|
||||
return { comment, pullRequests };
|
||||
}
|
||||
|
||||
async function openPullRequests(api: GitHubApi, config: FixedConfig): Promise<readonly LinkedPullRequest[]> {
|
||||
const [owner, name] = config.repo.split("/");
|
||||
const nextPage: NextPage = async (after) => {
|
||||
const response = await api.request<OpenPullRequestsResponse>("POST", "/graphql", {
|
||||
query: OPEN_PULL_REQUESTS_QUERY,
|
||||
variables: { owner, name, after },
|
||||
});
|
||||
const page = response.data?.repository?.pullRequests;
|
||||
if (page === undefined) {
|
||||
throw new Error(`open pull requests after cursor ${after} came back without data: ${JSON.stringify(response)}`);
|
||||
}
|
||||
return page;
|
||||
};
|
||||
return collectPages(await nextPage(null), nextPage);
|
||||
}
|
||||
|
||||
export async function sweep(api: GitHubApi, config: FixedConfig, pause: () => Promise<void>): Promise<SweepOutcome> {
|
||||
const open = await openPullRequests(api, config);
|
||||
const linked = open.filter((pullRequest) => pullRequest.closingIssuesReferences.nodes.length > 0);
|
||||
return { considered: open.length, pullRequests: await closePullRequests(api, config, linked, pause) };
|
||||
}
|
||||
|
||||
export function readConfig(
|
||||
env: Readonly<Record<string, string | undefined>>,
|
||||
): FixedConfig & { readonly token: string; readonly run: Run } {
|
||||
const token = env.GITHUB_TOKEN;
|
||||
const repo = env.GITHUB_REPOSITORY;
|
||||
const defaultBranch = env.DEFAULT_BRANCH;
|
||||
if (!token || !repo || !/^[\w.-]+\/[\w.-]+$/.test(repo) || !defaultBranch) {
|
||||
throw new Error("GITHUB_TOKEN, GITHUB_REPOSITORY (owner/repo) and DEFAULT_BRANCH are required");
|
||||
}
|
||||
const config = { token, repo, defaultBranch, commentDryRun: env.DRY_RUN === "true", closeDryRun: env.CLOSE_PRS_DRY_RUN === "true" };
|
||||
if (env.SWEEP === "true") {
|
||||
return { ...config, run: { kind: "sweep" } };
|
||||
}
|
||||
const issueNumber = Number(env.ISSUE_NUMBER);
|
||||
if (!Number.isInteger(issueNumber) || issueNumber <= 0) {
|
||||
throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}"`);
|
||||
throw new Error(`ISSUE_NUMBER must be a positive integer, got "${env.ISSUE_NUMBER}": dispatch with an issue_number or with sweep ticked`);
|
||||
}
|
||||
return { token, repo, issueNumber, defaultBranch, dryRun: env.DRY_RUN === "true" };
|
||||
return { ...config, run: { kind: "issue", number: issueNumber } };
|
||||
}
|
||||
|
||||
function describe(config: FixedConfig, verdict: FixedVerdict): string {
|
||||
if (verdict.kind === "skip") {
|
||||
return `#${config.issueNumber}: skipped, ${verdict.reason}`;
|
||||
const CLOSE_DRY_RUN_HINT = "set the ISSUE_FIXED_CLOSE_PRS_ENABLED repo variable to true to close pull requests";
|
||||
|
||||
function describeClose(config: FixedConfig, outcome: CloseOutcome): string {
|
||||
if (outcome.kind === "skip") {
|
||||
return `#${outcome.number}: left open, ${outcome.reason}`;
|
||||
}
|
||||
if (config.dryRun) {
|
||||
return `#${config.issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${verdict.body}`;
|
||||
}
|
||||
return `#${config.issueNumber}: commented, fixed by #${verdict.pullRequest} in ${verdict.tag}`;
|
||||
const text = outcome.body.replace(`${SUPERSEDED_MARKER}\n`, "");
|
||||
return config.closeDryRun ? `#${outcome.number}: DRY RUN, would close with: ${text}` : `#${outcome.number}: closed with: ${text}`;
|
||||
}
|
||||
|
||||
export function describeIssue(config: FixedConfig, issueNumber: number, outcome: IssueOutcome): string {
|
||||
const comment =
|
||||
outcome.comment.kind === "skip"
|
||||
? `#${issueNumber}: skipped, ${outcome.comment.reason}`
|
||||
: config.commentDryRun
|
||||
? `#${issueNumber}: DRY RUN, set the ISSUE_FIXED_COMMENT_ENABLED repo variable to true to post this:\n\n${outcome.comment.body}`
|
||||
: `#${issueNumber}: commented, fixed by #${outcome.comment.pullRequest} in ${outcome.comment.tag}`;
|
||||
const hint = config.closeDryRun && outcome.pullRequests.some((pullRequest) => pullRequest.kind === "closed") ? [`Closing is a DRY RUN, ${CLOSE_DRY_RUN_HINT}`] : [];
|
||||
return [comment, ...hint, ...outcome.pullRequests.map((pullRequest) => describeClose(config, pullRequest))].join("\n");
|
||||
}
|
||||
|
||||
export function describeSweep(config: FixedConfig, outcome: SweepOutcome): string {
|
||||
const closed = outcome.pullRequests.filter((pullRequest) => pullRequest.kind === "closed");
|
||||
const verb = config.closeDryRun ? `would be closed, DRY RUN, ${CLOSE_DRY_RUN_HINT}` : "closed";
|
||||
const header = `Swept ${outcome.considered} open pull requests, ${outcome.pullRequests.length} linked to an issue, ${closed.length} ${verb}`;
|
||||
return [header, ...closed.map((pullRequest) => describeClose(config, pullRequest))].join("\n");
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const { token, ...config } = readConfig(process.env);
|
||||
console.log(describe(config, await commentFixedIssue(githubApi(token), config)));
|
||||
const { token, run, ...config } = readConfig(process.env);
|
||||
const api = githubApi(token);
|
||||
const pause = (): Promise<void> => new Promise((resolve) => setTimeout(resolve, CLOSE_PAUSE_MS));
|
||||
console.log(
|
||||
run.kind === "sweep"
|
||||
? describeSweep(config, await sweep(api, config, pause))
|
||||
: describeIssue(config, run.number, await handleFixedIssue(api, config, run.number, pause)),
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -69,6 +69,9 @@ IGNORE_FUNCTIONS = [
|
|||
"_unqualified", # bounded by the qualifier depth of a static TypedDict annotation (Annotated, Required/NotRequired, ReadOnly around one type, no cycles possible).
|
||||
"_render_json", # bounded by the nesting depth of a pydantic-validated JsonValue from the operator's config (a finite JSON tree, no cycles possible).
|
||||
"completion_cost", # max depth 1: recursion only fires for mixed-tier Responses WS logging objects, and each split part carries a single service_tier so _split_responses_ws_logging_object_by_service_tier returns None.
|
||||
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
|
||||
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
|
||||
]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ Each subdirectory under `tests/e2e/` is one suite, scoped to an endpoint family
|
|||
- `router/` - routing and reliability behavior (fallbacks, cooldowns) plus the memory tests (`test_reliability_memory_e2e.py`: every worker's RSS as read at collection time, before any test traffic, must sit under a fixed idle budget, the release-gate check for a DB-backed boot that idles near the pod limit the way v1.100.x did; and a few hundred failing requests with retries and fallbacks must not grow proxy RSS past a fixed budget nor store a request snapshot past a fixed size, the release-gate check for the v1.100.0 retry-breadcrumb leak)
|
||||
- `load/` - performance-category tests, kept OUT of the main suite: throughput/load SLO tests are a different testing category from functional e2e (variance-driven, historically flaky) and live outside this suite until re-implemented as their own pipeline (LIT-5163); do not add a live load test that runs in the default collection. What lives here: the weekly session-anomaly test (`test_weekly_session_anomaly_e2e.py`, Claude Code-shaped multi-turn sessions against real providers with ceilings on error rate, cache read/write, turn time, and spend; marked `weekly` and deselected unless `E2E_WEEKLY_ANOMALY` is set, driven by `.github/workflows/weekly_load_anomaly.yml`), the Redis chaos test (`test_redis_chaos_e2e.py`, locust load against mock deployments split round robin over `/chat/completions` and `/v1/messages`, one endpoint per simulated user, with `CLIENT PAUSE ALL` on the proxy's Redis mid-run to simulate it being down outright, asserting zero failed requests on every endpoint, budgeting RSS and CPU-per-request as ratios against the same run's healthy phase, and holding p50/p90/p99 latency and log-bytes-per-request to flat ceilings (a ratio cannot bound those two: an open breaker skips Redis instead of waiting on it, so the chaos phase can measure cheaper than baseline while still being far slower than a user should see); needs a proxy booted from `gateway/redis_chaos_ci_config.yml` on the same host with `E2E_PROXY_PID` and `E2E_PROXY_LOG` set, marked `redis_chaos`, deselected unless `E2E_REDIS_CHAOS` is set and excluded from the per-PR selector like the rest of `load/`, driven by `.github/workflows/test-e2e-redis-chaos.yml` and by the Buildkite `e2e-redis-chaos` step in project-releaser, which runs the proxy, Postgres and Valkey co-located with pytest in one pod and sets the opt-in), and markerless harness unit tests for the locust, process-usage, and session-anomaly aggregation logic
|
||||
- `other/` - the holding-pen suite for the `other.*` registry cluster with no home of its own yet: the master-key auth gate, JWT auth (access tokens issued by a real Keycloak realm, `idp.py` plus `idp_realm.json`, whose JWKS the proxy's `JWT_PUBLIC_KEY_URL` points at; see CONTRIBUTING.md for the start command and config block), and the process-lifecycle health probes (liveness, public readiness, authenticated readiness diagnostics). Promote a cluster out once it is large/stable enough for its own suite
|
||||
- `secret_manager/` - the gateway's `key_management_system` against a real secret manager: deployment keys resolved from it (`os.environ/<name>` where the name exists only in the manager) and virtual keys written to and deleted from it. The tests are backend-agnostic and each backend is its own lane, because the setting is global to the proxy: `E2E_SECRET_MANAGER=<system>` opts in and picks the backend from `secret_backends.BACKENDS`, the proxy is booted from `gateway/secret_manager_<system>_ci_config.yml` against the live manager, and the tests reach that manager through the backend's `SecretStore` (`secret_store_<system>.py`). A test needing something not every backend does carries `requires_capability(...)` and is deselected on lanes that lack it. `secret_manager/backend.sh up <system>` runs a backend in Docker and writes the proxy's and the tests' env. Marked `secret_manager`, deselected unless `E2E_SECRET_MANAGER` is set, and kept out of the per-PR selector. Backends today: `hashicorp_vault` and `cyberark` (CyberArk Conjur, which cannot delete, so the delete test is Vault-only)
|
||||
- `gateway/` - proxy configuration only (`litellm-config.yml`); no tests
|
||||
- `claude_code/` - the Claude Code compatibility matrix: drives the real `claude` CLI (and HTTP probes) against a proxy for each feature x provider cell, reporting tagged-union outcomes via the `compat_result` fixture; ships its own driver/builder/publisher plus `_*_unit_tests/` trees. The HTTP probes ride the shared transport (`ProxyClient.count_tokens` / `ProxyClient.messages`); the CLI-driving path stays bespoke
|
||||
- `ui/` - the Admin UI browser suite: Playwright in TypeScript, driving the dashboard served by a live proxy on port 4000 (seeded postgres + mock LLM upstream; see its `run_e2e.sh`). It is a self-contained npm package with its own lockfile and does not use the Python harness, pytest markers, or the shared transport; the Python rules in this file (typed models, `Result` unions, basedpyright zero-error gate) do not apply inside it. Its only Python file, `fixtures/mock_llm_server/server.py`, is excluded from the e2e basedpyright gate via the root `pyrightconfig.json`
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ A couple of logging destinations are configured on the proxy rather than by the
|
|||
|
||||
### The pull request check
|
||||
|
||||
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, and `load/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set
|
||||
Every same-repository PR that adds, modifies, or renames a `tests/e2e/**/test_*.py` file runs those changed files three times. A change to the harness itself, meaning a root-level `tests/e2e/*.py` file or `pytest.ini`, `tests/e2e/gateway/`, `.github/e2e-stack/`, or the workflow, also runs the `access_control` suite and both JWT suites as canaries, because those files have no test of their own that exercises the stack. `.github/e2e-stack/select_tests.py` applies both rules. The stack config at `tests/e2e/gateway/stage_mirror_ci_config.yml` must declare every model the selected suites use; a missing one shows up as a failed test id in the public log. The suite's own single rerun for network errors and 5xx responses (see `pytest.ini`) applies on every pass, so a transport blip does not fail the check while a race inside a test still does. The stage-mirror stack has a control-plane backend, two gateways behind nginx, Postgres, Keycloak, Jaeger, and TLS cluster-mode Valkey. Realm-only edits also trigger these canaries. The stack exports every gateway address in `LITELLM_PROXY_REPLICA_URLS`, so model registration waits until each gateway lists the new model rather than whichever one the load balancer answered from. Documentation, deleted-file, and application-only changes do not start the stack or request environment approval. The `ui/`, `claude_code/`, `load/`, and `secret_manager/` directories, `batches/test_managed_files_enforcement_e2e.py`, `llm_translation/realtime/test_realtime_pipecat_audio_e2e.py`, and `guardrails/test_presidio_masking_e2e.py` remain outside this check because they use separate tooling or need a differently configured stack: the pipecat audio suite skips itself at import time unless the NLTK `punkt_tab` data is installed, and the presidio suite fails without the analyzer and anonymizer services this stack does not start. `logging/test_otel_v2_langfuse_generation_output_e2e.py` is marked `otel_v2` and deselects itself unless `E2E_OTEL_V2` is set, because it needs a gateway booted with `LITELLM_OTEL_V2=true` and Langfuse credentials, neither of which this stack provides, so run it with `E2E_OTEL_V2=1` against a local OTel v2 proxy. The Redis chaos test under `load/` needs a proxy it can pause the Redis of on the same host (`gateway/redis_chaos_ci_config.yml`), which `.github/workflows/test-e2e-redis-chaos.yml` boots, and which the Buildkite `e2e-redis-chaos` step in project-releaser runs co-located with Postgres and Valkey in one pod; it is deselected unless `E2E_REDIS_CHAOS` is set. The `secret_manager/` lanes each need a proxy configured against their own secret manager (see Secret manager lanes below)
|
||||
|
||||
Every selected file must execute at least one passing test in each pass, and any test failure, collection error, or entirely skipped or deselected file fails the check. A file whose tests are all marked skip therefore cannot pass this check, so unskip at least one of them, or add the file to `UNSUPPORTED` in `select_tests.py` with the reason, before changing one. A failed pass stops the run. The public log prints pytest's one-line summary for each pass, including the rerun count, and names each failed or errored test as `classname::name`, so a retried network error or a failing test is visible without the raw output. The final `e2e-changed-tests` job succeeds only when no supported test files changed or the approved run completed all three passes. Fork PRs with selected tests fail this gate until a maintainer brings the reviewed change onto a same-repository branch
|
||||
|
||||
|
|
@ -117,6 +117,31 @@ Fetched values of eight characters or more are masked before use, while shorter
|
|||
|
||||
To reproduce the CI topology on a dedicated machine, `bash .github/e2e-stack/up.sh` reads `tests/e2e/.env`, writes `stack.env` under `${E2E_STACK_DIR:-/tmp/litellm-e2e-stack}`, and `bash .github/e2e-stack/down.sh` stops it. Keep this directory private and remove its credential files and logs after use
|
||||
|
||||
### Secret manager lanes
|
||||
|
||||
`key_management_system` is global to the proxy, so the `secret_manager/` tests run once per backend, each against its own proxy. The backends are `hashicorp_vault` and `cyberark` (CyberArk Conjur). `E2E_SECRET_MANAGER` opts in and names the backend (a key of `secret_backends.BACKENDS`). The proxy boots from `gateway/secret_manager_<system>_ci_config.yml`, and the tests reach the same manager through that backend's `SecretStore`. The managers are enterprise features, so the proxy needs a license. `secret_manager/backend.sh` runs any backend in Docker and writes its env, so every lane runs the same way locally:
|
||||
|
||||
```bash
|
||||
bash tests/e2e/secret_manager/backend.sh up cyberark
|
||||
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/proxy.env; set +a; env -u OPENAI_API_KEY LITELLM_LICENSE=... \
|
||||
LITELLM_MASTER_KEY=sk-1234 DATABASE_URL=... uv run litellm --config tests/e2e/gateway/secret_manager_cyberark_ci_config.yml --port 4000)
|
||||
(set -a; . ~/.cache/litellm-e2e-secret-manager/cyberark/tests.env; set +a; OPENAI_API_KEY=... \
|
||||
uv run --group e2e-dev pytest tests/e2e/secret_manager/ -v)
|
||||
bash tests/e2e/secret_manager/backend.sh down cyberark
|
||||
```
|
||||
|
||||
`E2E_SECRET_MANAGER_PORT` moves the manager off its usual port (8200 for Vault, 8080 for Conjur), and `E2E_SECRET_MANAGER_DIR` moves the env files. Keep that directory private, because both files hold a working admin credential. Keep `OPENAI_API_KEY` out of the proxy's environment. The tests copy the runner's key into the manager under a fresh name per test, so a passing call proves the key came through the manager rather than the `os.environ` fallback `get_secret` takes when the manager errors
|
||||
|
||||
A backend declares what it supports in its `SecretBackend.capabilities`, and a test that needs something not every backend does carries `@pytest.mark.requires_capability(...)`, so it is deselected, not failed or skipped, on the lanes that lack it. CyberArk has no `deletes_stored_keys`, because the proxy's delete answers `not_supported` and Conjur keeps the key, so the delete test runs only on the Vault lane
|
||||
|
||||
To add a backend, leave the tests and markers alone and add:
|
||||
|
||||
1. `secret_manager/secret_store_<system>.py`: a `SecretStore` (`write`, `read` returning None when absent, idempotent `destroy`) over the manager's own API through `e2e_http`'s external helpers, read from `E2E_<SYSTEM>_*` env vars, and a `SecretBackend` whose `system` is the litellm `KeyManagementSystem` value and whose `capabilities` lists what it supports
|
||||
2. its entry in `secret_backends.BACKENDS`
|
||||
3. `gateway/secret_manager_<system>_ci_config.yml`, a copy of an existing lane's with only `key_management_system` changed
|
||||
4. an `up_<system>` function in `secret_manager/backend.sh` that starts the manager and writes `proxy.env` and `tests.env`
|
||||
5. a CI step that runs `backend.sh up <system>` (or the same containers as sidecars), boots the proxy with `proxy.env` and a license, and runs pytest with `tests.env`
|
||||
|
||||
### Record and replay
|
||||
|
||||
Record/replay scopes to the proxy's provider-bound traffic only. In `E2E_FIXTURE_MODE=record` the harness boots a local provider-edge server, edge-wired tests register their deployments with an `api_base` pointing at it, and every provider call the proxy makes is forwarded verbatim and written to a fixture bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`). `E2E_FIXTURE_MODE=replay` runs the same tests against the same live proxy and database, but the edge answers the proxy's provider calls from the bundle instead of the provider, so the run makes zero provider calls and spends nothing while key auth, routing, cost calculation, and spend-log writes all still execute for real. Unset (or `live`) behaves exactly as before the knob existed. Both record and replay need the proxy up; only the provider is taken out of the loop
|
||||
|
|
|
|||
41
tests/e2e/claude_code/cron_vm/Dockerfile
Normal file
41
tests/e2e/claude_code/cron_vm/Dockerfile
Normal file
|
|
@ -0,0 +1,41 @@
|
|||
FROM debian:bookworm-slim@sha256:3783cc01769c7b2b1b83a5c5ad96c815348e28ed7da68e2e3687004faa906251
|
||||
|
||||
ARG GH_VERSION=2.101.0
|
||||
ARG GH_SHA256=9bca2d1c16825f109907a23307628a2f0698fbf99662b73a5cf0b020293072b8
|
||||
ARG UV_VERSION=0.10.9
|
||||
ARG UV_SHA256=20d79708222611fa540b5c9ed84f352bcd3937740e51aacc0f8b15b271c57594
|
||||
ARG CLAUDE_CODE_VERSION=2.1.228
|
||||
ARG CLAUDE_CODE_SHA256=d535985e6941a3eb00179ccd7f52ceb0c6623a0305a518ebc4e6514f84a94c99
|
||||
|
||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates curl git jq procps iproute2 \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN curl -fsSLo /tmp/gh.tar.gz "https://github.com/cli/cli/releases/download/v${GH_VERSION}/gh_${GH_VERSION}_linux_amd64.tar.gz" \
|
||||
&& echo "${GH_SHA256} /tmp/gh.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/gh.tar.gz -C /usr/local/bin --strip-components=2 "gh_${GH_VERSION}_linux_amd64/bin/gh" \
|
||||
&& rm /tmp/gh.tar.gz
|
||||
|
||||
RUN curl -fsSLo /tmp/uv.tar.gz "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-x86_64-unknown-linux-gnu.tar.gz" \
|
||||
&& echo "${UV_SHA256} /tmp/uv.tar.gz" | sha256sum -c - \
|
||||
&& tar -xzf /tmp/uv.tar.gz -C /usr/local/bin --strip-components=1 uv-x86_64-unknown-linux-gnu/uv \
|
||||
&& rm /tmp/uv.tar.gz
|
||||
|
||||
RUN curl -fsSLo /tmp/claude "https://downloads.claude.ai/claude-code-releases/${CLAUDE_CODE_VERSION}/linux-x64/claude" \
|
||||
&& echo "${CLAUDE_CODE_SHA256} /tmp/claude" | sha256sum -c - \
|
||||
&& install -m 0755 /tmp/claude /usr/local/bin/claude \
|
||||
&& rm /tmp/claude
|
||||
|
||||
RUN groupadd --gid 1000 populator && useradd --uid 1000 --gid 1000 --create-home populator
|
||||
|
||||
ENV HOME=/home/populator \
|
||||
LITELLM_REPO=/opt/litellm \
|
||||
DISABLE_AUTOUPDATER=1
|
||||
|
||||
COPY --chown=populator:populator . /opt/litellm/tests/e2e/
|
||||
|
||||
USER populator
|
||||
WORKDIR /home/populator
|
||||
CMD ["/opt/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh"]
|
||||
|
|
@ -1,66 +1,59 @@
|
|||
# Cron VM setup for the Claude Code compatibility-matrix populator
|
||||
# Render cron job for the Claude Code compatibility-matrix populator
|
||||
|
||||
The populator runs daily on a dedicated GCP VM
|
||||
(`litellm-compatibility-matrix-populator`) rather than as a GitHub
|
||||
Action. Trade-offs:
|
||||
The populator runs daily as the Render cron job `litellm-compat-matrix`
|
||||
(Docker runtime, built from the `Dockerfile` in this directory) rather
|
||||
than as a GitHub Action or on a dedicated VM. Trade-offs:
|
||||
|
||||
- ✅ Real VM means we can `gh auth login` against an account that's
|
||||
already a collaborator on `BerriAI/litellm-docs`, instead of
|
||||
provisioning a GitHub App with `pull-requests: write`.
|
||||
- ✅ Persistent state (a single `~/litellm-cron-worktree/` and its `.venv`)
|
||||
is reused across runs, so each daily run does a fast `git checkout` +
|
||||
incremental `uv sync` rather than a fresh clone + cold sync.
|
||||
- ✅ No Docker dependency — the proxy runs directly via `uv run litellm`.
|
||||
- ⚠️ The VM has to actually be on. systemd's `Persistent=true` recovers
|
||||
from short outages, but a multi-day outage means the matrix goes
|
||||
stale until the VM is back.
|
||||
- ⚠️ Provider credentials live on the VM filesystem
|
||||
(`/etc/litellm-compat-matrix.env`) instead of GitHub secrets. Treat
|
||||
the VM as an environment with comparable blast radius to a CI runner.
|
||||
|
||||
This directory used to live at `tests/claude_code/cron_vm/` (paired with
|
||||
the standalone `tests/claude_code/` suite); it now runs the maintained
|
||||
`tests/e2e/claude_code/` suite instead. The pytest env interface changed
|
||||
accordingly: the runner exports `LITELLM_PROXY_URL` / `LITELLM_MASTER_KEY`
|
||||
(previously `LITELLM_PROXY_BASE_URL` / `LITELLM_PROXY_API_KEY`), the azure
|
||||
column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
||||
`AZURE_FOUNDRY_*`), and the GPT columns need `OPENAI_API_KEY` and
|
||||
`AZURE_API_BASE` / `AZURE_API_KEY` — see `litellm-compat-matrix.env.example`.
|
||||
- ✅ No machine to keep on or patch. Render builds the image from this
|
||||
directory on every push to `main` that touches `tests/e2e/**` and
|
||||
runs it on the schedule.
|
||||
- ✅ Credentials live in Render env vars and secret files, scoped to
|
||||
this one service, instead of on a VM filesystem.
|
||||
- ✅ The publish token still uses the `mateo-berri` account, which is a
|
||||
collaborator on `BerriAI/litellm-docs`, so no GitHub App with
|
||||
`pull-requests: write` has to be provisioned.
|
||||
- ⚠️ The disk is ephemeral, so every run starts from a fresh (blobless)
|
||||
clone of litellm plus a cold `uv sync`. That adds a few minutes on
|
||||
top of the ~10 minute test run; the job's 12 hour ceiling is nowhere
|
||||
near.
|
||||
- ⚠️ The Claude Code CLI version under test is pinned in the
|
||||
`Dockerfile` (`CLAUDE_CODE_VERSION` + its checksum). Bumping it is a
|
||||
PR, see the gotchas below.
|
||||
|
||||
## Layout
|
||||
|
||||
| File | Purpose |
|
||||
| --- | --- |
|
||||
| `run_daily.sh` | The actual cron job. Resolves versions, updates the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
|
||||
| `Dockerfile` | The image Render builds: Debian bookworm-slim plus pinned, checksum-verified `gh`, `uv`, and the Claude Code CLI, with this `tests/e2e/` tree copied to `/opt/litellm/tests/e2e/`. Runs as the non-root user `populator` (uid/gid 1000, which is what Render's secret files are readable by). |
|
||||
| `run_daily.sh` | The actual cron job. Resolves versions, clones the worktree, boots the proxy, runs pytest, builds the JSON, opens (or updates) a docs PR, sweeps stale compat-matrix PRs. |
|
||||
| `build_matrix.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.build_from_paths`. Exists only because the bash script needs *some* way to render the per-cell aggregation, and the builder is already Python. |
|
||||
| `check_regressions.py` | Tiny Python CLI that wraps `claude_code.matrix_builder.find_regressions`. Diffs the freshly built matrix against the currently-published one and exits `3` if any cell flipped green→red, which gates auto-merge. |
|
||||
| `litellm-compat-matrix.service` | systemd oneshot that invokes `run_daily.sh`. |
|
||||
| `litellm-compat-matrix.timer` | `OnCalendar=*-*-* 06:00:00 UTC`, `Persistent=true`. |
|
||||
| `litellm-compat-matrix.env.example` | Template for `/etc/litellm-compat-matrix.env`. |
|
||||
| `litellm-compat-matrix.env.example` | The service's env vars, one per line, with what each is for. |
|
||||
|
||||
## What `run_daily.sh` does
|
||||
|
||||
1. **Resolves the latest LiteLLM final release tag** (newest bare
|
||||
`vX.Y.Z`, skipping `-rc.N`/`-dev.N` pre-releases) by paging the
|
||||
GitHub Releases API (`curl | jq`).
|
||||
2. **Reads the local Claude Code CLI version** via `claude --version`.
|
||||
The cron does not auto-upgrade the CLI — operators do that
|
||||
out-of-band by running `npm install -g @anthropic-ai/claude-code@latest`.
|
||||
3. **Updates the persistent worktree** at `~/litellm-cron-worktree/`:
|
||||
`git fetch --tags --force`, `git reset --hard`,
|
||||
`git clean -fdx -e .venv -e .uv-bin`, `git checkout --force <tag>`.
|
||||
The `.venv` is preserved across runs so `uv sync --frozen` is
|
||||
incremental. Then **shims the test suite**: `tests/e2e/` in the
|
||||
worktree is rebuilt from the dev checkout — the `claude_code/` suite
|
||||
plus the five shared transport helpers it imports (`proxy_client.py`,
|
||||
`e2e_http.py`, `models.py`, `e2e_config.py`, `transport.py`) — so the
|
||||
cron always runs *today's* tests against the latest stable proxy. The
|
||||
tag's own `tests/e2e/` tree (including the EKS-harness `conftest.py`,
|
||||
whose imports the stable venv doesn't install) is deliberately not
|
||||
used.
|
||||
2. **Reads the Claude Code CLI version** via `claude --version`. That
|
||||
is whatever the `Dockerfile` pins; the job never upgrades it on its
|
||||
own.
|
||||
3. **Clones the worktree** at `~/litellm-cron-worktree/` (a
|
||||
`--filter=blob:none` clone, so only the checked-out tag's blobs are
|
||||
fetched), `git checkout --force <tag>`, then `uv sync --frozen
|
||||
--no-install-project` against a uv-managed CPython 3.12 followed by
|
||||
`uv pip install --no-build litellm==<version>`, so the proxy under
|
||||
test is the published PyPI wheel (what users install) rather than a
|
||||
source build: the tag builds a Rust extension through maturin, and
|
||||
the image ships no C or Rust toolchain. Then **shims the test suite**:
|
||||
`tests/e2e/` in the worktree is replaced by the image's copy of this
|
||||
whole tree, so the cron always runs *today's* tests against the
|
||||
latest stable proxy, and pytest runs with `--confcutdir` pointed at
|
||||
`claude_code/` so the tree's EKS-harness `conftest.py` (whose imports
|
||||
the stable venv doesn't install) is never loaded. The tag's own
|
||||
`tests/e2e/` is deliberately not used.
|
||||
4. **Boots the proxy** as a `setsid` background process on port `4100`
|
||||
(so it can't collide with a developer's `:4000`), then polls
|
||||
`/health/liveliness` until it's up.
|
||||
bound to loopback, then polls `/health/liveliness` until it's up.
|
||||
5. **Runs pytest** on `tests/e2e/claude_code/` with `LITELLM_PROXY_URL`
|
||||
pointed at the proxy and `COMPAT_RESULTS_PATH` set so the conftest
|
||||
hook writes the per-test results artifact. Test failures become
|
||||
|
|
@ -74,8 +67,9 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
|||
`mateo-berri` token has write access, so this is a same-repo branch,
|
||||
not a fork), `gh pr create`. A re-run on the same day fast-forwards
|
||||
the existing branch and `gh pr create` no-ops ("a pull request for
|
||||
branch ... already exists" is treated as success). These PRs are no
|
||||
longer gated on a second human review.
|
||||
branch ... already exists" is treated as success). If the JSON is
|
||||
byte-identical to what `main` already publishes, the push is skipped
|
||||
entirely. These PRs are not gated on a second human review.
|
||||
8. **Gates auto-merge on a regression check**: before enabling
|
||||
auto-merge, `check_regressions.py` diffs the new matrix against the
|
||||
one currently on `main`. Auto-merge (`gh pr merge --auto --squash`)
|
||||
|
|
@ -89,107 +83,155 @@ column reads `AZURE_AI_API_KEY` / `AZURE_AI_API_BASE` (previously
|
|||
human reviews before it lands on the public table. The check fails
|
||||
*closed*: if it errors, auto-merge is withheld.
|
||||
9. **Sweeps stale compat-matrix PRs**: once today's PR exists, every
|
||||
other open `compat-matrix/*` PR on the docs repo is closed (and its
|
||||
bot-owned branch deleted), so at most one compat-matrix PR is ever
|
||||
open — the newest.
|
||||
other open `compat-matrix/*` PR that the publishing account opened
|
||||
from a branch on the docs repo itself is closed (and its bot-owned
|
||||
branch deleted), so at most one compat-matrix PR is ever open — the
|
||||
newest. A contributor's PR under that prefix is never touched.
|
||||
|
||||
## One-time VM setup
|
||||
## The Render service
|
||||
|
||||
Run as `mateo` on the cron VM:
|
||||
Everything below is what the live service is set to; recreate it with
|
||||
the same values if it ever has to be rebuilt.
|
||||
|
||||
| Setting | Value |
|
||||
| --- | --- |
|
||||
| Workspace | Litellm (the one that already builds the other litellm services) |
|
||||
| Type | Cron job, Docker runtime |
|
||||
| Repo / branch | `BerriAI/litellm` @ `main` |
|
||||
| Dockerfile path | `tests/e2e/claude_code/cron_vm/Dockerfile` |
|
||||
| Docker build context | `tests/e2e` (the repo root `.dockerignore` excludes `tests`, so the context has to start below it) |
|
||||
| Build filter | included paths `tests/e2e/**` |
|
||||
| Schedule | `0 6 * * *` (06:00 UTC daily) |
|
||||
| Plan / region | `4c-16g` (4 CPU, 16 GB, what the dashboard calls Pro Max; the suite fans out to ~75 concurrent CLI calls) / Oregon |
|
||||
| Env vars | every key in `litellm-compat-matrix.env.example` |
|
||||
| Secret files | `github-token` (the publish PAT, one line) and `vertex-service-account.json` (the Vertex service-account key) |
|
||||
|
||||
Render mounts secret files at `/etc/secrets/<name>`, which is where
|
||||
`CREDENTIALS_DIRECTORY` and `GOOGLE_APPLICATION_CREDENTIALS` in the env
|
||||
example point. Render also passes env vars to `docker build` as build
|
||||
args, which is why the `Dockerfile` declares no `ARG` that could ever
|
||||
be given a secret's name.
|
||||
|
||||
Creating it through the API looks like this (fill `envVars` and
|
||||
`secretFiles` from the env example and the two secrets; `ownerId` is
|
||||
the workspace id from `GET /v1/owners`):
|
||||
|
||||
```bash
|
||||
# 1. Toolchain
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y git nodejs npm jq curl
|
||||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
sudo apt-get install -y gh # or follow https://cli.github.com/
|
||||
|
||||
# 2. Claude Code CLI (the cron does NOT auto-upgrade this; rerun this
|
||||
# line out-of-band when you want a fresh CLI to be tested)
|
||||
sudo npm install -g @anthropic-ai/claude-code@latest
|
||||
|
||||
# 3. Litellm checkout. Used by systemd's WorkingDirectory and as the
|
||||
# source of the .service / .timer files. The cron itself runs out
|
||||
# of the separate worktree at ~/litellm-cron-worktree/.
|
||||
mkdir -p ~/litellm
|
||||
git clone https://github.com/BerriAI/litellm.git ~/litellm/litellm
|
||||
git -C ~/litellm/litellm checkout litellm_internal_staging
|
||||
|
||||
# 4. gh auth — must be a collaborator on BerriAI/litellm-docs.
|
||||
gh auth login # follow prompts; pick HTTPS + token paste flow
|
||||
|
||||
# 5. Provider credentials + the publish token.
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.env.example \
|
||||
/etc/litellm-compat-matrix.env
|
||||
sudoedit /etc/litellm-compat-matrix.env # fill in real values
|
||||
sudo chmod 0600 /etc/litellm-compat-matrix.env
|
||||
# The mateo-berri PAT lives in its own file, mapped into the service via
|
||||
# systemd LoadCredential so it stays out of the test processes' env
|
||||
# (see the env.example comment for why).
|
||||
sudo install -m 0600 /dev/null /etc/litellm-compat-matrix-github-token
|
||||
sudoedit /etc/litellm-compat-matrix-github-token # single line: the PAT
|
||||
|
||||
# 6. systemd units.
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
|
||||
sudo cp ~/litellm/litellm/tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now litellm-compat-matrix.timer
|
||||
curl -fsS https://api.render.com/v1/services \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"type": "cron_job",
|
||||
"name": "litellm-compat-matrix",
|
||||
"ownerId": "<workspace id>",
|
||||
"repo": "https://github.com/BerriAI/litellm",
|
||||
"branch": "main",
|
||||
"autoDeploy": "yes",
|
||||
"buildFilter": {"paths": ["tests/e2e/**"], "ignoredPaths": []},
|
||||
"envVars": [{"key": "ANTHROPIC_API_KEY", "value": "..."}],
|
||||
"secretFiles": [{"name": "github-token", "content": "..."},
|
||||
{"name": "vertex-service-account.json", "content": "..."}],
|
||||
"serviceDetails": {
|
||||
"runtime": "docker",
|
||||
"schedule": "0 6 * * *",
|
||||
"plan": "4c-16g",
|
||||
"region": "oregon",
|
||||
"envSpecificDetails": {
|
||||
"dockerfilePath": "tests/e2e/claude_code/cron_vm/Dockerfile",
|
||||
"dockerContext": "tests/e2e"
|
||||
}
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
## Operating it
|
||||
|
||||
```bash
|
||||
# When does it run next?
|
||||
systemctl list-timers litellm-compat-matrix.timer
|
||||
# Trigger a real run right now (PRs to litellm-docs). The id is the
|
||||
# service id (`crn-...`) from the dashboard URL or `GET /v1/services`.
|
||||
curl -fsS -X POST "https://api.render.com/v1/cron-jobs/${CRON_ID}/runs" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Trigger a real run right now (PRs to litellm-docs).
|
||||
sudo systemctl start litellm-compat-matrix.service
|
||||
# Follow a run: the Logs tab on the service, or the API.
|
||||
curl -fsS "https://api.render.com/v1/logs?ownerId=${OWNER_ID}&resource=${CRON_ID}&limit=100" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Trigger a run that does NOT open a PR (good for first-time validation).
|
||||
SKIP_PUBLISH=1 ~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
# Rebuild the image after a merge that touches tests/e2e/** (see the
|
||||
# auto-deploy gotcha below). The deploy is done once its status is
|
||||
# `live`; a run triggered before that still uses the previous image.
|
||||
curl -fsS -X POST "https://api.render.com/v1/services/${CRON_ID}/deploys" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}" \
|
||||
-H 'Content-Type: application/json' -d '{"clearCache": "do_not_clear"}'
|
||||
curl -fsS "https://api.render.com/v1/services/${CRON_ID}/deploys?limit=1" \
|
||||
-H "Authorization: Bearer ${RENDER_API_KEY}"
|
||||
|
||||
# Narrow to one cell while debugging.
|
||||
SKIP_PUBLISH=1 PYTEST_K='basic_messaging_non_streaming and anthropic' \
|
||||
~/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
# A run that does NOT open a PR (first-time validation, CLI bumps):
|
||||
# set SKIP_PUBLISH=1 on the service, trigger a run, then remove it.
|
||||
# The matrix JSON is printed at the end of the run's log (nothing on
|
||||
# the container's disk outlives the run) and saved to
|
||||
# ~/compatibility-matrix.json for a local docker run.
|
||||
# PYTEST_K='basic_messaging_non_streaming and anthropic' narrows the
|
||||
# run to one cell the same way.
|
||||
|
||||
# Watch the most recent run.
|
||||
journalctl -u litellm-compat-matrix.service -f
|
||||
|
||||
# Read older runs.
|
||||
journalctl -u litellm-compat-matrix.service --since '2 days ago'
|
||||
|
||||
# Disable until further notice (e.g. while debugging).
|
||||
sudo systemctl disable --now litellm-compat-matrix.timer
|
||||
# Build and run the image locally (docker on Apple silicon needs the
|
||||
# platform flag; the context is tests/e2e, see the table above).
|
||||
docker build --platform linux/amd64 \
|
||||
-f tests/e2e/claude_code/cron_vm/Dockerfile -t compat-matrix tests/e2e
|
||||
docker run --rm --platform linux/amd64 \
|
||||
--env-file litellm-compat-matrix.env -e SKIP_PUBLISH=1 \
|
||||
-v "$PWD/secrets:/etc/secrets:ro" compat-matrix
|
||||
```
|
||||
|
||||
## Gotchas
|
||||
|
||||
- **The venv is pinned to Python 3.12 (`CRON_PYTHON_VERSION`).** The
|
||||
e2e suite uses PEP 695 `type` aliases, which the VM's system Python
|
||||
(3.11) can't parse; `run_daily.sh` has uv fetch a managed CPython
|
||||
e2e suite uses PEP 695 `type` aliases, which the image's Debian
|
||||
Python can't parse; `run_daily.sh` has uv fetch a managed CPython
|
||||
into `~/litellm-cron-worktree/.uv-python/` and syncs the venv against
|
||||
it. The first run after a version bump is a cold venv rebuild.
|
||||
- **The proxy port is `4100`, not `4000`.** This is so a developer SSH'd
|
||||
into the same VM with their own `:4000` proxy doesn't collide with a
|
||||
cron run. Override with `PROXY_PORT=...` in `/etc/litellm-compat-matrix.env`
|
||||
if you need to.
|
||||
it.
|
||||
- **The proxy port is `4100`, not `4000`.** Kept from the VM days so a
|
||||
developer running the script locally next to their own `:4000` proxy
|
||||
doesn't collide. Override with `PROXY_PORT=...`.
|
||||
- **`uv sync --frozen` requires the resolved tag to be tagged on
|
||||
GitHub.** If the latest stable release was made but not pushed as a
|
||||
git tag, the `git checkout` step fails. Push the tag, then rerun.
|
||||
GitHub, and the wheel install requires it on PyPI.** If the latest
|
||||
stable release was made but not pushed as a git tag, the `git
|
||||
checkout` step fails; push the tag, then rerun. PyPI has had every
|
||||
stable version days before its GitHub release so far (1.102.0 was
|
||||
uploaded 2026-09-20, released on GitHub 2026-09-22), so the
|
||||
`--no-build` install failing means the wheel is genuinely missing,
|
||||
not late.
|
||||
- **Pushes do not redeploy the service; deploy by hand.** `autoDeploy`
|
||||
is `yes` on the service, but Render only hears about pushes through
|
||||
its GitHub app, which is not installed on the `BerriAI` org (an org
|
||||
admin step), so no push to the branch has ever started a deploy.
|
||||
After a merge that changes anything under `tests/e2e/**`, run the
|
||||
deploy command from the operating section (or "Manual Deploy" on the
|
||||
dashboard) and wait for `live` before triggering a run, otherwise
|
||||
the next scheduled run still executes the old image.
|
||||
- **Publish-token rotation is your problem.** The cron does not
|
||||
refresh the token; if `mateo-berri`'s PAT in
|
||||
`/etc/litellm-compat-matrix-github-token` expires, the run fails at
|
||||
the `git push`/`gh pr create` step with a 401 ("Bad credentials" /
|
||||
"Authentication failed"). Mint a fresh PAT and update that file.
|
||||
The token needs write access to `BerriAI/litellm-docs` (classic
|
||||
`repo` scope, or fine-grained Contents:RW + Pull requests:RW). It is
|
||||
delivered via systemd `LoadCredential`, not the env file, so pytest,
|
||||
the proxy, and the claude CLI never inherit it; manual runs export
|
||||
`GITHUB_TOKEN` instead.
|
||||
- **First run after upgrading the Claude Code CLI is the riskiest one.**
|
||||
If the new CLI changes its wire format the matrix run can produce
|
||||
systematic failures. Always run with `SKIP_PUBLISH=1` after a CLI
|
||||
upgrade before letting the next scheduled fire happen.
|
||||
- **Disk:** the worktree's `.venv` is ~1.3 GB and the `.git` directory
|
||||
is ~1 GB. Plan for at least 5 GB free on the VM, otherwise
|
||||
`uv sync` will fail mid-run and leave you with a half-installed venv.
|
||||
refresh the token; if `mateo-berri`'s PAT in the `github-token`
|
||||
secret file expires, the run fails at the `git push`/`gh pr create`
|
||||
step with a 401 ("Bad credentials" / "Authentication failed"). Mint
|
||||
a fresh PAT and replace the secret file on the service. The token
|
||||
needs write access to `BerriAI/litellm-docs` (classic `repo` scope,
|
||||
or fine-grained Contents:RW + Pull requests:RW). It is delivered as
|
||||
a file, not an env var, so pytest, the proxy, and the claude CLI
|
||||
never inherit it; manual runs export `GITHUB_TOKEN` instead.
|
||||
- **Bumping the Claude Code CLI is a PR.** Change `CLAUDE_CODE_VERSION`
|
||||
in the `Dockerfile` and set `CLAUDE_CODE_SHA256` to the `linux-x64`
|
||||
checksum from
|
||||
`https://downloads.claude.ai/claude-code-releases/<version>/manifest.json`.
|
||||
The first run on a new CLI is the riskiest one: if the new CLI
|
||||
changes its wire format the matrix run can produce systematic
|
||||
failures, so trigger a `SKIP_PUBLISH=1` run before the next scheduled
|
||||
fire. `gh` and `uv` bump the same way, with the checksum from the
|
||||
release's `gh_<version>_checksums.txt` and the tarball's `.sha256`
|
||||
sidecar respectively.
|
||||
- **A local build on Apple silicon only proves the image assembles.**
|
||||
Under QEMU the Claude Code binary (a Bun executable) dies with
|
||||
`CPU lacks AVX support` and `gh` panics in the Go runtime, so
|
||||
`claude --version` and a full run are verified with a
|
||||
`SKIP_PUBLISH=1` run on Render, not locally.
|
||||
- **Nothing persists between runs.** A failed run leaves no
|
||||
half-installed venv behind, but also no cache: don't expect a rerun
|
||||
to be faster than the first one.
|
||||
|
|
|
|||
|
|
@ -1,9 +1,8 @@
|
|||
# Environment file consumed by `litellm-compat-matrix.service`.
|
||||
# Environment variables of the Render cron job `litellm-compat-matrix`.
|
||||
#
|
||||
# Install at `/etc/litellm-compat-matrix.env` and chmod 0600.
|
||||
# `EnvironmentFile=-` in the unit means the service is allowed to start
|
||||
# even if this file is missing, but the populator will fail at the
|
||||
# first provider request without these credentials.
|
||||
# Set every value here on the Render service (Environment tab, or the
|
||||
# `envVars` list of the create-service call in README.md). A local run
|
||||
# passes a filled-in copy with `docker run --env-file`.
|
||||
|
||||
# Anthropic
|
||||
ANTHROPIC_API_KEY=
|
||||
|
|
@ -17,11 +16,12 @@ AWS_BEARER_TOKEN_BEDROCK=
|
|||
AWS_REGION_NAME=us-east-1
|
||||
|
||||
# Vertex AI (vertex_ai + vertex_ai_gpt columns).
|
||||
# On the GCP VM, the default service-account ADC from the metadata server
|
||||
# is used -- no JSON key file is needed. If you ever need to run outside
|
||||
# GCP, also export GOOGLE_APPLICATION_CREDENTIALS=/path/to/sa.json.
|
||||
# The service-account key JSON is the Render secret file
|
||||
# `vertex-service-account.json`, mounted at /etc/secrets, and
|
||||
# GOOGLE_APPLICATION_CREDENTIALS points google-auth at it.
|
||||
VERTEXAI_PROJECT=
|
||||
VERTEXAI_LOCATION=global
|
||||
GOOGLE_APPLICATION_CREDENTIALS=/etc/secrets/vertex-service-account.json
|
||||
|
||||
# Azure AI Foundry (azure column — Claude models on Foundry)
|
||||
AZURE_AI_API_KEY=
|
||||
|
|
@ -35,19 +35,20 @@ AZURE_API_BASE=
|
|||
AZURE_API_KEY=
|
||||
|
||||
# The publish PAT (mateo-berri, write access on BerriAI/litellm-docs)
|
||||
# deliberately does NOT live in this file. Everything here lands in the
|
||||
# process environment of pytest, the proxy, and the model-driven claude
|
||||
# CLI, where any same-UID reader can lift it from /proc/<pid>/environ.
|
||||
# Instead, install the token at /etc/litellm-compat-matrix-github-token
|
||||
# (chmod 0600, single line); the service maps it in via systemd
|
||||
# LoadCredential and run_daily.sh keeps it out of every child process
|
||||
# env. Used to (a) resolve the latest stable release, (b) push the
|
||||
# daily compat-matrix branch directly to BerriAI/litellm-docs, (c) open
|
||||
# the same-repo PR, and (d) enable squash auto-merge on it. Scopes:
|
||||
# deliberately is NOT an env var. Everything here lands in the process
|
||||
# environment of pytest, the proxy, and the model-driven claude CLI,
|
||||
# where any same-UID reader can lift it from /proc/<pid>/environ.
|
||||
# Instead, the token is the Render secret file `github-token` (single
|
||||
# line), mounted under CREDENTIALS_DIRECTORY, and run_daily.sh reads it
|
||||
# from there and keeps it out of every child process env. Used to
|
||||
# (a) resolve the latest stable release, (b) push the daily
|
||||
# compat-matrix branch directly to BerriAI/litellm-docs, (c) open the
|
||||
# same-repo PR, and (d) enable squash auto-merge on it. Scopes:
|
||||
# classic `repo` + `workflow`, or fine-grained on BerriAI/litellm-docs
|
||||
# with Contents:RW + Pull requests:RW + Workflows:RW.
|
||||
# Manual runs export GITHUB_TOKEN instead, or skip publishing entirely
|
||||
# with SKIP_PUBLISH=1 (only writes the matrix JSON locally).
|
||||
CREDENTIALS_DIRECTORY=/etc/secrets
|
||||
|
||||
# Optional: the bedrock_mantle column is opt-in because the AWS account
|
||||
# needs the Mantle (OpenAI-on-Bedrock) models enabled. Without this the
|
||||
|
|
@ -59,9 +60,9 @@ AZURE_API_KEY=
|
|||
# usually run them. Skipped cells are recorded as not_tested.
|
||||
# COMPAT_OPENAI_GPT_CELLS=1
|
||||
|
||||
# Optional overrides; defaults are sensible for the cron VM.
|
||||
# Optional overrides; defaults are sensible for the cron job.
|
||||
# PROXY_PORT=4100
|
||||
# LITELLM_WORKTREE=/home/mateo/litellm-cron-worktree
|
||||
# LITELLM_WORKTREE=/home/populator/litellm-cron-worktree
|
||||
# DOCS_REPO=BerriAI/litellm-docs
|
||||
# DOCS_BRANCH=main
|
||||
# DOCS_TARGET_PATH=src/data/compatibility-matrix.json
|
||||
|
|
|
|||
|
|
@ -1,113 +0,0 @@
|
|||
# systemd service for the Claude Code compatibility-matrix populator.
|
||||
#
|
||||
# Triggered by `litellm-compat-matrix.timer`; not started directly. The
|
||||
# unit is a `Type=oneshot` so the timer's `OnCalendar=` semantics
|
||||
# describe "run once per day" cleanly — there's no long-lived daemon to
|
||||
# supervise; each invocation runs the populator end-to-end and exits.
|
||||
#
|
||||
# Install
|
||||
# -------
|
||||
#
|
||||
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.service /etc/systemd/system/
|
||||
# sudo cp tests/e2e/claude_code/cron_vm/litellm-compat-matrix.timer /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload
|
||||
# sudo systemctl enable --now litellm-compat-matrix.timer
|
||||
#
|
||||
# Paths are hard-coded to /home/mateo rather than using systemd's %h
|
||||
# specifier. Why: in *system* units (this one), %h is expanded at
|
||||
# parse time against the *manager's* home -- which is /root for PID 1
|
||||
# -- and *not* against the User= directive. That mismatch makes
|
||||
# ReadWritePaths point at /root/.cache (which doesn't exist), causing
|
||||
# the namespace setup to fail with status=226/NAMESPACE before the
|
||||
# script ever runs. The runtime user (`User=mateo`) must:
|
||||
#
|
||||
# * have a checkout of `BerriAI/litellm` at `~/litellm/litellm` so the
|
||||
# publisher module is importable;
|
||||
# * have a uv venv at `~/litellm/litellm/.venv` (created by
|
||||
# `uv sync --frozen` inside that checkout once);
|
||||
# * have `gh` already authenticated against an account with
|
||||
# `pull-requests: write` on `BerriAI/litellm-docs`;
|
||||
# * have provider credentials exported in `/etc/litellm-compat-matrix.env`
|
||||
# (see `litellm-compat-matrix.env.example` in this directory);
|
||||
# * have the mateo-berri publish PAT at
|
||||
# `/etc/litellm-compat-matrix-github-token` (chmod 0600, single
|
||||
# line), delivered via `LoadCredential=` below.
|
||||
|
||||
[Unit]
|
||||
Description=Claude Code compatibility-matrix populator (oneshot)
|
||||
Documentation=file:///home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/README.md
|
||||
Wants=network-online.target
|
||||
After=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
User=mateo
|
||||
Group=mateo
|
||||
|
||||
# Provider credentials + any gh/PROXY_PORT overrides live here. Format
|
||||
# is the standard `KEY=value` one line per env var.
|
||||
EnvironmentFile=-/etc/litellm-compat-matrix.env
|
||||
|
||||
# The mateo-berri publish PAT is mapped in via the credential store, NOT
|
||||
# the EnvironmentFile, so it never lands in the process environment that
|
||||
# pytest, the proxy, and the model-driven claude CLI inherit (any
|
||||
# same-UID process can read /proc/<pid>/environ). run_daily.sh reads
|
||||
# ${CREDENTIALS_DIRECTORY}/github-token and hands it to gh per call.
|
||||
# Unlike EnvironmentFile= above, this is deliberately NOT optional: a
|
||||
# missing token file fails the unit at start instead of 30 minutes in.
|
||||
LoadCredential=github-token:/etc/litellm-compat-matrix-github-token
|
||||
|
||||
# systemd starts with a minimal PATH (~/usr/local/bin:/usr/bin:/bin).
|
||||
# `uv` and `claude` are installed under the runtime user's `~/.local/bin`
|
||||
# so we have to prepend it explicitly; otherwise run_daily.sh fails at
|
||||
# the up-front command-presence check.
|
||||
Environment=PATH=/home/mateo/.local/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
||||
# `HOME` is auto-set to /home/mateo when User=mateo is honored, but be
|
||||
# explicit so anything that reads $HOME (e.g. uv's cache lookup, the
|
||||
# claude CLI's per-session dir) sees the right value even if a future
|
||||
# refactor flips DynamicUser= or PrivateUsers= on.
|
||||
Environment=HOME=/home/mateo
|
||||
|
||||
WorkingDirectory=/home/mateo/litellm/litellm
|
||||
|
||||
ExecStart=/home/mateo/litellm/litellm/tests/e2e/claude_code/cron_vm/run_daily.sh
|
||||
|
||||
# 90 minutes is generous: cold runs do `git clone` + `uv sync` of a new
|
||||
# tag's lockfile, which can take a couple of minutes on a 2-vCPU VM,
|
||||
# plus the full feature x provider grid of pytest cells hitting several
|
||||
# cloud providers.
|
||||
TimeoutStartSec=90min
|
||||
|
||||
# A failed run shouldn't restart automatically — the next timer fire is
|
||||
# the right retry. Reruns of the same day's matrix are idempotent.
|
||||
Restart=no
|
||||
|
||||
# Security hardening: the populator only reads the litellm checkout and
|
||||
# the env-file; everything else it writes lives in either the worktree
|
||||
# (managed) or `/tmp` (cleaned up by tempfile).
|
||||
#
|
||||
# ReadWritePaths whitelist:
|
||||
# * litellm-cron-worktree - the long-lived stable-tag checkout +
|
||||
# its `.venv` (`uv sync` rewrites every
|
||||
# run) + `.uv-bin` (pinned `uv` binary
|
||||
# cache).
|
||||
# * .cache - uv's wheel cache (~/.cache/uv) so we
|
||||
# don't redownload pinned deps each run.
|
||||
# * .claude - `claude` CLI's per-session state under
|
||||
# `~/.claude/projects/<sha>/`; created
|
||||
# on every `claude --print` invocation.
|
||||
# * .config/gh - `gh` CLI host config; technically not
|
||||
# needed when we pass GH_TOKEN inline,
|
||||
# but cheap to whitelist and prevents
|
||||
# future regressions if a code path
|
||||
# ever falls back to the host config.
|
||||
# * /tmp - mktemp -d workdir + proxy logs.
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=read-only
|
||||
ReadWritePaths=/home/mateo/litellm-cron-worktree /home/mateo/.cache /home/mateo/.claude /home/mateo/.config/gh /tmp
|
||||
PrivateTmp=true
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
# Daily timer for the compatibility-matrix populator.
|
||||
#
|
||||
# 06:00 UTC matches the original GitHub Actions cron schedule; chosen so
|
||||
# operators in US/EU timezones see fresh PRs at the start of their work
|
||||
# day.
|
||||
#
|
||||
# `Persistent=true` causes a missed run (VM was off / suspended) to
|
||||
# fire the next time the timer is started, which is the property we
|
||||
# want for a once-a-day job: the matrix should refresh as soon as the
|
||||
# VM is reachable again, not wait another 24h.
|
||||
#
|
||||
# `RandomizedDelaySec=10min` smears load if multiple matrix-style
|
||||
# pipelines are ever colocated on the same VM in the future.
|
||||
|
||||
[Unit]
|
||||
Description=Run the Claude Code compatibility-matrix populator daily
|
||||
|
||||
[Timer]
|
||||
OnCalendar=*-*-* 06:00:00 UTC
|
||||
Persistent=true
|
||||
RandomizedDelaySec=10min
|
||||
Unit=litellm-compat-matrix.service
|
||||
|
||||
[Install]
|
||||
WantedBy=timers.target
|
||||
|
|
@ -1,8 +1,8 @@
|
|||
#!/usr/bin/env bash
|
||||
# Daily Claude Code compatibility-matrix populator.
|
||||
#
|
||||
# Runs from the GCP VM `litellm-compatibility-matrix-populator` via the
|
||||
# systemd timer in this directory. The flow is:
|
||||
# Runs daily as the Render cron job `litellm-compat-matrix`, built from
|
||||
# the Dockerfile in this directory (see README.md). The flow is:
|
||||
#
|
||||
# 1. Resolve the latest LiteLLM final release tag from the GitHub
|
||||
# Releases API.
|
||||
|
|
@ -33,12 +33,12 @@
|
|||
# rather than spawning a new one. If the JSON is byte-identical to the
|
||||
# docs branch, we skip the push entirely.
|
||||
#
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, claude, npm.
|
||||
# Required commands on $PATH: git, uv, gh, jq, curl, claude.
|
||||
# Required state: a litellm checkout at $LITELLM_REPO (this file lives in
|
||||
# it), $WORKTREE is created on first run, gh is already authenticated.
|
||||
# it); $WORKTREE is created on first run.
|
||||
#
|
||||
# Override any default by setting the matching env var; see the systemd
|
||||
# unit for the production wiring.
|
||||
# Override any default by setting the matching env var; see README.md
|
||||
# for the production wiring.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
|
|
@ -52,12 +52,12 @@ DOCS_TARGET_PATH="${DOCS_TARGET_PATH:-src/data/compatibility-matrix.json}"
|
|||
SKIP_PUBLISH="${SKIP_PUBLISH:-0}"
|
||||
PYTEST_K="${PYTEST_K:-}"
|
||||
# The e2e suite uses PEP 695 `type` aliases, so the venv needs Python
|
||||
# >= 3.12 (also what repo CI runs) even when the VM's system python is
|
||||
# >= 3.12 (also what repo CI runs) even when the host's system python is
|
||||
# older. uv fetches a managed CPython of this version on first use --
|
||||
# checksum-verified against the manifest baked into the pinned uv
|
||||
# binary -- and installs it under ${WORKTREE}/.uv-python (see
|
||||
# UV_PYTHON_INSTALL_DIR below) so it lives inside the one tree the
|
||||
# systemd sandbox lets us write to.
|
||||
# UV_PYTHON_INSTALL_DIR below) so everything the run writes lives inside
|
||||
# the worktree.
|
||||
CRON_PYTHON_VERSION="${CRON_PYTHON_VERSION:-3.12}"
|
||||
# Merge method for auto-merge. BerriAI/litellm-docs only allows squash
|
||||
# merges (merge-commit and rebase are disabled at the repo level), so
|
||||
|
|
@ -113,9 +113,9 @@ for cmd in git uv gh jq curl claude; do
|
|||
done
|
||||
|
||||
# Publishing pushes the branch straight to BerriAI/litellm-docs and opens
|
||||
# the PR as mateo-berri, who has write access on the docs repo. Under
|
||||
# systemd the PAT arrives as a file via LoadCredential=, NOT via the
|
||||
# EnvironmentFile: several suite cells let the model-driven claude CLI
|
||||
# the PR as mateo-berri, who has write access on the docs repo. On
|
||||
# Render the PAT arrives as a secret file under ${CREDENTIALS_DIRECTORY},
|
||||
# NOT via an env var: several suite cells let the model-driven claude CLI
|
||||
# read arbitrary files as this user, and /proc/<pid>/environ of the
|
||||
# script, pytest, and the proxy would hand an env-borne token to any
|
||||
# same-UID reader. Kept as an unexported shell variable and passed per
|
||||
|
|
@ -125,13 +125,18 @@ done
|
|||
# quota.
|
||||
if [[ -z "${GITHUB_TOKEN:-}" && -n "${CREDENTIALS_DIRECTORY:-}" && -f "${CREDENTIALS_DIRECTORY}/github-token" ]]; then
|
||||
GITHUB_TOKEN="$(<"${CREDENTIALS_DIRECTORY}/github-token")"
|
||||
log "publish token source: systemd credential store"
|
||||
log "publish token source: ${CREDENTIALS_DIRECTORY}/github-token"
|
||||
elif [[ -n "${GITHUB_TOKEN:-}" ]]; then
|
||||
log "publish token source: process environment"
|
||||
fi
|
||||
if [[ "${SKIP_PUBLISH}" != "1" ]]; then
|
||||
[[ -n "${GITHUB_TOKEN:-}" ]] \
|
||||
|| die "publish token required: /etc/litellm-compat-matrix-github-token via LoadCredential under systemd, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|
||||
|| die "publish token required: the github-token secret file under CREDENTIALS_DIRECTORY, or an exported GITHUB_TOKEN for manual runs (or set SKIP_PUBLISH=1)"
|
||||
# The stale-PR sweep below closes only PRs this account opened, so the
|
||||
# login is resolved from the token once rather than hardcoded.
|
||||
PUBLISH_LOGIN="$(GH_TOKEN="${GITHUB_TOKEN}" gh api user --jq .login)" \
|
||||
|| die "could not resolve the publishing account from the github token"
|
||||
log "publishing as ${PUBLISH_LOGIN}"
|
||||
fi
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -205,7 +210,7 @@ log "local claude code: ${CLAUDE_CODE_VERSION}"
|
|||
if [[ ! -d "${WORKTREE}/.git" ]]; then
|
||||
log "first run: cloning litellm into ${WORKTREE}"
|
||||
mkdir -p "$(dirname "${WORKTREE}")"
|
||||
git clone https://github.com/BerriAI/litellm.git "${WORKTREE}"
|
||||
git clone --filter=blob:none https://github.com/BerriAI/litellm.git "${WORKTREE}"
|
||||
fi
|
||||
|
||||
log "updating worktree to ${LITELLM_VERSION}"
|
||||
|
|
@ -221,40 +226,27 @@ git -C "${WORKTREE}" clean -fdx -e .venv -e .uv-bin -e .uv-python
|
|||
git -C "${WORKTREE}" checkout --force "${LITELLM_VERSION}"
|
||||
|
||||
# Always rebuild tests/e2e/ in the worktree from the dev checkout,
|
||||
# regardless of what the resolved ${LITELLM_VERSION} tag ships. Two
|
||||
# reasons:
|
||||
# regardless of what the resolved ${LITELLM_VERSION} tag ships: the
|
||||
# matrix populator's job is to exercise *today's* tests against the
|
||||
# latest stable proxy, and the dev checkout carries the most recent
|
||||
# test fixes that haven't yet rolled into a stable release.
|
||||
#
|
||||
# * The matrix populator's job is to exercise *today's* tests against
|
||||
# the latest stable proxy. The dev checkout carries the most recent
|
||||
# test fixes that haven't yet rolled into a stable release, and we
|
||||
# want every cron run to pick those up the moment they land on
|
||||
# ${LITELLM_REPO}, not whenever the next stable release happens.
|
||||
# * The tag's own tests/e2e/ ships the full EKS e2e harness, whose
|
||||
# top-level conftest.py imports modules (e2e_db, lifecycle,
|
||||
# otel_client, ...) that the stable venv does not install. Copying
|
||||
# the whole tree would make pytest collection blow up on those
|
||||
# imports.
|
||||
#
|
||||
# So the shim is a fresh `rm -rf` of tests/e2e/ followed by copying ONLY
|
||||
# the claude_code suite plus the shared transport helpers it imports.
|
||||
# The whole tree is copied rather than an allowlist of the helpers the
|
||||
# suite imports: the helpers import each other (proxy_client ->
|
||||
# e2e_config -> fixture_mode -> ...), so a new edge in that graph turned
|
||||
# an allowlist into a ModuleNotFoundError at conftest load. The tree's
|
||||
# top-level conftest.py pulls in the full EKS harness (e2e_db,
|
||||
# lifecycle, ...), which the stable venv does not install, so the pytest
|
||||
# run below points --confcutdir at claude_code/ and never loads it.
|
||||
# pytest puts tests/e2e/ itself on sys.path (it has no __init__.py, while
|
||||
# claude_code/ does), which is what resolves both the `claude_code.*`
|
||||
# and the bare `proxy_client` / `e2e_http` imports inside the suite.
|
||||
E2E_HELPER_FILES=(proxy_client.py e2e_http.py models.py e2e_config.py transport.py)
|
||||
if [[ ! -d "${LITELLM_REPO}/tests/e2e/claude_code" ]]; then
|
||||
die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
|
||||
fi
|
||||
for helper in "${E2E_HELPER_FILES[@]}"; do
|
||||
[[ -f "${LITELLM_REPO}/tests/e2e/${helper}" ]] \
|
||||
|| die "missing shim helper: ${LITELLM_REPO}/tests/e2e/${helper}"
|
||||
done
|
||||
log "shimming tests/e2e/claude_code/ + helpers from ${LITELLM_REPO} (always-overwrite)"
|
||||
[[ -d "${LITELLM_REPO}/tests/e2e/claude_code" ]] \
|
||||
|| die "no shim source at ${LITELLM_REPO}/tests/e2e/claude_code"
|
||||
log "shimming tests/e2e/ from ${LITELLM_REPO} (always-overwrite)"
|
||||
rm -rf "${WORKTREE}/tests/e2e"
|
||||
mkdir -p "${WORKTREE}/tests/e2e"
|
||||
cp -r "${LITELLM_REPO}/tests/e2e/claude_code" "${WORKTREE}/tests/e2e/"
|
||||
for helper in "${E2E_HELPER_FILES[@]}"; do
|
||||
cp "${LITELLM_REPO}/tests/e2e/${helper}" "${WORKTREE}/tests/e2e/"
|
||||
done
|
||||
cp -r "${LITELLM_REPO}/tests/e2e/." "${WORKTREE}/tests/e2e/"
|
||||
|
||||
# litellm pins an exact uv version in pyproject.toml's [tool.uv]
|
||||
# `required-version` field, so a system uv that's newer or older
|
||||
|
|
@ -305,10 +297,18 @@ fi
|
|||
# actually serve. `--group proxy-dev` brings in pytest and the rest of
|
||||
# what tests/e2e/claude_code/ needs. `--python` pins the venv to
|
||||
# ${CRON_PYTHON_VERSION}; the first run after a version bump recreates
|
||||
# the venv from scratch (a one-time cold sync).
|
||||
# the venv from scratch (a one-time cold sync). `--no-install-project`
|
||||
# leaves litellm itself out: the tag builds a Rust extension through
|
||||
# maturin, which needs a C and Rust toolchain the image does not carry,
|
||||
# so the published PyPI wheel (what users install) goes in right after,
|
||||
# and every later `uv run` passes `--no-sync` so uv never tries to put
|
||||
# the source build back.
|
||||
export UV_PYTHON_INSTALL_DIR="${WORKTREE}/.uv-python"
|
||||
log "uv sync --frozen --group proxy-dev --extra proxy --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
|
||||
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --python "${CRON_PYTHON_VERSION}")
|
||||
log "uv sync --frozen --group proxy-dev --extra proxy --no-install-project --python ${CRON_PYTHON_VERSION} (uv ${PINNED_UV_VERSION:-system})"
|
||||
(cd "${WORKTREE}" && "${WORKTREE_UV}" sync --frozen --group proxy-dev --extra proxy --no-install-project --python "${CRON_PYTHON_VERSION}")
|
||||
LITELLM_WHEEL_VERSION="${LITELLM_VERSION#v}"
|
||||
log "installing the published litellm==${LITELLM_WHEEL_VERSION} wheel from PyPI"
|
||||
"${WORKTREE_UV}" pip install --python "${WORKTREE}/.venv/bin/python" --no-deps --no-build "litellm==${LITELLM_WHEEL_VERSION}"
|
||||
|
||||
PROXY_CONFIG="${WORKTREE}/tests/e2e/claude_code/test_config.yaml"
|
||||
[[ -f "${PROXY_CONFIG}" ]] || die "proxy config not found at ${PROXY_CONFIG} (shim incomplete?)"
|
||||
|
|
@ -321,10 +321,10 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
|
|||
# Bind the proxy to loopback only. The populator proxy is talked to
|
||||
# exclusively by the pytest run on the same host (the health check and
|
||||
# the test env set `LITELLM_PROXY_URL=http://127.0.0.1:...`),
|
||||
# so there's no reason to expose it on the VM's external interfaces.
|
||||
# so there's no reason to expose it on the container's external interfaces.
|
||||
# Without `--host`, `litellm` defaults to 0.0.0.0, which combined with
|
||||
# the predictable default `LITELLM_MASTER_KEY=sk-cron-matrix` would
|
||||
# allow anything that can reach :${PROXY_PORT} on the VM to authenticate
|
||||
# allow anything that can reach :${PROXY_PORT} on the host to authenticate
|
||||
# and burn upstream provider credentials.
|
||||
#
|
||||
# `setsid` puts the proxy in its own session+pgroup so cleanup() can
|
||||
|
|
@ -334,7 +334,7 @@ log "starting proxy on 127.0.0.1:${PROXY_PORT}"
|
|||
setsid env LITELLM_MASTER_KEY="${PROXY_API_KEY}" bash -c '
|
||||
echo "$$" > "$0"
|
||||
cd "$1"
|
||||
exec "$2" run litellm --config "$3" --host 127.0.0.1 --port "$4"
|
||||
exec "$2" run --no-sync litellm --config "$3" --host 127.0.0.1 --port "$4"
|
||||
' "${PROXY_PID_FILE}" "${WORKTREE}" "${WORKTREE_UV}" "${PROXY_CONFIG}" "${PROXY_PORT}" \
|
||||
>"${WORKDIR}/proxy.log" 2>&1 &
|
||||
disown
|
||||
|
|
@ -359,6 +359,7 @@ RESULTS_JSON="${WORKDIR}/compat-results.json"
|
|||
# the cron skips them if/when they land in the suite.
|
||||
PYTEST_ARGS=(
|
||||
tests/e2e/claude_code/
|
||||
--confcutdir=tests/e2e/claude_code
|
||||
"--ignore-glob=*_unit_tests*"
|
||||
)
|
||||
if [[ -n "${PYTEST_K}" ]]; then
|
||||
|
|
@ -373,7 +374,7 @@ set +e
|
|||
&& LITELLM_PROXY_URL="http://127.0.0.1:${PROXY_PORT}" \
|
||||
LITELLM_MASTER_KEY="${PROXY_API_KEY}" \
|
||||
COMPAT_RESULTS_PATH="${RESULTS_JSON}" \
|
||||
"${WORKTREE_UV}" run pytest "${PYTEST_ARGS[@]}"
|
||||
"${WORKTREE_UV}" run --no-sync pytest "${PYTEST_ARGS[@]}"
|
||||
)
|
||||
PYTEST_EXIT=$?
|
||||
set -e
|
||||
|
|
@ -392,7 +393,7 @@ MATRIX_JSON="${WORKDIR}/compatibility-matrix.json"
|
|||
log "building ${MATRIX_JSON}"
|
||||
(
|
||||
cd "${WORKTREE}" \
|
||||
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/build_matrix.py" \
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/build_matrix.py" \
|
||||
--manifest "${WORKTREE}/tests/e2e/claude_code/manifest.yaml" \
|
||||
--results "${RESULTS_JSON}" \
|
||||
--output "${MATRIX_JSON}" \
|
||||
|
|
@ -405,8 +406,9 @@ log "building ${MATRIX_JSON}"
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
if [[ "${SKIP_PUBLISH}" == "1" ]]; then
|
||||
cp "${MATRIX_JSON}" "${LITELLM_REPO}/compatibility-matrix.json"
|
||||
log "SKIP_PUBLISH=1; matrix written to ${LITELLM_REPO}/compatibility-matrix.json"
|
||||
cp "${MATRIX_JSON}" "${HOME}/compatibility-matrix.json"
|
||||
log "SKIP_PUBLISH=1; matrix saved to ${HOME}/compatibility-matrix.json and printed below"
|
||||
cat "${MATRIX_JSON}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
|
|
@ -415,7 +417,7 @@ BRANCH_NAME="compat-matrix/${LITELLM_VERSION}-${CLAUDE_CODE_VERSION}-${DATE_UTC}
|
|||
DOCS_CLONE="${WORKDIR}/litellm-docs"
|
||||
|
||||
log "cloning ${DOCS_REPO}@${DOCS_BRANCH}"
|
||||
gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
|
||||
GH_TOKEN="${GITHUB_TOKEN}" gh repo clone "${DOCS_REPO}" "${DOCS_CLONE}" -- --depth 1 --branch "${DOCS_BRANCH}"
|
||||
|
||||
cd "${DOCS_CLONE}"
|
||||
git config user.email "litellm-bot@berri.ai"
|
||||
|
|
@ -452,7 +454,7 @@ log "checking for green->red regressions vs the published matrix"
|
|||
set +e
|
||||
REGRESSION_REPORT="$(
|
||||
cd "${WORKTREE}" \
|
||||
&& "${WORKTREE_UV}" run python "${POPULATOR_DIR}/check_regressions.py" \
|
||||
&& "${WORKTREE_UV}" run --no-sync python "${POPULATOR_DIR}/check_regressions.py" \
|
||||
--old "${PUBLISHED_MATRIX}" \
|
||||
--new "${MATRIX_JSON}"
|
||||
)"
|
||||
|
|
@ -492,7 +494,7 @@ git commit -m "${COMMIT_MSG}"
|
|||
#
|
||||
# Plain --force (not --force-with-lease) is acceptable here: the
|
||||
# compat-matrix/* branch is bot-owned, only this script ever writes to
|
||||
# it, and runs are serialized by the systemd timer. --force-with-lease
|
||||
# it, and runs are serialized by the cron schedule. --force-with-lease
|
||||
# would require a fetch to populate the remote-tracking ref before each
|
||||
# push and adds no safety in this single-writer setup.
|
||||
PUBLISH_PUSH_URL="https://x-access-token:${GITHUB_TOKEN}@github.com/${DOCS_REPO}.git"
|
||||
|
|
@ -553,7 +555,7 @@ Generated by \`tests/e2e/claude_code/cron_vm/run_daily.sh\`. Close without mergi
|
|||
EOF
|
||||
)"
|
||||
|
||||
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as mateo-berri)"
|
||||
log "opening PR from ${BRANCH_NAME} -> ${DOCS_REPO}:${DOCS_BRANCH} (as ${PUBLISH_LOGIN})"
|
||||
# GH_TOKEN is mateo-berri's write-scoped token, the same identity used
|
||||
# for release-listing above. The branch lives on ${DOCS_REPO} itself, so
|
||||
# --head is a bare branch name (a same-repo PR), not `OWNER:BRANCH`.
|
||||
|
|
@ -644,15 +646,25 @@ fi
|
|||
#
|
||||
# Non-fatal: a sweep failure (rate limit, transient API error) leaves
|
||||
# stale PRs for the next run to retry; it must not fail the pipeline.
|
||||
#
|
||||
# The docs repo carries a few hundred open PRs, so the list has to page
|
||||
# past gh's default 30 (and the earlier 100, which never reached a
|
||||
# week-old compat-matrix PR and left it open for good).
|
||||
#
|
||||
# `compat-matrix/` is only a naming convention, so the prefix alone does
|
||||
# not make a PR this job's: a contributor can open a fork PR under that
|
||||
# name. Only PRs the publishing account itself opened from a branch on
|
||||
# the docs repo qualify; anything else stays untouched.
|
||||
log "sweeping stale compat-matrix PRs (keeping ${BRANCH_NAME})"
|
||||
set +e
|
||||
STALE_PRS="$(
|
||||
GH_TOKEN="${GITHUB_TOKEN}" gh pr list \
|
||||
--repo "${DOCS_REPO}" \
|
||||
--state open \
|
||||
--limit 100 \
|
||||
--json number,headRefName \
|
||||
--jq '.[] | select(.headRefName | startswith("compat-matrix/")) | "\(.number)\t\(.headRefName)"'
|
||||
--author "${PUBLISH_LOGIN}" \
|
||||
--limit 1000 \
|
||||
--json number,headRefName,isCrossRepository \
|
||||
--jq '.[] | select((.headRefName | startswith("compat-matrix/")) and (.isCrossRepository | not)) | "\(.number)\t\(.headRefName)"'
|
||||
)"
|
||||
while IFS=$'\t' read -r stale_pr stale_head; do
|
||||
[[ -z "${stale_pr}" ]] && continue
|
||||
|
|
@ -660,7 +672,7 @@ while IFS=$'\t' read -r stale_pr stale_head; do
|
|||
GH_TOKEN="${GITHUB_TOKEN}" gh pr close "${stale_pr}" \
|
||||
--repo "${DOCS_REPO}" \
|
||||
--delete-branch \
|
||||
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open." 2>&1 | sed 's/^/ /'
|
||||
--comment "Superseded by the newer daily compat-matrix PR from \`${BRANCH_NAME}\`; the populator keeps only the most recent compat-matrix PR open" 2>&1 | sed 's/^/ /'
|
||||
if [[ ${PIPESTATUS[0]} -eq 0 ]]; then
|
||||
log "closed stale compat-matrix PR #${stale_pr} (${stale_head})"
|
||||
else
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from e2e_config import (
|
|||
PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
PROXY_BASE_URL,
|
||||
REDIS_CHAOS_OPT_IN_ENV,
|
||||
SECRET_MANAGER_OPT_IN_ENV,
|
||||
WEEKLY_ANOMALY_OPT_IN_ENV,
|
||||
unique_marker,
|
||||
)
|
||||
|
|
@ -70,6 +71,7 @@ OPT_IN_MARKERS: Final = MappingProxyType(
|
|||
"provider_edge_host": PROVIDER_EDGE_HOST_OPT_IN_ENV,
|
||||
"otel_v2": OTEL_V2_OPT_IN_ENV,
|
||||
"otel_tls": OTEL_TLS_OPT_IN_ENV,
|
||||
"secret_manager": SECRET_MANAGER_OPT_IN_ENV,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -172,6 +174,11 @@ def pytest_configure(config: pytest.Config) -> None:
|
|||
"markers",
|
||||
"otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set",
|
||||
)
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
"secret_manager: needs a proxy booted from gateway/secret_manager_<system>_ci_config.yml against that live "
|
||||
"secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)",
|
||||
)
|
||||
|
||||
|
||||
def pytest_sessionstart(session: pytest.Session) -> None:
|
||||
|
|
|
|||
|
|
@ -40,7 +40,10 @@
|
|||
- {id: other.config.runtime_update.applies_at_runtime, module: other, tier: P0, area: config, assertions: [applies_at_runtime], source: "proxy_server.py:14014-14060", rationale: "/config/update persists to DB + invalidates cache"}
|
||||
- {id: other.config.passthrough.headers_forwarded, module: other, tier: P0, area: config, assertions: [headers_forwarded], source: "passthrough/utils.py forward_headers_from_request", rationale: "Custom pass-through static headers and x-pass-* client headers reach the upstream"}
|
||||
- {id: other.config.general_settings.alert_webhook_side_effect, module: other, tier: P1, area: config, assertions: [alert_webhook_side_effect], source: "proxy_server.py:14215", rationale: "alert_to_webhook_url auto-enables slack alerting"}
|
||||
- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "proxy_server.py:3984-4010", rationale: "Resolves secrets from Vault/KMS at startup"}
|
||||
- {id: other.config.secret_resolution.kms_integration, module: other, tier: P1, area: config, assertions: [kms_integration], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "A deployment whose api_key is os.environ/<name> gets its key from the configured secret manager when that name exists only in the manager"}
|
||||
- {id: other.config.secret_resolution.manager_value_used, module: other, tier: P1, area: config, assertions: [manager_value_used], source: "secret_managers/main.py get_secret / secret_manager/test_secret_manager_e2e.py", rationale: "The value the manager holds is what reaches the provider: a bogus key in the manager is rejected by the provider with 401, so a passing resolution test cannot be an env fallback"}
|
||||
- {id: other.config.secret_manager.virtual_key_stored, module: other, tier: P1, area: config, assertions: [virtual_key_stored], source: "key_management_event_hooks.py _store_virtual_key_in_secret_manager", rationale: "With store_virtual_keys, /key/generate writes the new key under prefix_for_stored_virtual_keys + key_alias in the manager"}
|
||||
- {id: other.config.secret_manager.virtual_key_deleted, module: other, tier: P1, area: config, assertions: [virtual_key_deleted], source: "key_management_event_hooks.py _delete_virtual_keys_from_secret_manager", rationale: "/key/delete removes the stored key from the manager, so a revoked key does not linger there"}
|
||||
- {id: other.config.overrides.audit_logged, module: other, tier: P1, area: config, assertions: [audit_logged], source: "config_override_endpoints.py:67-100", rationale: "Config override mutations audit-logged, values redacted"}
|
||||
- {id: other.key_mgmt.regenerate.grace_period_honored, module: other, tier: P1, area: auth, assertions: [grace_period_honored], source: "key_management_endpoints.py:4503-4560", rationale: "Old key valid during grace_period then revoked"}
|
||||
- {id: other.key_mgmt.spend_reset.resets_to_value, module: other, tier: P1, area: auth, assertions: [resets_to_value], source: "key_management_endpoints.py:4841", rationale: "reset_spend resets accumulated spend"}
|
||||
|
|
|
|||
|
|
@ -150,6 +150,7 @@ MCP_OAUTH_LIVE_OPT_IN_ENV: Final = "E2E_MCP_OAUTH_LIVE"
|
|||
PROVIDER_EDGE_HOST_OPT_IN_ENV: Final = "E2E_PROVIDER_EDGE_HOST_REACHABLE"
|
||||
OTEL_V2_OPT_IN_ENV: Final = "E2E_OTEL_V2"
|
||||
OTEL_TLS_OPT_IN_ENV: Final = "E2E_OTEL_EXPORTER_ENDPOINT"
|
||||
SECRET_MANAGER_OPT_IN_ENV: Final = "E2E_SECRET_MANAGER"
|
||||
ANOMALY_SESSIONS = int(os.environ.get("E2E_ANOMALY_SESSIONS", "6"))
|
||||
ANOMALY_TURNS_PER_SESSION = int(os.environ.get("E2E_ANOMALY_TURNS_PER_SESSION", "6"))
|
||||
ANOMALY_TURN_ATTEMPTS = int(os.environ.get("E2E_ANOMALY_TURN_ATTEMPTS", "3"))
|
||||
|
|
|
|||
|
|
@ -129,9 +129,9 @@ class ProbeResult(BaseModel):
|
|||
|
||||
|
||||
class ExternalWrite(BaseModel):
|
||||
"""Outcome of a write to a non-proxy API (an identity provider's admin API)
|
||||
that answers with a status and, on create, a Location header naming the new
|
||||
resource rather than a JSON body."""
|
||||
"""Outcome of a call to a non-proxy API (an identity provider's admin API, a
|
||||
secret manager) that answers with a status, on create a Location header naming
|
||||
the new resource, and a body kept as text rather than parsed as JSON."""
|
||||
|
||||
status_code: int
|
||||
location: str = ""
|
||||
|
|
@ -345,6 +345,50 @@ def request_with_retry[T: RetryableResponse](
|
|||
return issue()
|
||||
|
||||
|
||||
PROVIDER_RATE_LIMIT_MARKER: Final = "litellm.RateLimitError"
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS: Final = 4
|
||||
PROVIDER_RATE_LIMIT_BACKOFF_SECONDS: Final = 5.0
|
||||
|
||||
|
||||
def tolerate_provider_rate_limit[R: BaseModel](
|
||||
issue: Callable[[], Result[R]],
|
||||
*,
|
||||
attempts: int = PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
sleep: Callable[[float], None] = time.sleep,
|
||||
) -> Result[R]:
|
||||
"""Retry a call up to `attempts` times while the proxy relays the provider's own 429;
|
||||
any other outcome, the proxy's own 429 included, comes back at once."""
|
||||
for attempt in range(1, attempts):
|
||||
match issue():
|
||||
case RateLimitedError(body=body, retry_after_seconds=retry_after) if PROVIDER_RATE_LIMIT_MARKER in body:
|
||||
delay = retry_after or PROVIDER_RATE_LIMIT_BACKOFF_SECONDS * (1 << (attempt - 1))
|
||||
print(
|
||||
f"e2e-http: provider rate limit relayed by the proxy; retry {attempt}/{attempts - 1} in {delay}s",
|
||||
flush=True,
|
||||
)
|
||||
sleep(delay)
|
||||
case result:
|
||||
return result
|
||||
return issue()
|
||||
|
||||
|
||||
class ProxyErrorDetail(BaseModel):
|
||||
message: str
|
||||
type: str
|
||||
code: str
|
||||
|
||||
|
||||
class _ProxyErrorBody(BaseModel):
|
||||
error: ProxyErrorDetail
|
||||
|
||||
|
||||
def relayed_provider_rate_limit(outcome: RateLimitedError) -> ProxyErrorDetail | None:
|
||||
"""The provider's own 429 as the proxy relayed it, or None when the 429 is the proxy's own."""
|
||||
if PROVIDER_RATE_LIMIT_MARKER not in outcome.body:
|
||||
return None
|
||||
return _ProxyErrorBody.model_validate_json(outcome.body).error
|
||||
|
||||
|
||||
class ClassifiableResponse(Protocol):
|
||||
"""What classifying an outcome reads off a response. requests.Response satisfies
|
||||
it, and so does a fake, so the classification rules are testable on their own."""
|
||||
|
|
@ -491,6 +535,30 @@ def post_json_external(
|
|||
)
|
||||
|
||||
|
||||
def send_text_external(
|
||||
method: Literal["GET", "POST", "PATCH"],
|
||||
url: str,
|
||||
*,
|
||||
headers: BaseModel,
|
||||
content: str | None = None,
|
||||
timeout: float = 30.0,
|
||||
) -> ExternalWrite:
|
||||
"""Send an absolute URL outside the proxy a raw text body (or none) and keep the
|
||||
answer as text, for an API that takes and returns neither JSON nor forms: CyberArk
|
||||
Conjur takes a secret value or a YAML policy and returns a secret as its raw value."""
|
||||
try:
|
||||
resp = requests.request(
|
||||
method,
|
||||
url,
|
||||
headers=_headers(headers),
|
||||
data=content.encode() if content is not None else None,
|
||||
timeout=timeout,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
return ExternalWrite(status_code=-1, body=str(exc))
|
||||
return ExternalWrite(status_code=resp.status_code, body=resp.text)
|
||||
|
||||
|
||||
def delete_external(url: str, *, headers: BaseModel, timeout: float = 30.0) -> ExternalWrite:
|
||||
try:
|
||||
resp = requests.delete(url, headers=_headers(headers), timeout=timeout)
|
||||
|
|
@ -907,7 +975,10 @@ class PreparedForward:
|
|||
|
||||
|
||||
def prepare_forward(
|
||||
method: str, url: str, headers: dict[str, str], body: bytes | None,
|
||||
method: str,
|
||||
url: str,
|
||||
headers: dict[str, str],
|
||||
body: bytes | None,
|
||||
) -> PreparedForward | NetworkError:
|
||||
try:
|
||||
with requests.Session() as session:
|
||||
|
|
@ -926,7 +997,8 @@ def forward_prepared_stream(prepared: PreparedForward, timeout: float) -> Stream
|
|||
except requests.RequestException as exc:
|
||||
return NetworkError(message=str(exc))
|
||||
return StreamHead(
|
||||
resp.status_code, {name.lower(): value for name, value in resp.headers.items()},
|
||||
resp.status_code,
|
||||
{name.lower(): value for name, value in resp.headers.items()},
|
||||
primed_steps(_stream_steps(resp)),
|
||||
)
|
||||
|
||||
|
|
|
|||
8
tests/e2e/gateway/secret_manager_cyberark_ci_config.yml
Normal file
8
tests/e2e/gateway/secret_manager_cyberark_ci_config.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
store_model_in_db: true
|
||||
key_management_system: cyberark
|
||||
key_management_settings:
|
||||
access_mode: read_and_write
|
||||
store_virtual_keys: true
|
||||
prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/
|
||||
|
|
@ -0,0 +1,8 @@
|
|||
general_settings:
|
||||
master_key: os.environ/LITELLM_MASTER_KEY
|
||||
store_model_in_db: true
|
||||
key_management_system: hashicorp_vault
|
||||
key_management_settings:
|
||||
access_mode: read_and_write
|
||||
store_virtual_keys: true
|
||||
prefix_for_stored_virtual_keys: litellm-e2e/virtual-keys/
|
||||
|
|
@ -13,7 +13,13 @@ Each case asserts the feature actually happened, not just a 200. Coverage matrix
|
|||
call, so a never-seen prefix must come back cached on its very first call
|
||||
(Gemini's implicit caching cannot hit a cold prefix), the cached count must
|
||||
cover the marked block, and the spend row must be billed below the uncached
|
||||
price of the prompt.
|
||||
price of the cached tokens. Vertex's cache create is nondeterministic:
|
||||
identical bodies come back 200 or with the minimum-token 400 ("The cached
|
||||
content is of 1 tokens"), in failure bursts of 45 seconds and more, so up to
|
||||
eight never-seen prefixes are tried with a pause after each rejection. The
|
||||
billing check prices the cached tokens rather than prompt_tokens, which
|
||||
Vertex reports inclusive of the cached prefix on some calls and exclusive
|
||||
of it on others.
|
||||
- Anthropic (claude-haiku-4-5, direct): the same ``cache_control`` prefix over
|
||||
the OpenAI-compatible route; the second call must report cache-read tokens > 0.
|
||||
- OpenAI (gpt-5.6): automatic prompt caching needs no request marker, so the
|
||||
|
|
@ -50,7 +56,8 @@ VERTEX_MODEL = "vertex_ai/gemini-2.5-flash"
|
|||
ANTHROPIC_MODEL = "anthropic/claude-haiku-4-5-20251001"
|
||||
OPENAI_MODEL = "openai/gpt-5.6"
|
||||
VERTEX_CACHE_TTL: Final = "300s"
|
||||
VERTEX_COLD_CALL_ATTEMPTS: Final = 3
|
||||
VERTEX_COLD_CALL_ATTEMPTS: Final = 8
|
||||
VERTEX_COLD_CALL_PAUSE_SECONDS: Final = 15.0
|
||||
VERTEX_MINIMUM_CACHED_TOKENS: Final = 1024
|
||||
CACHED_SHARE_OF_PROMPT: Final = 0.9
|
||||
VERTEX_CACHE_REJECTION_MARKER: Final = "minimum token count to start explicit caching"
|
||||
|
|
@ -156,26 +163,36 @@ def _cold_cache_call(send: Callable[[str], Result[ChatResponse]]) -> ChatRespons
|
|||
return unwrap(result)
|
||||
|
||||
|
||||
def _first_engaged_cold_call(send: Callable[[str], Result[ChatResponse]]) -> ChatResponse | None:
|
||||
for attempt in range(1, VERTEX_COLD_CALL_ATTEMPTS + 1):
|
||||
candidate = _cold_cache_call(send)
|
||||
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS:
|
||||
return candidate
|
||||
if attempt < VERTEX_COLD_CALL_ATTEMPTS:
|
||||
print(
|
||||
f"cache_control: vertex did not engage the cache on cold attempt {attempt}/{VERTEX_COLD_CALL_ATTEMPTS}; "
|
||||
f"pausing {VERTEX_COLD_CALL_PAUSE_SECONDS}s before the next never-seen prefix",
|
||||
flush=True,
|
||||
)
|
||||
time.sleep(VERTEX_COLD_CALL_PAUSE_SECONDS)
|
||||
return None
|
||||
|
||||
|
||||
def _first_cold_call_reads_cache(model: str, send: Callable[[str], Result[ChatResponse]]) -> ChatResponse:
|
||||
completion: Final = next(
|
||||
(
|
||||
candidate
|
||||
for candidate in (_cold_cache_call(send) for _ in range(VERTEX_COLD_CALL_ATTEMPTS))
|
||||
if candidate is not None and _cached_read_tokens(candidate.usage) >= VERTEX_MINIMUM_CACHED_TOKENS
|
||||
),
|
||||
None,
|
||||
)
|
||||
completion: Final = _first_engaged_cold_call(send)
|
||||
assert completion is not None, (
|
||||
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control were each either "
|
||||
f"rejected by Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} "
|
||||
"cached tokens on their first call; explicit context caching did not engage"
|
||||
f"{model}: {VERTEX_COLD_CALL_ATTEMPTS} never-seen prompts marked with cache_control, spread over "
|
||||
f"{VERTEX_COLD_CALL_PAUSE_SECONDS * (VERTEX_COLD_CALL_ATTEMPTS - 1):.0f}s, were each either rejected by "
|
||||
f"Vertex's minimum-token check or served with fewer than {VERTEX_MINIMUM_CACHED_TOKENS} cached tokens on "
|
||||
"their first call; explicit context caching did not engage"
|
||||
)
|
||||
assert completion.choices, f"{model}: cached call returned no choices: {completion}"
|
||||
usage: Final = completion.usage
|
||||
cached: Final = _cached_read_tokens(usage)
|
||||
assert usage and usage.prompt_tokens and cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
|
||||
f"{model}: only {cached} of {usage.prompt_tokens if usage else None} prompt tokens were served from the "
|
||||
"cache; the cache_control block was not cached whole"
|
||||
assert usage and usage.prompt_tokens, f"{model}: cached completion carried no prompt_tokens: {usage}"
|
||||
assert cached >= CACHED_SHARE_OF_PROMPT * usage.prompt_tokens, (
|
||||
f"{model}: only {cached} of {usage.prompt_tokens} prompt tokens were served from the cache; the "
|
||||
"cache_control block was not cached whole"
|
||||
)
|
||||
return completion
|
||||
|
||||
|
|
@ -196,10 +213,11 @@ def _assert_billed_below_uncached_prompt(client: PassthroughClient, model: str,
|
|||
assert row.prompt_tokens == usage.prompt_tokens, (
|
||||
f"{model}: spend row prompt_tokens {row.prompt_tokens} != response prompt_tokens {usage.prompt_tokens}"
|
||||
)
|
||||
uncached_prompt_cost: Final = usage.prompt_tokens * _input_rate(client, model)
|
||||
assert row.spend is not None and row.spend < uncached_prompt_cost, (
|
||||
f"{model}: spend {row.spend} is not below the uncached price of the prompt alone ({uncached_prompt_cost} for "
|
||||
f"{usage.prompt_tokens} tokens); cache-read pricing was not applied"
|
||||
cached: Final = _cached_read_tokens(usage)
|
||||
uncached_read_cost: Final = cached * _input_rate(client, model)
|
||||
assert row.spend is not None and row.spend < uncached_read_cost, (
|
||||
f"{model}: spend {row.spend} is not below the uncached price of the {cached} tokens read from the cache "
|
||||
f"({uncached_read_cost}); cache-read pricing was not applied"
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -110,10 +110,11 @@ def _vision_messages() -> list[ChatMessage]:
|
|||
|
||||
def _assert_describes_cat(response: ChatResponse) -> None:
|
||||
assert response.choices, f"vision returned no choices: {response}"
|
||||
message = response.choices[0].message
|
||||
content = (message.content if message else None) or ""
|
||||
choice = response.choices[0]
|
||||
content = (choice.message.content if choice.message else None) or ""
|
||||
assert any(term in content.lower() for term in ("cat", "feline", "kitten", "kitty")), (
|
||||
f"vision response did not describe the image: {content[:200]}"
|
||||
f"vision response did not describe the image: {content[:200]!r} "
|
||||
f"(finish_reason={choice.finish_reason!r}, usage={response.usage})"
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -421,7 +422,11 @@ class TestVertexChatCompletions:
|
|||
model = self._register(client, resources, "e2e-vertex-vision")
|
||||
key = resources.key()
|
||||
|
||||
response = unwrap(client.proxy.chat(key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32)))
|
||||
response = unwrap(
|
||||
client.proxy.chat(
|
||||
key, ChatBody(model=model, messages=_vision_messages(), max_tokens=32, reasoning_effort="none")
|
||||
)
|
||||
)
|
||||
_assert_describes_cat(response)
|
||||
|
||||
@pytest.mark.covers(
|
||||
|
|
|
|||
|
|
@ -11,16 +11,28 @@ well-formed OCR document comes back. Per the e2e hard-fail contract, a case
|
|||
fails when no proxy answers and also fails once a request reaches it: the proxy
|
||||
fetches each provider's referenced secrets, so a
|
||||
missing credential surfaces as a live provider error rather than silent green.
|
||||
|
||||
The provider keys are shared with other pipelines, so a provider's rate limit can
|
||||
hold across the bounded retries. The case then accepts the gateway's faithful relay
|
||||
of that 429 (throttling_error, code 429) as its second expected outcome; any other
|
||||
non-success still fails at once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Protocol
|
||||
from typing import Final, Protocol
|
||||
|
||||
import pytest
|
||||
from e2e_config import unique_marker
|
||||
from e2e_http import assert_client_error, unwrap
|
||||
from e2e_http import (
|
||||
PROVIDER_RATE_LIMIT_ATTEMPTS,
|
||||
RateLimitedError,
|
||||
Success,
|
||||
assert_client_error,
|
||||
relayed_provider_rate_limit,
|
||||
tolerate_provider_rate_limit,
|
||||
)
|
||||
from lifecycle import ResourceManager
|
||||
from models import LiteLLMParamsBody, OcrBody, OcrDocument, OcrResponse
|
||||
from proxy_client import ProxyClient
|
||||
|
|
@ -146,24 +158,36 @@ def _assert_ocr_document(response: OcrResponse) -> None:
|
|||
assert response.pages[0].markdown is not None, "first page has no markdown"
|
||||
|
||||
|
||||
def _assert_provider_rate_limit_relayed(model: str, outcome: RateLimitedError) -> None:
|
||||
detail: Final = relayed_provider_rate_limit(outcome)
|
||||
assert detail is not None, f"{model}: the 429 is the gateway's own, not the provider's: {outcome.body}"
|
||||
assert (detail.type, detail.code) == ("throttling_error", "429"), f"{model}: provider 429 relayed as {detail!r}"
|
||||
print(
|
||||
f"{model}: the provider's rate limit held across {PROVIDER_RATE_LIMIT_ATTEMPTS} attempts; "
|
||||
f"the gateway relayed it as {detail.type} {detail.code}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
|
||||
class TestRustOcrGateway:
|
||||
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
|
||||
def test_rust_ocr_response(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase
|
||||
) -> None:
|
||||
def test_rust_ocr_response(self, proxy: ProxyClient, resources: ResourceManager, case: _OcrCase) -> None:
|
||||
model = f"rust-ocr-{case.suffix}-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, case.provider.litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
key = resources.key()
|
||||
|
||||
response = unwrap(proxy.ocr(key, OcrBody(model=model, document=case.document)))
|
||||
_assert_ocr_document(response)
|
||||
match tolerate_provider_rate_limit(lambda: proxy.ocr(key, OcrBody(model=model, document=case.document))):
|
||||
case Success(data=response):
|
||||
_assert_ocr_document(response)
|
||||
case RateLimitedError() as outcome:
|
||||
_assert_provider_rate_limit_relayed(model, outcome)
|
||||
case outcome:
|
||||
pytest.fail(f"{model}: {outcome!r}")
|
||||
|
||||
@pytest.mark.skip(reason="stage red: product gap, /v1/ocr 500s (aocr TypeError) on missing document instead of 400")
|
||||
@pytest.mark.covers("llm.ocr.openai.input_validation.nonstream.works")
|
||||
def test_missing_document_returns_error(
|
||||
self, proxy: ProxyClient, resources: ResourceManager
|
||||
) -> None:
|
||||
def test_missing_document_returns_error(self, proxy: ProxyClient, resources: ResourceManager) -> None:
|
||||
model = f"rust-ocr-val-{unique_marker()}"
|
||||
model_id = proxy.create_model(model, MistralOcr().litellm_params())
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
|
|
@ -174,4 +198,3 @@ class TestRustOcrGateway:
|
|||
json=_OptionalOcrBody(model=model),
|
||||
)
|
||||
assert_client_error(result, "ocr missing document")
|
||||
|
||||
|
|
|
|||
|
|
@ -17,3 +17,4 @@ markers =
|
|||
provider_edge_host: routes provider traffic through the pytest host's edge in every fixture mode, so the gateway must reach the pytest host; deselected unless E2E_PROVIDER_EDGE_HOST_REACHABLE is set
|
||||
otel_v2: needs a proxy running with LITELLM_OTEL_V2=true; deselected unless E2E_OTEL_V2 is set
|
||||
otel_tls: needs a stack whose gateway exports OTLP over TLS signed by the CA in SSL_CERT_FILE; deselected unless E2E_OTEL_EXPORTER_ENDPOINT is set
|
||||
secret_manager: needs a proxy booted from gateway/secret_manager_<system>_ci_config.yml against that live secret manager; deselected unless E2E_SECRET_MANAGER names the backend (see secret_manager/secret_backends.py)
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ reliability behavior.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import ValidationError
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from proxy_client import ProxyClient
|
||||
from e2e_config import CHEAP_OPENAI_MODEL, PROXY_BASE_URL, unique_marker
|
||||
|
|
@ -367,6 +368,26 @@ def model_id_of(resp: StreamingResponse) -> str | None:
|
|||
return resp.headers.get("x-litellm-model-id")
|
||||
|
||||
|
||||
class _AzurePromptFilterResult(BaseModel):
|
||||
content_filter_results: Mapping[str, object] | None = None
|
||||
|
||||
|
||||
class _AzureAnnotatedChatBody(BaseModel):
|
||||
prompt_filter_results: Sequence[_AzurePromptFilterResult] | None = None
|
||||
|
||||
|
||||
def azure_prompt_filter_skipped(resp: StreamingResponse) -> bool:
|
||||
"""True when Azure's 200 recorded no prompt-filter verdict (every `content_filter_results`
|
||||
empty), so the prompt has to be sent again."""
|
||||
try:
|
||||
annotated: Final = _AzureAnnotatedChatBody.model_validate_json(resp.body)
|
||||
except ValidationError:
|
||||
return False
|
||||
if not annotated.prompt_filter_results:
|
||||
return False
|
||||
return all(not entry.content_filter_results for entry in annotated.prompt_filter_results)
|
||||
|
||||
|
||||
def _parsed(resp: StreamingResponse) -> ChatResponse | None:
|
||||
try:
|
||||
return ChatResponse.model_validate_json(resp.body)
|
||||
|
|
|
|||
|
|
@ -16,10 +16,16 @@ failure: the provider refuses the prompt itself, on length or on policy, and
|
|||
reroute those, not `fallbacks`. The policy refusal is a real one, from an Azure
|
||||
OpenAI content filter rejecting a jailbreak prompt, and a control call first
|
||||
proves the refusal reaches the customer as a 400 when no reroute is configured.
|
||||
Azure intermittently answers without running its prompt filter at all (the
|
||||
body's prompt_filter_results carry no verdict), which is not a pass, so both
|
||||
calls send the prompt again, a bounded number of times, until the filter ran.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from complexity_router_client import ComplexityRouterClient
|
||||
|
|
@ -29,6 +35,7 @@ from lifecycle import ResourceManager
|
|||
from models import RouterSettingsOverride
|
||||
from reliability_support import (
|
||||
CONTENT_POLICY_PROMPT,
|
||||
azure_prompt_filter_skipped,
|
||||
chat_override,
|
||||
completion_tokens_of,
|
||||
content_of,
|
||||
|
|
@ -64,6 +71,28 @@ def _assert_served_by_fallback(resp: StreamingResponse) -> None:
|
|||
assert int(attempted) >= 1, f"x-litellm-attempted-fallbacks should be >= 1, got {attempted!r}"
|
||||
|
||||
|
||||
AZURE_FILTER_ATTEMPTS: Final = 3
|
||||
|
||||
|
||||
def _chat_once_azure_runs_its_filter(send: Callable[[], StreamingResponse]) -> StreamingResponse:
|
||||
for attempt in range(1, AZURE_FILTER_ATTEMPTS):
|
||||
resp = send()
|
||||
if not azure_prompt_filter_skipped(resp):
|
||||
return resp
|
||||
print(
|
||||
"e2e: azure answered without running its prompt filter; sending the jailbreak prompt again "
|
||||
f"({attempt}/{AZURE_FILTER_ATTEMPTS - 1})",
|
||||
flush=True,
|
||||
)
|
||||
return send()
|
||||
|
||||
|
||||
def _filter_verdict(resp: StreamingResponse) -> str:
|
||||
if azure_prompt_filter_skipped(resp):
|
||||
return "azure skipped its prompt filter on every attempt"
|
||||
return "the filter ran and let the prompt through"
|
||||
|
||||
|
||||
class TestReliabilityFallbacks:
|
||||
@pytest.mark.covers("reliability.fallback.5xx.routes_to_fallback")
|
||||
def test_5xx_routes_to_fallback(
|
||||
|
|
@ -124,17 +153,21 @@ class TestReliabilityFallbacks:
|
|||
model_id = create_content_filtered_deployment(client.proxy, primary)
|
||||
resources.defer(lambda: client.proxy.delete_model(model_id))
|
||||
|
||||
refused = chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
|
||||
refused = _chat_once_azure_runs_its_filter(
|
||||
lambda: chat_override(client.proxy, scoped_key, primary, f"{CONTENT_POLICY_PROMPT} {unique_marker()}")
|
||||
)
|
||||
assert refused.status_code == 400, (
|
||||
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code}: "
|
||||
f"{refused.body[:300]}"
|
||||
f"the content filter should have refused the jailbreak prompt with a 400, got {refused.status_code} "
|
||||
f"({_filter_verdict(refused)}): {refused.body[:300]}"
|
||||
)
|
||||
|
||||
resp = chat_override(
|
||||
client.proxy,
|
||||
scoped_key,
|
||||
primary,
|
||||
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
|
||||
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
|
||||
resp = _chat_once_azure_runs_its_filter(
|
||||
lambda: chat_override(
|
||||
client.proxy,
|
||||
scoped_key,
|
||||
primary,
|
||||
f"{CONTENT_POLICY_PROMPT} {unique_marker()}",
|
||||
override=RouterSettingsOverride(content_policy_fallbacks=[{primary: ["gpt-5.5"]}]),
|
||||
)
|
||||
)
|
||||
_assert_served_by_fallback(resp)
|
||||
|
|
|
|||
76
tests/e2e/secret_manager/backend.sh
Executable file
76
tests/e2e/secret_manager/backend.sh
Executable file
|
|
@ -0,0 +1,76 @@
|
|||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
|
||||
usage() {
|
||||
local systems
|
||||
systems=$(declare -F | sed -n 's/^declare -f up_//p' | paste -sd '|' -)
|
||||
echo "usage: $0 up|down $systems" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
action=${1:-}
|
||||
system=${2:-}
|
||||
dir=${E2E_SECRET_MANAGER_DIR:-$HOME/.cache/litellm-e2e-secret-manager}/$system
|
||||
name=litellm-e2e-$system
|
||||
|
||||
wait_for() {
|
||||
local url=$1
|
||||
for _ in $(seq 1 90); do
|
||||
if curl -sf -o /dev/null "$url"; then
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
echo "$system did not answer at $url" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
down() {
|
||||
docker rm -f "$name" "$name-db" >/dev/null 2>&1 || true
|
||||
docker network rm "$name" >/dev/null 2>&1 || true
|
||||
rm -rf "$dir"
|
||||
}
|
||||
|
||||
up_hashicorp_vault() {
|
||||
local port=${E2E_SECRET_MANAGER_PORT:-8200}
|
||||
local token
|
||||
token=e2e-$(openssl rand -hex 16)
|
||||
docker run -d --name "$name" -p "127.0.0.1:$port:8200" --cap-add IPC_LOCK \
|
||||
-e VAULT_DEV_ROOT_TOKEN_ID="$token" hashicorp/vault:1.20 >/dev/null
|
||||
wait_for "http://127.0.0.1:$port/v1/sys/health"
|
||||
printf 'HCP_VAULT_ADDR=http://127.0.0.1:%s\nHCP_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/proxy.env"
|
||||
printf 'E2E_VAULT_ADDR=http://127.0.0.1:%s\nE2E_VAULT_TOKEN=%s\n' "$port" "$token" >"$dir/tests.env"
|
||||
}
|
||||
|
||||
up_cyberark() {
|
||||
local port=${E2E_SECRET_MANAGER_PORT:-8080}
|
||||
local data_key api_key
|
||||
docker network create "$name" >/dev/null
|
||||
docker run -d --name "$name-db" --network "$name" -e POSTGRES_HOST_AUTH_METHOD=trust postgres:15 >/dev/null
|
||||
data_key=$(docker run --rm cyberark/conjur:1.24 data-key generate)
|
||||
docker run -d --name "$name" --network "$name" -p "127.0.0.1:$port:80" \
|
||||
-e DATABASE_URL="postgres://postgres@$name-db/postgres" -e CONJUR_DATA_KEY="$data_key" \
|
||||
-e CONJUR_AUTHENTICATORS=authn cyberark/conjur:1.24 server >/dev/null
|
||||
wait_for "http://127.0.0.1:$port/"
|
||||
docker exec "$name" conjurctl account create --name default >/dev/null
|
||||
api_key=$(docker exec "$name" conjurctl role retrieve-key default:user:admin | tr -d '\r\n')
|
||||
printf 'CYBERARK_API_BASE=http://127.0.0.1:%s\nCYBERARK_ACCOUNT=default\nCYBERARK_USERNAME=admin\nCYBERARK_API_KEY=%s\n' \
|
||||
"$port" "$api_key" >"$dir/proxy.env"
|
||||
printf 'E2E_CYBERARK_API_BASE=http://127.0.0.1:%s\nE2E_CYBERARK_ACCOUNT=default\nE2E_CYBERARK_USERNAME=admin\nE2E_CYBERARK_API_KEY=%s\n' \
|
||||
"$port" "$api_key" >"$dir/tests.env"
|
||||
}
|
||||
|
||||
[[ $# -eq 2 && -n $system ]] && declare -F "up_$system" >/dev/null || usage
|
||||
|
||||
case $action in
|
||||
up)
|
||||
down
|
||||
mkdir -p "$dir"
|
||||
"up_$system"
|
||||
echo "E2E_SECRET_MANAGER=$system" >>"$dir/tests.env"
|
||||
echo "$system is up; env in $dir/proxy.env (proxy) and $dir/tests.env (pytest)"
|
||||
;;
|
||||
down) down ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
57
tests/e2e/secret_manager/conftest.py
Normal file
57
tests/e2e/secret_manager/conftest.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import SECRET_MANAGER_OPT_IN_ENV
|
||||
from proxy_client import ProxyClient
|
||||
from secret_backends import BACKENDS, selected_backend
|
||||
from secret_store import SecretBackend, SecretStore
|
||||
|
||||
REQUIRES_CAPABILITY: Final = "requires_capability"
|
||||
|
||||
|
||||
def pytest_configure(config: pytest.Config) -> None:
|
||||
config.addinivalue_line(
|
||||
"markers",
|
||||
f"{REQUIRES_CAPABILITY}(capability): secret_manager test deselected when the backend "
|
||||
f"{SECRET_MANAGER_OPT_IN_ENV} names lacks the capability (secret_store.Capability)",
|
||||
)
|
||||
|
||||
|
||||
def _lacks_capability(item: pytest.Item, backend: SecretBackend) -> bool:
|
||||
marker: Final = item.get_closest_marker(REQUIRES_CAPABILITY)
|
||||
return marker is not None and marker.args[0] not in backend.capabilities
|
||||
|
||||
|
||||
def pytest_collection_modifyitems(config: pytest.Config, items: list[pytest.Item]) -> None:
|
||||
backend: Final = BACKENDS.get(os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip())
|
||||
if backend is None:
|
||||
return
|
||||
deselected: Final = [item for item in items if _lacks_capability(item, backend)]
|
||||
if deselected:
|
||||
config.hook.pytest_deselected(items=deselected)
|
||||
items[:] = [item for item in items if not _lacks_capability(item, backend)]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretManagerClient:
|
||||
proxy: ProxyClient
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def client(proxy: ProxyClient) -> SecretManagerClient:
|
||||
return SecretManagerClient(proxy)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def backend() -> SecretBackend:
|
||||
return selected_backend()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def store(backend: SecretBackend) -> SecretStore:
|
||||
return backend.from_env()
|
||||
25
tests/e2e/secret_manager/secret_backends.py
Normal file
25
tests/e2e/secret_manager/secret_backends.py
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from e2e_config import SECRET_MANAGER_OPT_IN_ENV
|
||||
from secret_store import SecretBackend
|
||||
from secret_store_cyberark import CYBERARK
|
||||
from secret_store_hashicorp_vault import HASHICORP_VAULT
|
||||
|
||||
BACKENDS: Final = MappingProxyType({backend.system: backend for backend in (HASHICORP_VAULT, CYBERARK)})
|
||||
|
||||
|
||||
def selected_backend() -> SecretBackend:
|
||||
system: Final = os.environ.get(SECRET_MANAGER_OPT_IN_ENV, "").strip()
|
||||
backend: Final = BACKENDS.get(system)
|
||||
if backend is None:
|
||||
pytest.fail(
|
||||
f"{SECRET_MANAGER_OPT_IN_ENV}={system!r} names no secret manager backend; "
|
||||
f"set it to one of {sorted(BACKENDS)}"
|
||||
)
|
||||
return backend
|
||||
29
tests/e2e/secret_manager/secret_store.py
Normal file
29
tests/e2e/secret_manager/secret_store.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Final, Literal, Protocol
|
||||
|
||||
SECRET_MANAGER_CONFIG_DIR: Final = "gateway"
|
||||
|
||||
|
||||
class SecretStore(Protocol):
|
||||
def write(self, name: str, value: str) -> None: ...
|
||||
|
||||
def read(self, name: str) -> str | None: ...
|
||||
|
||||
def destroy(self, name: str) -> None: ...
|
||||
|
||||
|
||||
Capability = Literal["deletes_stored_keys"]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class SecretBackend:
|
||||
system: str
|
||||
from_env: Callable[[], SecretStore]
|
||||
capabilities: frozenset[Capability]
|
||||
|
||||
@property
|
||||
def proxy_config(self) -> str:
|
||||
return f"{SECRET_MANAGER_CONFIG_DIR}/secret_manager_{self.system}_ci_config.yml"
|
||||
118
tests/e2e/secret_manager/secret_store_cyberark.py
Normal file
118
tests/e2e/secret_manager/secret_store_cyberark.py
Normal file
|
|
@ -0,0 +1,118 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final, Literal
|
||||
from urllib.parse import quote
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from e2e_http import ExternalWrite, Headers, send_text_external
|
||||
from pydantic import Field
|
||||
|
||||
from secret_store import SecretBackend
|
||||
|
||||
CYBERARK_API_BASE_ENV: Final = "E2E_CYBERARK_API_BASE"
|
||||
CYBERARK_ACCOUNT_ENV: Final = "E2E_CYBERARK_ACCOUNT"
|
||||
CYBERARK_USERNAME_ENV: Final = "E2E_CYBERARK_USERNAME"
|
||||
CYBERARK_API_KEY_ENV: Final = "E2E_CYBERARK_API_KEY"
|
||||
|
||||
# The same defaults CyberArkSecretManager falls back to for CYBERARK_*.
|
||||
DEFAULT_API_BASE: Final = "http://127.0.0.1:8080"
|
||||
DEFAULT_ACCOUNT: Final = "default"
|
||||
DEFAULT_USERNAME: Final = "admin"
|
||||
|
||||
SYSTEM: Final = "cyberark"
|
||||
|
||||
_START_HINT: Final = (
|
||||
f"Start one with `bash tests/e2e/secret_manager/backend.sh up {SYSTEM}`, which writes the env for "
|
||||
f"the proxy (booted from gateway/secret_manager_{SYSTEM}_ci_config.yml) and for the tests"
|
||||
)
|
||||
|
||||
|
||||
class ConjurHeaders(Headers):
|
||||
authorization: str = Field(repr=False)
|
||||
content_type: str | None = Field(default=None, serialization_alias="Content-Type")
|
||||
|
||||
|
||||
def _policy_scalar(name: str) -> str:
|
||||
# Quoted the way CyberArkSecretManager._ensure_variable_exists quotes it.
|
||||
return yaml.safe_dump(name, default_style='"').strip()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Conjur:
|
||||
base_url: str
|
||||
account: str
|
||||
username: str
|
||||
api_key: str = field(repr=False)
|
||||
|
||||
def _fail_unless_reached(self, result: ExternalWrite, action: str) -> None:
|
||||
if result.status_code == -1:
|
||||
pytest.fail(f"No live Conjur at {self.base_url}: {result.body}. {_START_HINT}")
|
||||
if result.status_code == 401:
|
||||
pytest.fail(f"Conjur rejected {self.username}'s credentials while trying to {action}. {_START_HINT}")
|
||||
|
||||
def _headers(self, content_type: str | None = None) -> ConjurHeaders:
|
||||
# Tokens last about eight minutes, so each call authenticates afresh rather than
|
||||
# letting a long session outlive a cached one.
|
||||
auth: Final = send_text_external(
|
||||
"POST",
|
||||
f"{self.base_url}/authn/{self.account}/{quote(self.username, safe='')}/authenticate",
|
||||
headers=Headers(),
|
||||
content=self.api_key,
|
||||
)
|
||||
self._fail_unless_reached(auth, "authenticate")
|
||||
if not auth.ok:
|
||||
pytest.fail(f"Conjur refused to authenticate {self.username}: HTTP {auth.status_code} {auth.body[:300]}")
|
||||
token: Final = base64.b64encode(auth.body.encode()).decode()
|
||||
return ConjurHeaders(authorization=f'Token token="{token}"', content_type=content_type)
|
||||
|
||||
def _secret_url(self, name: str) -> str:
|
||||
return f"{self.base_url}/secrets/{self.account}/variable/{quote(name, safe='')}"
|
||||
|
||||
def _update_root_policy(self, method: Literal["POST", "PATCH"], policy: str, action: str) -> None:
|
||||
result: Final = send_text_external(
|
||||
method,
|
||||
f"{self.base_url}/policies/{self.account}/policy/root",
|
||||
headers=self._headers(content_type="application/x-yaml"),
|
||||
content=policy,
|
||||
)
|
||||
self._fail_unless_reached(result, action)
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to {action}: HTTP {result.status_code} {result.body[:300]}")
|
||||
|
||||
def write(self, name: str, value: str) -> None:
|
||||
self._update_root_policy("POST", f"- !variable {_policy_scalar(name)}\n", f"declare {name}")
|
||||
result: Final = send_text_external("POST", self._secret_url(name), headers=self._headers(), content=value)
|
||||
self._fail_unless_reached(result, f"write {name}")
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to write {name}: HTTP {result.status_code} {result.body[:300]}")
|
||||
|
||||
def read(self, name: str) -> str | None:
|
||||
result: Final = send_text_external("GET", self._secret_url(name), headers=self._headers())
|
||||
self._fail_unless_reached(result, f"read {name}")
|
||||
if result.status_code == 404:
|
||||
return None
|
||||
if not result.ok:
|
||||
pytest.fail(f"Conjur refused to read {name}: HTTP {result.status_code} {result.body[:300]}")
|
||||
return result.body
|
||||
|
||||
def destroy(self, name: str) -> None:
|
||||
self._update_root_policy("PATCH", f"- !delete\n record: !variable {_policy_scalar(name)}\n", f"destroy {name}")
|
||||
|
||||
|
||||
def conjur_from_env() -> Conjur:
|
||||
api_key: Final = os.environ.get(CYBERARK_API_KEY_ENV, "").strip()
|
||||
if not api_key:
|
||||
pytest.fail(f"The {SYSTEM} lane needs {CYBERARK_API_KEY_ENV} to reach its Conjur. {_START_HINT}")
|
||||
return Conjur(
|
||||
base_url=os.environ.get(CYBERARK_API_BASE_ENV, "").strip().rstrip("/") or DEFAULT_API_BASE,
|
||||
account=os.environ.get(CYBERARK_ACCOUNT_ENV, "").strip() or DEFAULT_ACCOUNT,
|
||||
username=os.environ.get(CYBERARK_USERNAME_ENV, "").strip() or DEFAULT_USERNAME,
|
||||
api_key=api_key,
|
||||
)
|
||||
|
||||
|
||||
CYBERARK: Final = SecretBackend(system=SYSTEM, from_env=conjur_from_env, capabilities=frozenset())
|
||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue