diff --git a/.github/workflows/auto-close-duplicates.yml b/.github/workflows/auto-close-duplicates.yml
index d8256917805..9a362b4c85c 100644
--- a/.github/workflows/auto-close-duplicates.yml
+++ b/.github/workflows/auto-close-duplicates.yml
@@ -22,7 +22,7 @@ on:
permissions: {}
jobs:
- test:
+ sweep-tests:
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
timeout-minutes: 5
diff --git a/.github/workflows/cost-map-guard.yml b/.github/workflows/cost-map-guard.yml
new file mode 100644
index 00000000000..61a56f1f47e
--- /dev/null
+++ b/.github/workflows/cost-map-guard.yml
@@ -0,0 +1,45 @@
+name: Cost map guard
+
+on: # zizmor: ignore[dangerous-triggers] runs the base branch's code only; the PR's cost map files are read as data and never executed
+ pull_request_target:
+ branches:
+ - main
+ - litellm_internal_staging
+ - litellm_oss_staging
+ - "litellm_**"
+
+permissions:
+ contents: read
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+ cancel-in-progress: true
+
+jobs:
+ cost-map-guard:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ with:
+ persist-credentials: false
+ - name: Fetch the pull request head and its merge base
+ id: revisions
+ env:
+ GH_TOKEN: ${{ github.token }}
+ BASE_SHA: ${{ github.event.pull_request.base.sha }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ run: |
+ merge_base="$(gh api "repos/${GITHUB_REPOSITORY}/compare/${BASE_SHA}...${HEAD_SHA}" --jq '.merge_base_commit.sha')"
+ git fetch --no-tags --depth=1 origin "$merge_base" "$HEAD_SHA"
+ echo "merge_base=$merge_base" >> "$GITHUB_OUTPUT"
+ - name: Set up uv
+ uses: ./.github/actions/setup-uv-with-retries
+ with:
+ version: "0.10.9"
+ - name: Run the guard
+ env:
+ MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
+ HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+ HEAD_REF: ${{ github.event.pull_request.head.ref }}
+ run: |
+ uv run --frozen python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"
diff --git a/.github/workflows/report-rust-release-wheel.yml b/.github/workflows/report-rust-release-wheel.yml
deleted file mode 100644
index 74e6be69604..00000000000
--- a/.github/workflows/report-rust-release-wheel.yml
+++ /dev/null
@@ -1,129 +0,0 @@
-name: Report LiteLLM Rust release wheel
-
-on: # zizmor: ignore[dangerous-triggers] reporter executes no PR code and consumes no PR artifacts or outputs
- workflow_run:
- workflows:
- - LiteLLM Rust
- types:
- - completed
-
-permissions: {}
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.workflow_run.pull_requests[0].number || github.event.workflow_run.id }}
- cancel-in-progress: false
-
-jobs:
- report-release-wheel:
- name: report release wheel
- if: >-
- github.event.workflow_run.event == 'pull_request' &&
- github.event.workflow_run.path == '.github/workflows/test-rust.yml' &&
- github.event.workflow_run.head_repository.full_name == github.repository &&
- github.event.workflow_run.pull_requests[0].number != null
- runs-on: ubuntu-latest
- timeout-minutes: 5
- permissions:
- pull-requests: write
-
- steps:
- - name: Link release wheel report on PR
- uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
- env:
- COMMENT_MARKER: ""
- with:
- script: |
- const marker = process.env.COMMENT_MARKER;
- const workflowRun = context.payload.workflow_run;
- const allowedConclusions = new Set([
- "action_required",
- "cancelled",
- "failure",
- "neutral",
- "skipped",
- "stale",
- "startup_failure",
- "success",
- "timed_out",
- ]);
- if (
- !allowedConclusions.has(workflowRun.conclusion) ||
- workflowRun.event !== "pull_request" ||
- workflowRun.path !== ".github/workflows/test-rust.yml" ||
- workflowRun.head_repository?.full_name !==
- `${context.repo.owner}/${context.repo.repo}` ||
- workflowRun.pull_requests?.length !== 1
- ) {
- throw new Error("unexpected source workflow");
- }
- const pullRequest = workflowRun.pull_requests[0];
- const pullRequestNumber = pullRequest.number;
- const headSha = workflowRun.head_sha;
- const runId = workflowRun.id;
- if (
- !Number.isSafeInteger(pullRequestNumber) ||
- pullRequestNumber <= 0 ||
- !Number.isSafeInteger(runId) ||
- runId <= 0 ||
- !/^[0-9a-f]{40}$/.test(headSha) ||
- pullRequest.head?.sha !== headSha
- ) {
- throw new Error("invalid source workflow metadata");
- }
- const runUrl =
- `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}` +
- `/actions/runs/${runId}`;
- const result =
- workflowRun.conclusion === "success"
- ? "successfully"
- : `with \`${workflowRun.conclusion}\``;
- const body = [
- marker,
- "## LiteLLM Rust workflow",
- "",
- `Workflow completed ${result} for \`${headSha}\``,
- "",
- `[View workflow run](${runUrl})`,
- ].join("\n");
- const comments = await github.paginate(github.rest.issues.listComments, {
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: pullRequestNumber,
- per_page: 100,
- });
- const existing = comments.find(
- (comment) =>
- comment.user?.login === "github-actions[bot]" &&
- comment.body?.startsWith(marker),
- );
- const currentPullRequest = (
- await github.rest.pulls.get({
- owner: context.repo.owner,
- repo: context.repo.repo,
- pull_number: pullRequestNumber,
- })
- ).data;
- if (
- currentPullRequest.state !== "open" ||
- currentPullRequest.head.repo?.full_name !==
- `${context.repo.owner}/${context.repo.repo}` ||
- currentPullRequest.head.sha !== headSha
- ) {
- core.info("source workflow no longer matches the current pull request head");
- return;
- }
- if (existing) {
- await github.rest.issues.updateComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- comment_id: existing.id,
- body,
- });
- } else {
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: pullRequestNumber,
- body,
- });
- }
diff --git a/.github/workflows/test-code-quality.yml b/.github/workflows/test-code-quality.yml
index 36fd656c231..e6d2264fbf0 100644
--- a/.github/workflows/test-code-quality.yml
+++ b/.github/workflows/test-code-quality.yml
@@ -74,6 +74,12 @@ jobs:
- name: check_workflow_startup_safety
run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_startup_safety.py
+ - name: check_workflow_job_name_collisions
+ run: uv run --no-sync python ./tests/code_coverage_tests/check_workflow_job_name_collisions.py
+
+ - name: test_workflow_job_name_collisions
+ run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_workflow_job_name_collisions.py
+
- name: test_e2e_changed_gate
run: uv run --no-sync pytest -q --noconftest -p no:cacheprovider -c /dev/null tests/code_coverage_tests/test_e2e_changed_gate.py
diff --git a/.github/workflows/test-model-map.yml b/.github/workflows/test-model-map.yml
deleted file mode 100644
index c2770e5da4c..00000000000
--- a/.github/workflows/test-model-map.yml
+++ /dev/null
@@ -1,37 +0,0 @@
-name: Validate model_prices_and_context_window.json
-
-on:
- pull_request:
- branches:
- - main
- - litellm_internal_staging
- - litellm_oss_staging
- - "litellm_**"
-
-permissions:
- contents: read
-
-concurrency:
- group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}
- cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-jobs:
- validate-model-prices-json:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
- with:
- persist-credentials: false
-
- - name: Validate model_prices_and_context_window.json
- run: |
- jq empty model_prices_and_context_window.json
-
- - name: Set up uv
- uses: ./.github/actions/setup-uv-with-retries
- with:
- version: "0.10.9"
-
- - name: Check model_prices_and_context_window.schema.json is in sync
- run: |
- uv run --frozen python ci_cd/generate_model_prices_schema.py --check
diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml
index 9b8b132df62..4f56e78ddee 100644
--- a/.github/workflows/test-rust.yml
+++ b/.github/workflows/test-rust.yml
@@ -7,6 +7,7 @@ on:
- ".cargo/**"
- "pyproject.toml"
- "rust-toolchain.toml"
+ - ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
- "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
@@ -22,6 +23,7 @@ on:
- ".cargo/**"
- "pyproject.toml"
- "rust-toolchain.toml"
+ - ".github/actions/setup-uv-with-retries/**"
- ".github/scripts/smoke_test_native_wheel.py"
- ".github/scripts/verify_linux_native_wheel.py"
- "tests/test_litellm/rust_bridge/native_route_wheel_test.py"
@@ -34,102 +36,89 @@ concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
+env:
+ CARGO_TERM_COLOR: always
+
jobs:
- rust-checks:
- name: rustfmt, clippy, test
+ rust-lint:
runs-on: ubuntu-latest
timeout-minutes: 10
defaults:
run:
working-directory: litellm-rust
- env:
- CARGO_TERM_COLOR: always
-
steps:
- - name: Checkout repository
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- - name: Set up Rust
- run: rustup toolchain install
+ - run: rustup toolchain install --no-self-update
- - name: Cache Cargo registry and target
- uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ - run: cargo fmt --check
+
+ - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cargo/registry
~/.cargo/git
litellm-rust/target
- key: ${{ runner.os }}-cargo-${{ hashFiles('rust-toolchain.toml', 'litellm-rust/Cargo.lock') }}
+ key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
restore-keys: |
- ${{ runner.os }}-cargo-
+ ${{ runner.os }}-cargo-${{ github.job }}-
- - name: Check Rust formatting
- run: cargo fmt --check
+ - run: cargo clippy --workspace --all-targets --locked -- -D warnings
- - name: Run Clippy
- run: cargo clippy --workspace --all-targets --locked -- -D warnings
+ - run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
- - name: Run Clippy with Bedrock auth
- run: cargo clippy -p litellm-core --all-targets --features bedrock-auth --locked -- -D warnings
+ - run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
- - name: Run Clippy with all gateway features
- run: cargo clippy -p litellm-ai-gateway --all-targets --all-features --locked -- -D warnings
-
- - name: Run Rust tests
- run: cargo test --workspace --locked
-
- - name: Run core tests with Bedrock auth
- run: cargo test -p litellm-core --features bedrock-auth --locked
-
- # Not --all-features: python-config links libpython, which this job does not install.
- - name: Run gateway tests with the server feature
- run: cargo test -p litellm-ai-gateway --features server --locked
-
- release-wheel:
- name: release wheel
+ rust-test:
runs-on: ubuntu-latest
- timeout-minutes: 20
- permissions:
- contents: read
- env:
- CARGO_TERM_COLOR: always
+ timeout-minutes: 30
steps:
- - name: Checkout repository
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+ - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- - name: Set up Python
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- - name: Set up uv
- uses: ./.github/actions/setup-uv-with-retries
+ - uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- - name: Set up Rust
- run: rustup toolchain install
+ - run: rustup toolchain install --no-self-update
- - name: Build release wheel
- run: uv build --wheel --out-dir dist
+ - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
+ with:
+ path: |
+ ~/.cargo/registry
+ ~/.cargo/git
+ litellm-rust/target
+ key: ${{ runner.os }}-cargo-${{ github.job }}-${{ hashFiles('rust-toolchain.toml', '.cargo/**', 'litellm-rust/Cargo.lock') }}
+ restore-keys: |
+ ${{ runner.os }}-cargo-${{ github.job }}-
- - name: Build panic contract wheel
- run: >-
+ - run: cargo test --workspace --locked
+ working-directory: litellm-rust
+
+ - run: cargo test -p litellm-core --features bedrock-auth --locked
+ working-directory: litellm-rust
+
+ - run: cargo test -p litellm-ai-gateway --features server --locked
+ working-directory: litellm-rust
+
+ - run: uv build --wheel --out-dir dist
+
+ - run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
+ env:
+ RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+
+ - run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+
+ - run: >-
uv build --wheel --out-dir panic-dist
--config-setting "maturin.build-args=--features panic-test,extension-module"
- - name: Smoke-test native panic unwinding
- run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl
-
- - name: Verify stripped native extension
- env:
- RELEASE_WHEEL_COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
- run: python .github/scripts/verify_linux_native_wheel.py dist/*.whl
-
- - name: Test native route wheel
- run: python tests/test_litellm/rust_bridge/native_route_wheel_test.py dist/*.whl
+ - run: python .github/scripts/smoke_test_native_wheel.py panic-dist/*.whl
diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml
index 33245ec5b5f..cc606339a20 100644
--- a/.github/workflows/test-unit.yml
+++ b/.github/workflows/test-unit.yml
@@ -116,6 +116,7 @@ jobs:
tests/test_litellm/rerank_api
tests/test_litellm/rust_bridge
tests/test_litellm/sandbox
+ tests/test_litellm/skills
tests/test_litellm/test_router
tests/test_litellm/vector_stores
tests/test_litellm/videos
diff --git a/CLAUDE.md b/CLAUDE.md
index d9e9e8f1586..2bc39332817 100644
--- a/CLAUDE.md
+++ b/CLAUDE.md
@@ -37,7 +37,7 @@ Same applies for filing bug reports and feature requests, with .github/ISSUE_TEM
If you're resolving a linear ticket, in the "## Linear ticket" section of the PR, say "Resolves LIT-1234", replacing "LIT-1234" with the actual ticket id that you're resolving. If you don't have the ticket id, don't make one up or search for it. Just leave the section blank
-Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, just tell me which URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, what fields to fill out, etc. along with the other commands to run in an ordered list, and I'll do it myself and post the screenshots after you make the PR
+Never use `pytest` commands or the like as "Screenshots / Proof of Fix". We prefer curl'ing a live proxy instance running on localhost:4000 (I like to run it with `python litellm/proxy/proxy_cli.py --config litellm/proxy/dev_config.yaml --detailed_debug --reload --use_v2_migration_resolver 2>&1 | tee litellm.log`; the Admin UI dev server is `npm run dev` in `ui/litellm-dashboard`, served on port 3000) and showing both the command run and the output. Also, it should hit real LLM provider APIs, not mocks, and cost real $$$ because that is the most realistic test. The proof of fix should be exactly what the end user / customer would see / do. The run logs in PR #27703 is a prime example of how to do it (not a huge fan of using a python test script that future me and the team will have no visibility into; I prefer just curl commands or a short list of bash commands (e.g., using `for`)). If it's a UI thing, or the main use case runs through a headful agentic coding tool like Claude Code or Codex, drive that surface yourself and embed your own before and after screenshots of it in the PR (the Admin UI page, or what the coding tool shows), next to an ordered list of the URLs to go to (e.g., http://localhost:4000/ui/?page=logs), where to click, and what fields to fill out so a reviewer can reproduce it
If you ever write any human-facing text (pull requests, issues, commit messages, discussion posts, github comments, release notes, docs, etc.), always follow these guidelines to sound less AI-y:
- don't use emojis
@@ -52,7 +52,7 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a
Python max line length is 120, not 88
-When you fix violations gated by `ruff-strict-budget.json`, `type-discipline-budget.json`, or `basedpyright-code-budget.json`, run `make lint-budget-update` and commit the lowered limits so the ceilings ratchet down instead of leaving stale headroom. It measures the working tree, so it must contain exactly the fixes you're committing
+Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on `litellm_internal_staging` in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. If your branch already carries a budget edit, drop it before opening the PR
`make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice
diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json
index 0b0a61192e6..57ca267e504 100644
--- a/basedpyright-code-budget.json
+++ b/basedpyright-code-budget.json
@@ -84,7 +84,7 @@
"limit": 56
},
"reportPrivateUsage": {
- "limit": 1808
+ "limit": 1804
},
"reportRedeclaration": {
"limit": 8
@@ -135,7 +135,7 @@
"limit": 21
},
"reportUnusedFunction": {
- "limit": 138
+ "limit": 136
},
"reportUnusedImport": {
"limit": 542
diff --git a/ci_cd/cost_map_guard.py b/ci_cd/cost_map_guard.py
new file mode 100644
index 00000000000..50aa40ba220
--- /dev/null
+++ b/ci_cd/cost_map_guard.py
@@ -0,0 +1,146 @@
+"""Guard the cost map on pull requests.
+
+Every pull request gets the file checks: the three cost map files parse, the backup copy matches the root file,
+and the JSON schema is in sync and validates the map. Pull requests from the cost map sync bot (branches named
+litellm_cost_map_sync_*) additionally may only touch those three files and may only add or update models.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+from collections.abc import Sequence
+from dataclasses import dataclass
+from typing import Final
+
+from generate_model_prices_schema import SPECIAL_ROOT_KEYS, build_schema, render, validation_errors
+
+COST_MAP_PATH: Final = "model_prices_and_context_window.json"
+BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
+SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
+GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH, SCHEMA_PATH)
+BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
+
+CostMap = dict[str, object]
+
+
+@dataclass(frozen=True, slots=True)
+class Snapshot:
+ cost_map: str
+ backup: str
+ schema: str
+
+
+def _parse_object(text: str, path: str) -> CostMap | str:
+ try:
+ parsed: Final = json.loads(text)
+ except json.JSONDecodeError as error:
+ return f"{path} is not valid JSON: {error}"
+ return parsed if isinstance(parsed, dict) else f"{path} must be a JSON object at the root"
+
+
+def _rendered_schema(cost_map: CostMap) -> str:
+ try:
+ return render(build_schema(cost_map))
+ except SystemExit as error:
+ return str(error)
+
+
+def _file_failures(head: Snapshot, head_map: CostMap) -> tuple[str, ...]:
+ schema_text: Final = _rendered_schema(head_map)
+ if not schema_text.startswith("{"):
+ return (schema_text,)
+ backup_failure: Final = (
+ ()
+ if head.backup == head.cost_map
+ else (f"{BACKUP_PATH} differs from {COST_MAP_PATH}; copy the root file over it",)
+ )
+ schema_failure: Final = (
+ ()
+ if head.schema == schema_text
+ else (
+ f"{SCHEMA_PATH} is out of sync with {COST_MAP_PATH}; "
+ "run `python ci_cd/generate_model_prices_schema.py` and commit the result",
+ )
+ )
+ return (
+ *backup_failure,
+ *schema_failure,
+ *(
+ f"{COST_MAP_PATH} does not validate against its schema: {error}"
+ for error in validation_errors(head_map, json.loads(schema_text))[:20]
+ ),
+ )
+
+
+def _entries(cost_map: CostMap) -> dict[str, dict[str, object]]:
+ return {key: entry for key, entry in cost_map.items() if isinstance(entry, dict)}
+
+
+def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str]) -> tuple[str, ...]:
+ base_map: Final = _parse_object(base.cost_map, COST_MAP_PATH)
+ if isinstance(base_map, str):
+ return (f"merge base: {base_map}",)
+ base_entries: Final = _entries(base_map)
+ head_entries: Final = _entries(head_map)
+ removed_fields: Final = tuple(
+ f"{key}.{field}"
+ for key, entry in base_entries.items()
+ if key in head_entries
+ for field in entry
+ if field not in head_entries[key]
+ )
+ return (
+ *(
+ f"bot PRs may only change the cost map files, not {path}"
+ for path in changed_files
+ if path not in GUARDED_PATHS
+ ),
+ *(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
+ *(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
+ *(
+ f"bot PRs may not change {key}"
+ for key in sorted(SPECIAL_ROOT_KEYS)
+ if base_map.get(key) != head_map.get(key)
+ ),
+ )
+
+
+def guard_failures(base: Snapshot, head: Snapshot, changed_files: Sequence[str], bot: bool) -> tuple[str, ...]:
+ head_map: Final = _parse_object(head.cost_map, COST_MAP_PATH)
+ if isinstance(head_map, str):
+ return (head_map,)
+ return (*_file_failures(head, head_map), *(_bot_failures(base, head_map, changed_files) if bot else ()))
+
+
+def _git(*args: str) -> str:
+ result: Final = subprocess.run(("git", *args), check=False, capture_output=True, text=True)
+ return result.stdout if result.returncode == 0 else ""
+
+
+def snapshot(revision: str) -> Snapshot:
+ return Snapshot(*(_git("show", f"{revision}:{path}") for path in GUARDED_PATHS))
+
+
+def main(argv: Sequence[str]) -> int:
+ parser: Final = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--base", required=True, help="merge base of the pull request")
+ parser.add_argument("--head", required=True, help="head commit of the pull request")
+ parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
+ args: Final = parser.parse_args(argv)
+ bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
+ changed_files: Final = tuple(_git("diff", "--name-only", args.base, args.head).splitlines())
+ failures: Final = guard_failures(snapshot(args.base), snapshot(args.head), changed_files, bot)
+ contract: Final = "bot contract enforced" if bot else "human PR, file checks only"
+ if failures:
+ print(f"cost map guard failed ({contract}):")
+ print("\n".join(f"- {failure}" for failure in failures))
+ return 1
+ print(f"cost map guard passed ({contract})")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main(sys.argv[1:]))
diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock
index 62e943d0f42..43b9ec1aac2 100644
--- a/litellm-rust/Cargo.lock
+++ b/litellm-rust/Cargo.lock
@@ -1415,6 +1415,8 @@ dependencies = [
"litellm-config",
"litellm-core",
"reqwest",
+ "rustls 0.23.42",
+ "rustls-native-certs",
"serde",
"serde_json",
"sha2 0.10.9",
diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml
index 720c4545181..82de7f40069 100644
--- a/litellm-rust/Cargo.toml
+++ b/litellm-rust/Cargo.toml
@@ -28,6 +28,8 @@ pythonize = "0.29.0"
rand = "0.8"
reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "multipart", "rustls-tls", "http2", "stream"] }
rstest = "0.26.1"
+rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] }
+rustls-native-certs = "0.8"
serde = { version = "1.0", features = ["derive"] }
serde_json = { version = "1.0", features = ["float_roundtrip"] }
sha2 = "0.10"
diff --git a/litellm-rust/crates/ai-gateway/Cargo.toml b/litellm-rust/crates/ai-gateway/Cargo.toml
index 10369fa3bfd..74cf66e88a2 100644
--- a/litellm-rust/crates/ai-gateway/Cargo.toml
+++ b/litellm-rust/crates/ai-gateway/Cargo.toml
@@ -20,6 +20,10 @@ litellm-config.workspace = true
# reqwest (rustls + json) is used by io/ocr and ships realtime logs to the
# Python proxy callbacks API.
reqwest.workspace = true
+# rustls and its root store are direct dependencies so `io::tls` can build the
+# one TLS config the outbound dials use; see that module for why it has to.
+rustls.workspace = true
+rustls-native-certs.workspace = true
# `sync` powers the bounded mpsc channel the realtime logger drains.
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "net", "time", "sync"] }
tokio-tungstenite.workspace = true
diff --git a/litellm-rust/crates/ai-gateway/src/io/mod.rs b/litellm-rust/crates/ai-gateway/src/io/mod.rs
index cce56dd2121..7098d67993f 100644
--- a/litellm-rust/crates/ai-gateway/src/io/mod.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/mod.rs
@@ -3,3 +3,4 @@ pub mod ocr;
pub mod realtime;
pub mod realtime_pool;
pub mod responses_ws;
+pub(crate) mod tls;
diff --git a/litellm-rust/crates/ai-gateway/src/io/realtime.rs b/litellm-rust/crates/ai-gateway/src/io/realtime.rs
index 662f7328982..207c31dffa0 100644
--- a/litellm-rust/crates/ai-gateway/src/io/realtime.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/realtime.rs
@@ -23,10 +23,12 @@ use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::http::HeaderValue;
use tokio_tungstenite::tungstenite::http::header::AUTHORIZATION;
-use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async};
+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
use litellm_core::providers::openai::realtime::transformation::OPENAI_REALTIME_CONFIG;
+use crate::io::tls::connect_upstream;
+
/// Environment variable holding the OpenAI API key (last-resort fallback).
const OPENAI_API_KEY_ENV: &str = "OPENAI_API_KEY";
@@ -84,7 +86,7 @@ pub(crate) async fn dial_upstream(
.map_err(|err| Error::Auth(err.to_string()))?,
);
- let (upstream, _response) = connect_async(request)
+ let (upstream, _response) = connect_upstream(request)
.await
.map_err(|err| Error::Network(err.to_string()))?;
Ok(upstream)
@@ -284,6 +286,33 @@ mod tests {
serde_json::from_str(raw).expect("valid event json")
}
+ /// The realtime dial has to reach a `wss://` upstream without a process-wide
+ /// crypto provider installed, which is what dialing through `io::tls` buys.
+ #[tokio::test]
+ async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() {
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind a loopback port");
+ let port = listener
+ .local_addr()
+ .expect("read the bound address")
+ .port();
+ tokio::spawn(async move {
+ while let Ok((stream, _peer)) = listener.accept().await {
+ drop(stream);
+ }
+ });
+
+ let result = dial_upstream(
+ "gpt-realtime",
+ "sk-test",
+ Some(&format!("wss://127.0.0.1:{port}")),
+ )
+ .await;
+
+ assert!(matches!(result, Err(Error::Network(_))));
+ }
+
#[test]
fn resolve_api_key_prefers_param_then_blank_falls_through() {
assert_eq!(resolve_api_key(Some("sk-test")).unwrap(), "sk-test");
diff --git a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs
index 0b01747b1a5..9df3d0c6cc5 100644
--- a/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs
+++ b/litellm-rust/crates/ai-gateway/src/io/responses_ws.rs
@@ -14,7 +14,9 @@ use tokio_tungstenite::tungstenite::Message;
use tokio_tungstenite::tungstenite::client::IntoClientRequest;
use tokio_tungstenite::tungstenite::http::HeaderValue;
use tokio_tungstenite::tungstenite::http::header::{AUTHORIZATION, HeaderName};
-use tokio_tungstenite::{MaybeTlsStream, WebSocketStream, connect_async};
+use tokio_tungstenite::{MaybeTlsStream, WebSocketStream};
+
+use crate::io::tls::connect_upstream;
use crate::constants::{
DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS, DEFAULT_RESPONSES_WS_IDLE_TIMEOUT_SECS,
@@ -49,14 +51,14 @@ impl ResponsesWebSocketConnection {
.map_err(|error| Error::InvalidRequest(error.to_string()))?;
request.headers_mut().insert(header_name, header_value);
}
- let connect = connect_async(request);
+ let connect = connect_upstream(request);
let result = match timeout {
Some(timeout) => tokio::time::timeout(timeout, connect).await.map_err(|_| {
Error::Network("Responses WebSocket connection timed out".to_string())
})?,
None => connect.await,
};
- let (socket, _) = result.map_err(|error| match error {
+ let (socket, _) = result.map_err(|error| match *error {
tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http {
status: response.status().as_u16(),
body: String::new(),
@@ -138,13 +140,13 @@ async fn dial_upstream(
);
let result = tokio::time::timeout(
Duration::from_secs(DEFAULT_RESPONSES_WS_CONNECT_TIMEOUT_SECS),
- connect_async(request),
+ connect_upstream(request),
)
.await
.map_err(|_| Error::Network("Responses WebSocket connection timed out".to_string()))?;
result
.map(|(socket, _)| socket)
- .map_err(|error| match error {
+ .map_err(|error| match *error {
tokio_tungstenite::tungstenite::Error::Http(response) => Error::Http {
status: response.status().as_u16(),
body: String::new(),
@@ -324,6 +326,29 @@ mod tests {
use tokio::net::TcpListener;
use tokio_tungstenite::accept_async;
+ /// The Responses dial has to reach a `wss://` upstream without a process-wide
+ /// crypto provider installed, which is what dialing through `io::tls` buys.
+ #[tokio::test]
+ async fn dial_upstream_over_wss_reports_an_error_instead_of_panicking() {
+ let listener = TcpListener::bind("127.0.0.1:0")
+ .await
+ .expect("bind a loopback port");
+ let port = listener
+ .local_addr()
+ .expect("read the bound address")
+ .port();
+ tokio::spawn(async move {
+ while let Ok((stream, _peer)) = listener.accept().await {
+ drop(stream);
+ }
+ });
+
+ let result =
+ dial_upstream("gpt-5", "sk-test", Some(&format!("wss://127.0.0.1:{port}"))).await;
+
+ assert!(matches!(result, Err(Error::Network(_))));
+ }
+
async fn websocket_base() -> (String, tokio::task::JoinHandle<()>) {
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind");
let address = listener.local_addr().expect("local address");
diff --git a/litellm-rust/crates/ai-gateway/src/io/tls.rs b/litellm-rust/crates/ai-gateway/src/io/tls.rs
new file mode 100644
index 00000000000..a2562f60345
--- /dev/null
+++ b/litellm-rust/crates/ai-gateway/src/io/tls.rs
@@ -0,0 +1,80 @@
+//! Outbound WebSocket dials over a TLS config this crate builds once and owns.
+//!
+//! `reqwest/rustls-tls` enables `rustls/ring` and `litellm-core`'s `bedrock-auth`
+//! enables `rustls/aws-lc-rs`, so the bare `ClientConfig::builder()` that
+//! `tokio-tungstenite` uses when handed no connector panics rather than guess
+//! between them. Naming ring on a connector of our own settles that for these
+//! dials without touching the process-wide default, and building the config
+//! once keeps the platform trust store, which `tokio-tungstenite` would
+//! otherwise re-read on every dial, off the dial path.
+
+use std::io;
+use std::sync::{Arc, OnceLock};
+
+use rustls::{ClientConfig, RootCertStore};
+use tokio::net::TcpStream;
+use tokio_tungstenite::tungstenite::Error;
+use tokio_tungstenite::tungstenite::client::IntoClientRequest;
+use tokio_tungstenite::tungstenite::error::TlsError;
+use tokio_tungstenite::tungstenite::handshake::client::Response;
+use tokio_tungstenite::{
+ Connector, MaybeTlsStream, WebSocketStream, connect_async_tls_with_config,
+};
+
+static TLS_CONFIG: OnceLock