Merge remote-tracking branch 'refs/remotes/origin/litellm_redis_client_scripts' into HEAD

This commit is contained in:
Devin AI 2026-10-27 00:28:15 +00:00
commit a578ad80e7
237 changed files with 16694 additions and 10774 deletions

View file

@ -0,0 +1,52 @@
---
name: database-access
description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
---
# Database access
## Adding new database access
- Put Python database access behind an operation in `litellm/repositories`
- Give callers typed inputs and outputs, not Prisma clients, models, or query dictionaries
- Keep query execution and backend selection inside the repository
- Prisma is fine for the implementation, adding a query does not require a Rust port
- Native Rust callers use `litellm-rust/crates/db` directly
## Asked to move existing access behind a repository
- Identify the calls being moved and their transaction boundary
- Move the existing Prisma implementation into the relevant repository
- Replace those direct database calls with repository operations
- Preserve return values, errors, and transaction behavior, with focused regression tests
- Keep this an extraction, not a backend migration or unrelated cleanup
## An equivalent Rust or SQLx query already exists
- Leave it and the current backend selection unchanged unless integration is explicitly requested
- Its existence alone does not justify switching a Prisma caller to Rust
- Repository extraction can keep using Prisma even when the Rust query is available
## Asked to port a Prisma query to Rust or SQLx
- Identify the existing inputs, outputs, errors, and transaction boundary
- Check `litellm-rust/crates/db` for an equivalent query and reuse it if its behavior matches
- Otherwise follow `litellm-rust/crates/db/AGENTS.md`
- Put static SQL in `queries/` and compile-check it with SQLx
- Keep typed inputs and reusable row projections in `src/`
- Refresh SQLx query metadata after SQL changes
- Keep Python conversion in the Python bridge, not in `litellm-rust/crates/db`
- When caller integration is requested, expose the Rust operation through the bridge and call it from the repository
- A query-only port can land without switching callers
- Verify behavior against real PostgreSQL before switching callers
- Check relevant null handling, conflicts, ordering, errors, and transaction semantics
- Remove the old implementation only when no callers need it
## Migration context and boundaries
- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations incrementally with Rust APIs
- Mixed Prisma and Rust implementations are expected during the transition, unrelated existing calls can stay as they are
- Both backends may have separate pools through PgBouncer, size them together and verify pooling-mode compatibility
- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction
- Never retry a failed Rust write through Prisma when it may already have committed
- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port

View file

@ -1,319 +0,0 @@
import asyncio
import aiohttp
import json
import math
from typing import Any
# Asynchronously fetch data from a given URL
async def fetch_data(url):
try:
# Create an asynchronous session
async with aiohttp.ClientSession() as session:
# Send a GET request to the URL
async with session.get(url) as resp:
# Raise an error if the response status is not OK
resp.raise_for_status()
# Parse the response JSON
resp_json = await resp.json()
print("Fetch the data from URL.")
# Return the 'data' field from the JSON response
return resp_json['data']
except Exception as e:
# Print an error message if fetching data fails
print("Error fetching data from URL:", e)
return None
FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
FRIENDLI_PROVIDER = "friendliai"
INHERITABLE_BASE_KEYS = (
"supports_pdf_input",
"supports_assistant_prefill",
"supports_adaptive_thinking",
"supports_output_config",
)
REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
if not base_model:
return None
bm_tail = base_model.split("/")[-1].lower()
if base_model in local_data:
return base_model
for key in local_data:
if key.startswith("sample_spec") or key == "fallback_generalizations":
continue
if key.split("/")[-1].lower() == bm_tail:
return key
return None
def _reasoning_effort_levels(reasoning_options: list) -> list:
offered = {
val
for opt in reasoning_options or []
if opt.get("type") == "effort"
for val in opt.get("values", [])
}
return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
def _valid_token_price(value: object) -> bool:
try:
price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except
except (TypeError, ValueError):
return False
return math.isfinite(price) and price >= 0
def _has_valid_token_prices(pricing: dict | None) -> bool:
prices = pricing or {}
return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
def _pricing(pricing: dict) -> dict:
out: dict[str, Any] = {}
if not pricing:
return out
if "input" in pricing:
out["input_cost_per_token"] = float(pricing["input"])
if "output" in pricing:
out["output_cost_per_token"] = float(pricing["output"])
if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
return out
def _modality_flags(input_mods: list) -> dict:
mods = input_mods or []
has_image = "image" in mods
return {
"supports_vision": has_image,
"supports_image_input": has_image,
"supports_video_input": "video" in mods,
}
def transform_friendli_data(data: list, local_data: dict) -> dict:
transformed: dict[str, dict] = {}
if not data:
return transformed
for model in data:
# An unpriced row must never wholesale-replace an already priced local entry:
# missing prices cost-calculate as zero, silently zeroing tracked spend
if not _has_valid_token_prices(model.get("pricing")):
continue
model_id = model["id"]
base_model = model.get("base_model") or ""
entry: dict[str, Any] = {
"litellm_provider": FRIENDLI_PROVIDER,
}
base_key = _find_base_model_entry(base_model, local_data)
if base_key:
base_entry = local_data[base_key]
for k in INHERITABLE_BASE_KEYS:
if k in base_entry:
entry[k] = base_entry[k]
ctx = model.get("context_length")
if ctx is not None:
entry["max_input_tokens"] = int(ctx)
max_out = model.get("max_completion_tokens")
if max_out is not None:
entry["max_output_tokens"] = int(max_out)
entry["max_tokens"] = int(max_out)
pricing = _pricing(model.get("pricing", {}))
entry.update(pricing)
entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
reasoning = model.get("reasoning") is True
entry["supports_reasoning"] = reasoning
if reasoning:
entry["reasoning_effort_levels"] = _reasoning_effort_levels(
model.get("reasoning_options", [])
)
func = model.get("functionality", {})
entry["supports_function_calling"] = func.get("tool_call") is True
entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
is_struct = func.get("structured_output") is True
entry["supports_response_schema"] = is_struct
entry["supports_native_structured_output"] = is_struct
entry["supports_system_messages"] = func.get("system_messages") is True
entry["supports_tool_choice"] = func.get("tool_choice") is True
entry.update(_modality_flags(model.get("input_modalities", [])))
entry["mode"] = model.get("mode", "chat")
desc = model.get("description")
if desc:
entry["comment"] = desc
dep = model.get("deprecation_date")
if dep:
entry["deprecation_date"] = dep.split("T")[0]
entry["source"] = FRIENDLI_API_URL
transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
return transformed
# Synchronize local data with remote data
def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
# Update existing keys in local_data with values from remote_data
# (replace_keys entries are swapped wholesale so a field the remote catalog
# dropped, e.g. cache pricing, cannot survive as a stale value)
for key in (set(local_data) & set(remote_data)):
if key in replace_keys:
local_data[key] = remote_data[key]
else:
local_data[key].update(remote_data[key])
# Add new keys from remote_data to local_data
for key in (set(remote_data) - set(local_data)):
local_data[key] = remote_data[key]
# Write data to the json file
def write_to_file(file_path, data):
try:
# Open the file in write mode
with open(file_path, "w") as file:
# Dump the data as JSON into the file
json.dump(data, file, indent=4)
print("Values updated successfully.")
except Exception as e:
# Print an error message if writing to file fails
print("Error updating JSON file:", e)
# Update the existing models and add the missing models for OpenRouter
def transform_openrouter_data(data):
transformed = {}
if not data:
return transformed
for row in data:
# Add the fields 'max_tokens' and 'input_cost_per_token'
obj = {
"max_tokens": row["context_length"],
"input_cost_per_token": float(row["pricing"]["prompt"]),
}
# Add 'max_output_tokens' as a field if it is not None
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
# Add the field 'output_cost_per_token'
obj.update({
"output_cost_per_token": float(row["pricing"]["completion"]),
})
# Add field 'input_cost_per_image' if it exists and is non-zero
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
obj['input_cost_per_image'] = float(row["pricing"]["image"])
# Add the fields 'litellm_provider' and 'mode'
obj.update({
"litellm_provider": "openrouter",
"mode": "chat"
})
# Add the 'supports_vision' field if the modality is 'multimodal'
if row.get('architecture', {}).get('modality') == 'multimodal':
obj['supports_vision'] = True
# Use a composite key to store the transformed object
transformed[f'openrouter/{row["id"]}'] = obj
return transformed
# Update the existing models and add the missing models for Vercel AI Gateway
def transform_vercel_ai_gateway_data(data):
transformed = {}
if not data:
return transformed
for row in data:
# Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
row.get("pricing", {}).get(k) is None for k in ("input", "output")
):
continue
obj = {
"max_tokens": row["context_window"],
"input_cost_per_token": float(row["pricing"]["input"]),
"output_cost_per_token": float(row["pricing"]["output"]),
'max_output_tokens': row['max_tokens'],
'max_input_tokens': row["context_window"],
}
# Handle cache pricing if available
if "pricing" in row:
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
return transformed
# Load local data from a specified file
def load_local_data(file_path):
try:
# Open the file in read mode
with open(file_path, "r") as file:
# Load and return the JSON data
return json.load(file)
except FileNotFoundError:
# Print an error message if the file is not found
print("File not found:", file_path)
return None
except json.JSONDecodeError as e:
# Print an error message if JSON decoding fails
print("Error decoding JSON:", e)
return None
def main():
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
# Load local data from file
local_data = load_local_data(local_file_path)
# Fetch OpenRouter data
openrouter_data = asyncio.run(fetch_data(openrouter_url))
# Transform the fetched OpenRouter data
openrouter_data = transform_openrouter_data(openrouter_data)
# Fetch Vercel AI Gateway data
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
# Transform the fetched Vercel AI Gateway data
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
friendli_data = transform_friendli_data(friendli_data, local_data)
# Combine both datasets
all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
# If both local and openrouter data are available, synchronize and save
if local_data and all_remote_data:
sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
write_to_file(local_file_path, local_data)
else:
print("Failed to fetch model data from either local file or URL.")
# Entry point of the script
if __name__ == "__main__":
main()

View file

@ -1,36 +0,0 @@
name: Updates model_prices_and_context_window.json and Create Pull Request
on:
schedule:
- cron: "0 0 * * 0" # Run every Sundays at midnight
#- cron: "0 0 * * *" # Run daily at midnight
permissions:
contents: write
pull-requests: write
jobs:
auto_update_price_and_context_window:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Update JSON Data
run: |
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
- name: Create Pull Request
run: |
git add model_prices_and_context_window.json
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
gh pr create --title "Update model_prices_and_context_window.json file" \
--body "Automated update for model_prices_and_context_window.json" \
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
--base main
env:
GH_TOKEN: ${{ secrets.GH_TOKEN }}

View file

@ -1,65 +0,0 @@
name: Sync Together AI model registry
on:
schedule:
- cron: "30 6 * * *"
workflow_dispatch:
permissions:
contents: write
pull-requests: write
jobs:
sync_together_ai_models:
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
env:
BASE_BRANCH: ${{ github.event.repository.default_branch }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
ref: ${{ env.BASE_BRANCH }}
persist-credentials: false
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Look for an already-open sync PR
id: existing
run: |
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
if [ -n "$open_pr" ]; then
echo "An open sync PR already exists on branch $open_pr; skipping this run."
fi
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
- name: Run the sync
if: steps.existing.outputs.open_pr == ''
run: |
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
env:
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
- name: Create a pull request when the registry changed
if: steps.existing.outputs.open_pr == ''
run: |
if git diff --quiet; then
echo "Registry already in sync; no PR needed."
exit 0
fi
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
git config user.name "github-actions[bot]"
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
git checkout -b "$branch"
git add model_prices_and_context_window.json \
litellm/model_prices_and_context_window_backup.json
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
gh auth setup-git
git push origin "$branch"
gh pr create --title "feat(models): sync together_ai model registry" \
--body-file "$RUNNER_TEMP/pr_body.md" \
--head "$branch" \
--base "$BASE_BRANCH"
env:
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}

View file

@ -45,7 +45,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
@ -132,7 +132,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:

View file

@ -33,7 +33,7 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
env:
LITELLM_LOCAL_MODEL_COST_MAP: "True"
steps:
@ -76,6 +76,12 @@ jobs:
- name: Import and CLI checks
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
- name: Routing and UUID compatibility
run: >-
$IN_NETNS .venv/bin/python -m pytest
tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions
tests/unit/test_uuid_helper.py -q
- name: Proxy startup check
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"

106
.github/workflows/test-rust-db.yml vendored Normal file
View file

@ -0,0 +1,106 @@
name: LiteLLM Rust DB
on:
push:
paths:
- "litellm-rust/crates/db/**"
- "litellm-rust/Cargo.toml"
- "litellm-rust/Cargo.lock"
- "litellm-rust/clippy.toml"
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
- "**/schema.prisma"
- ".cargo/**"
- "rust-toolchain.toml"
- "pyproject.toml"
- "uv.lock"
- ".github/actions/setup-uv-with-retries/**"
- ".github/workflows/test-rust-db.yml"
pull_request:
branches:
- main
- "litellm_**"
paths:
- "litellm-rust/crates/db/**"
- "litellm-rust/Cargo.toml"
- "litellm-rust/Cargo.lock"
- "litellm-rust/clippy.toml"
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
- "**/schema.prisma"
- ".cargo/**"
- "rust-toolchain.toml"
- "pyproject.toml"
- "uv.lock"
- ".github/actions/setup-uv-with-retries/**"
- ".github/workflows/test-rust-db.yml"
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
CARGO_TERM_COLOR: always
jobs:
db-queries:
runs-on: ubuntu-latest
timeout-minutes: 30
services:
postgres:
image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
env:
POSTGRES_DB: litellm
POSTGRES_PASSWORD: postgres
POSTGRES_USER: postgres
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U postgres -d litellm"
--health-interval 10s
--health-timeout 5s
--health-retries 5
env:
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm
SQLX_OFFLINE: "false"
defaults:
run:
working-directory: litellm-rust
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- run: rustup toolchain install --no-self-update
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
with:
workspaces: litellm-rust
cache-on-failure: true
- name: Install Python dependencies
working-directory: .
run: uv sync --frozen --extra extra-proxy
- name: Deploy Prisma migrations
working-directory: .
run: uv run --no-sync prisma migrate deploy --schema litellm-proxy-extras/litellm_proxy_extras/schema.prisma
- name: Install SQLx CLI
run: cargo install sqlx-cli --version 0.9.0 --locked --no-default-features -F postgres
- name: Check SQLx query metadata
working-directory: litellm-rust/crates/db
run: cargo sqlx prepare --check
- name: Test database queries
run: cargo test -p litellm-db --locked -- --include-ignored

View file

@ -0,0 +1,76 @@
# Use your ChatGPT subscription through LiteLLM
Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution.
An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding.
## Add a model to the proxy
Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy:
```yaml
model_list:
- model_name: gpt-6-luna
litellm_params:
model: chatgpt/gpt-6-luna
model_info:
mode: responses
```
Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic.
## Configure Codex
Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration.
Add this provider to `~/.codex/config.toml`, preserving your other settings:
```toml
model = "gpt-6-luna"
model_provider = "litellm_subscription"
[model_providers.litellm_subscription]
name = "LiteLLM subscription"
base_url = "https://litellm.example.com/v1"
wire_api = "responses"
supports_websockets = false
requires_openai_auth = true
env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" }
```
Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway.
For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway).
## Native compatibility and Pi
For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name.
### Configure Pi
Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi:
```json
{
"providers": {
"openai-codex": {
"baseUrl": "https://litellm.example.com/backend-api/codex",
"headers": {
"x-litellm-api-key": "LITELLM_API_KEY"
}
}
}
}
```
Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file.
## Verify account usage
Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person.
The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate.
This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes.

View file

@ -3716,8 +3716,15 @@ dependencies = [
name = "litellm-db"
version = "0.1.0"
dependencies = [
"chrono",
"macro_rules_attribute",
"rstest",
"serde",
"serde_json",
"sqlx",
"strum",
"thiserror 2.0.19",
"tokio",
]
[[package]]
@ -8257,18 +8264,18 @@ dependencies = [
[[package]]
name = "webpki-root-certs"
version = "1.0.10"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
dependencies = [
"rustls-pki-types",
]
[[package]]
name = "webpki-roots"
version = "1.0.10"
version = "1.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
dependencies = [
"rustls-pki-types",
]

View file

@ -110,6 +110,7 @@ serde_with = { version = "=3.21.0", default-features = false, features = ["std",
sha2 = "0.10"
syn = { version = "2", default-features = false }
sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
chrono = { version = "0.4.45", default-features = false, features = ["std"] }
proc-macro2 = "1"
quote = "1"
subtle = "2"

View file

@ -51,7 +51,7 @@ mod rules;
mod status;
mod vertex_ai;
pub use original::{ExceptionFamily, OriginalException};
pub use original::{ExceptionFamily, OriginalException, TimeoutPhase};
pub use public::{MappedFailure, PublicError, UpstreamResponse};
use rules::{Rule, contains_any, first_match};
@ -67,7 +67,6 @@ const TIMEOUT_MARKERS: &[&str] = &[
pub struct ExceptionContext {
pub model: String,
pub custom_llm_provider: String,
pub asynchronous: bool,
pub vertex_project: Option<String>,
pub vertex_location: Option<String>,
pub model_group: Option<String>,
@ -105,11 +104,12 @@ pub fn exception_type(
(None, message.clone(), None)
}
OriginalException::Timeout {
phase,
timeout_seconds,
elapsed_seconds,
} => (
None,
timeout_message(context.asynchronous, *timeout_seconds, *elapsed_seconds),
timeout_message(*phase, *timeout_seconds, *elapsed_seconds),
None,
),
};
@ -161,21 +161,16 @@ fn family_rules(family: ExceptionFamily) -> &'static [Rule] {
}
}
/// The text the Python HTTP handler's timeout carries: the sync and async handlers word it
/// differently.
fn timeout_message(
asynchronous: bool,
/// The text the Python HTTP handler's timeout carries: the phase that expired and the limit
/// httpx enforced for it, both rendered as Python would print them.
pub fn timeout_message(
phase: TimeoutPhase,
timeout_seconds: Option<f64>,
elapsed_seconds: Option<f64>,
) -> String {
let timeout = python_float(timeout_seconds);
if asynchronous {
let elapsed =
python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
format!("Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
} else {
format!("Connection timed out after {timeout} seconds.")
}
let elapsed = python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
format!("{phase} timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
}
fn python_float(value: Option<f64>) -> String {
@ -356,6 +351,7 @@ mod tests {
#[test]
fn a_handler_timeout_is_a_408_without_a_response() {
let original = OriginalException::Timeout {
phase: TimeoutPhase::Read,
timeout_seconds: Some(0.5),
elapsed_seconds: Some(0.5031),
};
@ -363,7 +359,7 @@ mod tests {
mapped("mistral", &original),
MappedFailure {
error: PublicError::Timeout { status: 408 },
message: "MistralException - Connection timed out after 0.5 seconds.".into(),
message: "MistralException - Read timed out. Timeout passed=0.5, time taken=0.503 seconds".into(),
upstream: None,
debug_info: DEBUG.into(),
}
@ -440,39 +436,44 @@ mod tests {
}
#[rstest::rstest]
#[case::sync(
false,
#[case::connect(TimeoutPhase::Connect, "Connection timed out.")]
#[case::read(TimeoutPhase::Read, "Read timed out.")]
#[case::write(TimeoutPhase::Write, "Write timed out.")]
#[case::pool(TimeoutPhase::Pool, "Connection pool timed out.")]
fn timeout_text_names_the_phase_that_expired(
#[case] phase: TimeoutPhase,
#[case] expected: &str,
) {
assert_eq!(
timeout_message(phase, Some(0.5), Some(0.5031)),
format!("{expected} Timeout passed=0.5, time taken=0.503 seconds")
);
}
#[rstest::rstest]
#[case::rounds_the_elapsed_time(
Some(0.5),
Some(0.5031),
"Connection timed out after 0.5 seconds."
)]
#[case::async_rounds_the_elapsed_time(
true,
Some(0.5),
Some(0.5031),
"Connection timed out. Timeout passed=0.5, time taken=0.503 seconds"
"Timeout passed=0.5, time taken=0.503 seconds"
)]
#[case::whole_seconds_keep_a_decimal(
true,
Some(600.0),
Some(2.0),
"Connection timed out. Timeout passed=600.0, time taken=2.0 seconds"
"Timeout passed=600.0, time taken=2.0 seconds"
)]
#[case::unknown_values_render_as_none(
true,
None,
None,
"Connection timed out. Timeout passed=None, time taken=None seconds"
"Timeout passed=None, time taken=None seconds"
)]
fn timeout_text_follows_the_delivery_mode(
#[case] asynchronous: bool,
fn timeout_text_renders_durations_as_python_floats(
#[case] timeout_seconds: Option<f64>,
#[case] elapsed_seconds: Option<f64>,
#[case] expected: &str,
) {
assert_eq!(
timeout_message(asynchronous, timeout_seconds, elapsed_seconds),
expected
timeout_message(TimeoutPhase::Read, timeout_seconds, elapsed_seconds),
format!("Read timed out. {expected}")
);
}

View file

@ -10,6 +10,7 @@ pub enum OriginalException {
message: String,
},
Timeout {
phase: TimeoutPhase,
timeout_seconds: Option<f64>,
elapsed_seconds: Option<f64>,
},
@ -20,6 +21,19 @@ pub enum OriginalException {
},
}
/// The httpx timeout that expired, spelled the way the Python HTTP handler names it.
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::VariantArray)]
pub enum TimeoutPhase {
#[strum(serialize = "Connection")]
Connect,
#[strum(serialize = "Read")]
Read,
#[strum(serialize = "Write")]
Write,
#[strum(serialize = "Connection pool")]
Pool,
}
/// Which provider-specific text rules apply before the shared status table.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ExceptionFamily {

View file

@ -0,0 +1,23 @@
{
"db_name": "PostgreSQL",
"query": "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS \"locked!\"\n",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "locked!",
"type_info": "Bool",
"origin": "Expression"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
null
]
},
"hash": "085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f"
}

View file

@ -0,0 +1,40 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_SpendLogs\" (\n request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, \"startTime\", \"endTime\", \"completionStartTime\",\n request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,\n \"user\", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,\n end_user, requester_ip_address, messages, response, proxy_server_request,\n session_id, status, litellm_call_id\n)\nSELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,\n l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),\n l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,\n l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,\n l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,\n l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,\n '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id\nFROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],\n $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],\n $13::text[], $14::text[], $15::text[], $16::text[], $17::text[],\n $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],\n $23::text[], $24::text[], $25::text[], $26::text[], $27::text[])\n AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, start_time, end_time, completion_start_time,\n request_duration_ms, model, model_id, model_group, custom_llm_provider,\n api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,\n end_user, requester_ip_address, session_id, status, litellm_call_id)\nON CONFLICT DO NOTHING\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"TextArray",
"Float8Array",
"Int4Array",
"Int4Array",
"Int4Array",
"TimestampArray",
"TimestampArray",
"TimestampArray",
"Int4Array",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"JsonbArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray"
]
},
"nullable": []
},
"hash": "32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b"
}

View file

@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_EndUserTable\" (user_id, spend, blocked)\nSELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\nGROUP BY id\nORDER BY id\nON CONFLICT (user_id) DO UPDATE\nSET spend = \"LiteLLM_EndUserTable\".spend + EXCLUDED.spend\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Float8Array"
]
},
"nullable": []
},
"hash": "345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a"
}

View file

@ -0,0 +1,78 @@
{
"db_name": "PostgreSQL",
"query": "SELECT e.spend AS \"spend?\",\n e.blocked AS \"blocked?\",\n b.max_budget AS \"max_budget?\",\n b.rpm_limit AS \"rpm_limit?\",\n b.tpm_limit AS \"tpm_limit?\"\nFROM (SELECT 1) AS one\nLEFT JOIN \"LiteLLM_EndUserTable\" AS e ON e.user_id = $1\nLEFT JOIN \"LiteLLM_BudgetTable\" AS assigned ON assigned.budget_id = e.budget_id\nLEFT JOIN \"LiteLLM_BudgetTable\" AS key_budget ON key_budget.budget_id = $3\nLEFT JOIN \"LiteLLM_BudgetTable\" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)\n",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "spend?",
"type_info": "Float8",
"origin": {
"Table": {
"table": "\"LiteLLM_EndUserTable\"",
"name": "spend"
}
}
},
{
"ordinal": 1,
"name": "blocked?",
"type_info": "Bool",
"origin": {
"Table": {
"table": "\"LiteLLM_EndUserTable\"",
"name": "blocked"
}
}
},
{
"ordinal": 2,
"name": "max_budget?",
"type_info": "Float8",
"origin": {
"Table": {
"table": "\"LiteLLM_BudgetTable\"",
"name": "max_budget"
}
}
},
{
"ordinal": 3,
"name": "rpm_limit?",
"type_info": "Int8",
"origin": {
"Table": {
"table": "\"LiteLLM_BudgetTable\"",
"name": "rpm_limit"
}
}
},
{
"ordinal": 4,
"name": "tpm_limit?",
"type_info": "Int8",
"origin": {
"Table": {
"table": "\"LiteLLM_BudgetTable\"",
"name": "tpm_limit"
}
}
}
],
"parameters": {
"Left": [
"Text",
"Text",
"Text"
]
},
"nullable": [
true,
true,
true,
true,
true
]
},
"hash": "377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e"
}

View file

@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT t.token FROM \"LiteLLM_VerificationToken\" t\n WHERE t.token IN (SELECT id FROM added)\n ORDER BY t.token FOR UPDATE\n)\nUPDATE \"LiteLLM_VerificationToken\" t\nSET spend = t.spend + added.cost,\n total_spend = t.total_spend + added.cost,\n last_active = (clock_timestamp() AT TIME ZONE 'UTC'),\n updated_at = (clock_timestamp() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.token = added.id AND t.token IN (SELECT token FROM locked)\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Float8Array"
]
},
"nullable": []
},
"hash": "3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563"
}

View file

@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT u.user_id FROM \"LiteLLM_UserTable\" u\n WHERE u.user_id IN (SELECT id FROM added)\n ORDER BY u.user_id FOR UPDATE\n)\nUPDATE \"LiteLLM_UserTable\" u\nSET spend = u.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Float8Array"
]
},
"nullable": []
},
"hash": "43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54"
}

View file

@ -0,0 +1,21 @@
{
"db_name": "PostgreSQL",
"query": "SELECT set_config('lock_timeout', '5000ms', true)",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "set_config",
"type_info": "Text",
"origin": "Expression"
}
],
"parameters": {
"Left": []
},
"nullable": [
null
]
},
"hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4"
}

View file

@ -0,0 +1,31 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_DailyTeamSpend\" (\n id, team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyTeamSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyTeamSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyTeamSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyTeamSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyTeamSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyTeamSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyTeamSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyTeamSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Float8Array",
"TextArray"
]
},
"nullable": []
},
"hash": "9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416"
}

View file

@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_TeamMembership\" (user_id, team_id, spend, total_spend)\nSELECT member.user_id, member.team_id, member.cost, member.cost\nFROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)\nWHERE EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id = member.team_id\n AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))\n)\n OR EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamMembership\" m\n WHERE m.user_id = member.user_id AND m.team_id = member.team_id\n)\nON CONFLICT (user_id, team_id) DO UPDATE\nSET spend = \"LiteLLM_TeamMembership\".spend + EXCLUDED.spend,\n total_spend = \"LiteLLM_TeamMembership\".total_spend + EXCLUDED.total_spend\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb"
]
},
"nullable": []
},
"hash": "b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147"
}

View file

@ -0,0 +1,31 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_DailyUserSpend\" (\n id, user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Float8Array",
"TextArray"
]
},
"nullable": []
},
"hash": "bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe"
}

View file

@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n),\nlocked AS (\n SELECT t.team_id FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id IN (SELECT id FROM added)\n ORDER BY t.team_id FOR UPDATE\n)\nUPDATE \"LiteLLM_TeamTable\" t\nSET spend = t.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"Float8Array"
]
},
"nullable": []
},
"hash": "c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce"
}

View file

@ -0,0 +1,31 @@
{
"db_name": "PostgreSQL",
"query": "INSERT INTO \"LiteLLM_DailyEndUserSpend\" (\n id, end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyEndUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyEndUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyEndUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyEndUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyEndUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyEndUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyEndUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyEndUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
"describe": {
"columns": [],
"parameters": {
"Left": [
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"TextArray",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Int8Array",
"Float8Array",
"TextArray"
]
},
"nullable": []
},
"hash": "f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d"
}

View file

@ -0,0 +1,25 @@
# Database crate
Keep each static query in `queries/` and call it through a typed SQLx query function. Rust does not create or migrate the database
- Own reusable row projections in `src/` and export those consumed by other crates or integration tests
- Apply `#[macro_rules_attribute::apply(crate::db_row)]` to row types for shared derives, without adding serialization or schema derives automatically
- Apply `#[macro_rules_attribute::apply(crate::db_input)]` to write inputs, adding `Default` only where supported
- Keep function tests inline in their owning query module and reuse the strict database fixture in `test_support`
- Transaction-local setup in `begin` is the approved exception to static query files
- Keep write inputs separate from stored rows and keep test-only assertions out of public types
- `FromRow` supports runtime reads, but production static reads still use `query_file_as!` for compile-time checking
Database integration tests are ignored by default, but still compile in workspace checks. Run them explicitly against a local database already migrated by Prisma. Missing or invalid database configuration must fail the selected tests:
```sh
DATABASE_URL=postgresql://... cargo test -p litellm-db --locked -- --include-ignored
```
After changing a query, refresh the checked query metadata from this directory:
```sh
SQLX_OFFLINE=false cargo sqlx prepare
```
Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes

View file

@ -6,5 +6,14 @@ license.workspace = true
repository.workspace = true
[dependencies]
chrono.workspace = true
macro_rules_attribute.workspace = true
serde.workspace = true
serde_json.workspace = true
sqlx.workspace = true
strum.workspace = true
thiserror.workspace = true
[dev-dependencies]
rstest.workspace = true
tokio.workspace = true

View file

@ -0,0 +1,10 @@
SELECT e.spend AS "spend?",
e.blocked AS "blocked?",
b.max_budget AS "max_budget?",
b.rpm_limit AS "rpm_limit?",
b.tpm_limit AS "tpm_limit?"
FROM (SELECT 1) AS one
LEFT JOIN "LiteLLM_EndUserTable" AS e ON e.user_id = $1
LEFT JOIN "LiteLLM_BudgetTable" AS assigned ON assigned.budget_id = e.budget_id
LEFT JOIN "LiteLLM_BudgetTable" AS key_budget ON key_budget.budget_id = $3
LEFT JOIN "LiteLLM_BudgetTable" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)

View file

@ -0,0 +1,6 @@
INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
SELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
GROUP BY id
ORDER BY id
ON CONFLICT (user_id) DO UPDATE
SET spend = "LiteLLM_EndUserTable".spend + EXCLUDED.spend

View file

@ -0,0 +1,16 @@
WITH added AS (
SELECT id, SUM(cost) AS cost
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
GROUP BY id
), locked AS (
SELECT t.token FROM "LiteLLM_VerificationToken" t
WHERE t.token IN (SELECT id FROM added)
ORDER BY t.token FOR UPDATE
)
UPDATE "LiteLLM_VerificationToken" t
SET spend = t.spend + added.cost,
total_spend = t.total_spend + added.cost,
last_active = (clock_timestamp() AT TIME ZONE 'UTC'),
updated_at = (clock_timestamp() AT TIME ZONE 'UTC')
FROM added
WHERE t.token = added.id AND t.token IN (SELECT token FROM locked)

View file

@ -0,0 +1,15 @@
WITH added AS (
SELECT id, SUM(cost) AS cost
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
GROUP BY id
),
locked AS (
SELECT t.team_id FROM "LiteLLM_TeamTable" t
WHERE t.team_id IN (SELECT id FROM added)
ORDER BY t.team_id FOR UPDATE
)
UPDATE "LiteLLM_TeamTable" t
SET spend = t.spend + added.cost,
updated_at = (NOW() AT TIME ZONE 'UTC')
FROM added
WHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)

View file

@ -0,0 +1,14 @@
WITH added AS (
SELECT id, SUM(cost) AS cost
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
GROUP BY id
), locked AS (
SELECT u.user_id FROM "LiteLLM_UserTable" u
WHERE u.user_id IN (SELECT id FROM added)
ORDER BY u.user_id FOR UPDATE
)
UPDATE "LiteLLM_UserTable" u
SET spend = u.spend + added.cost,
updated_at = (NOW() AT TIME ZONE 'UTC')
FROM added
WHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)

View file

@ -0,0 +1,26 @@
INSERT INTO "LiteLLM_SpendLogs" (
request_id, call_type, api_key, spend, total_tokens, prompt_tokens,
completion_tokens, "startTime", "endTime", "completionStartTime",
request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,
"user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,
end_user, requester_ip_address, messages, response, proxy_server_request,
session_id, status, litellm_call_id
)
SELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,
l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),
l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,
l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,
l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,
l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,
'{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id
FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],
$7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],
$13::text[], $14::text[], $15::text[], $16::text[], $17::text[],
$18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],
$23::text[], $24::text[], $25::text[], $26::text[], $27::text[])
AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,
completion_tokens, start_time, end_time, completion_start_time,
request_duration_ms, model, model_id, model_group, custom_llm_provider,
api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,
end_user, requester_ip_address, session_id, status, litellm_call_id)
ON CONFLICT DO NOTHING

View file

@ -0,0 +1 @@
SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS "locked!"

View file

@ -0,0 +1,40 @@
INSERT INTO "LiteLLM_DailyEndUserSpend" (
id, end_user_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
completion_tokens, api_requests, successful_requests, failed_requests,
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, updated_at
)
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
r.completion_tokens, r.api_requests, r.successful_requests,
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
FROM UNNEST(
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
$16::int8[], $17::float8[], $18::text[]
) AS r(
entity_id, date, api_key, model, custom_llm_provider, endpoint,
model_group, prompt_tokens, completion_tokens, api_requests,
successful_requests, failed_requests, cache_read, cache_creation,
response_time, timed_requests, spend, mcp_namespaced_tool_name
)
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
r.mcp_namespaced_tool_name, r.endpoint
ON CONFLICT (
end_user_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint
) DO UPDATE SET
prompt_tokens = "LiteLLM_DailyEndUserSpend".prompt_tokens + EXCLUDED.prompt_tokens,
completion_tokens = "LiteLLM_DailyEndUserSpend".completion_tokens + EXCLUDED.completion_tokens,
api_requests = "LiteLLM_DailyEndUserSpend".api_requests + EXCLUDED.api_requests,
successful_requests = "LiteLLM_DailyEndUserSpend".successful_requests + EXCLUDED.successful_requests,
failed_requests = "LiteLLM_DailyEndUserSpend".failed_requests + EXCLUDED.failed_requests,
cache_read_input_tokens = "LiteLLM_DailyEndUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
cache_creation_input_tokens = "LiteLLM_DailyEndUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
total_response_time_ms = "LiteLLM_DailyEndUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
timed_requests = "LiteLLM_DailyEndUserSpend".timed_requests + EXCLUDED.timed_requests,
spend = "LiteLLM_DailyEndUserSpend".spend + EXCLUDED.spend,
updated_at = (NOW() AT TIME ZONE 'UTC')

View file

@ -0,0 +1,40 @@
INSERT INTO "LiteLLM_DailyTeamSpend" (
id, team_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
completion_tokens, api_requests, successful_requests, failed_requests,
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, updated_at
)
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
r.completion_tokens, r.api_requests, r.successful_requests,
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
FROM UNNEST(
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
$16::int8[], $17::float8[], $18::text[]
) AS r(
entity_id, date, api_key, model, custom_llm_provider, endpoint,
model_group, prompt_tokens, completion_tokens, api_requests,
successful_requests, failed_requests, cache_read, cache_creation,
response_time, timed_requests, spend, mcp_namespaced_tool_name
)
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
r.mcp_namespaced_tool_name, r.endpoint
ON CONFLICT (
team_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint
) DO UPDATE SET
prompt_tokens = "LiteLLM_DailyTeamSpend".prompt_tokens + EXCLUDED.prompt_tokens,
completion_tokens = "LiteLLM_DailyTeamSpend".completion_tokens + EXCLUDED.completion_tokens,
api_requests = "LiteLLM_DailyTeamSpend".api_requests + EXCLUDED.api_requests,
successful_requests = "LiteLLM_DailyTeamSpend".successful_requests + EXCLUDED.successful_requests,
failed_requests = "LiteLLM_DailyTeamSpend".failed_requests + EXCLUDED.failed_requests,
cache_read_input_tokens = "LiteLLM_DailyTeamSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
cache_creation_input_tokens = "LiteLLM_DailyTeamSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
total_response_time_ms = "LiteLLM_DailyTeamSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
timed_requests = "LiteLLM_DailyTeamSpend".timed_requests + EXCLUDED.timed_requests,
spend = "LiteLLM_DailyTeamSpend".spend + EXCLUDED.spend,
updated_at = (NOW() AT TIME ZONE 'UTC')

View file

@ -0,0 +1,40 @@
INSERT INTO "LiteLLM_DailyUserSpend" (
id, user_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
completion_tokens, api_requests, successful_requests, failed_requests,
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, updated_at
)
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
r.completion_tokens, r.api_requests, r.successful_requests,
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
FROM UNNEST(
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
$16::int8[], $17::float8[], $18::text[]
) AS r(
entity_id, date, api_key, model, custom_llm_provider, endpoint,
model_group, prompt_tokens, completion_tokens, api_requests,
successful_requests, failed_requests, cache_read, cache_creation,
response_time, timed_requests, spend, mcp_namespaced_tool_name
)
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
r.mcp_namespaced_tool_name, r.endpoint
ON CONFLICT (
user_id, date, api_key, model, custom_llm_provider,
mcp_namespaced_tool_name, endpoint
) DO UPDATE SET
prompt_tokens = "LiteLLM_DailyUserSpend".prompt_tokens + EXCLUDED.prompt_tokens,
completion_tokens = "LiteLLM_DailyUserSpend".completion_tokens + EXCLUDED.completion_tokens,
api_requests = "LiteLLM_DailyUserSpend".api_requests + EXCLUDED.api_requests,
successful_requests = "LiteLLM_DailyUserSpend".successful_requests + EXCLUDED.successful_requests,
failed_requests = "LiteLLM_DailyUserSpend".failed_requests + EXCLUDED.failed_requests,
cache_read_input_tokens = "LiteLLM_DailyUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
cache_creation_input_tokens = "LiteLLM_DailyUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
total_response_time_ms = "LiteLLM_DailyUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
timed_requests = "LiteLLM_DailyUserSpend".timed_requests + EXCLUDED.timed_requests,
spend = "LiteLLM_DailyUserSpend".spend + EXCLUDED.spend,
updated_at = (NOW() AT TIME ZONE 'UTC')

View file

@ -0,0 +1,15 @@
INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend, total_spend)
SELECT member.user_id, member.team_id, member.cost, member.cost
FROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)
WHERE EXISTS (
SELECT 1 FROM "LiteLLM_TeamTable" t
WHERE t.team_id = member.team_id
AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))
)
OR EXISTS (
SELECT 1 FROM "LiteLLM_TeamMembership" m
WHERE m.user_id = member.user_id AND m.team_id = member.team_id
)
ON CONFLICT (user_id, team_id) DO UPDATE
SET spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend,
total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend

View file

@ -0,0 +1,76 @@
use sqlx::types::chrono::NaiveDateTime;
#[macro_rules_attribute::apply(crate::db_row)]
pub struct DailyUserRow {
pub id: String,
pub user_id: Option<String>,
pub api_key: String,
pub model: Option<String>,
pub custom_llm_provider: Option<String>,
pub endpoint: Option<String>,
pub model_group: Option<String>,
pub mcp_namespaced_tool_name: Option<String>,
pub prompt_tokens: i64,
pub completion_tokens: i64,
pub api_requests: i64,
pub successful_requests: i64,
pub failed_requests: i64,
pub cache_read_input_tokens: i64,
pub cache_creation_input_tokens: i64,
pub total_response_time_ms: i64,
pub timed_requests: i64,
pub spend: f64,
pub date: String,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
}
#[macro_rules_attribute::apply(crate::db_row)]
pub struct DailyTeamRow {
pub id: String,
pub team_id: Option<String>,
pub api_key: String,
pub model: Option<String>,
pub custom_llm_provider: Option<String>,
pub endpoint: Option<String>,
pub model_group: Option<String>,
pub mcp_namespaced_tool_name: Option<String>,
pub prompt_tokens: i64,
pub completion_tokens: i64,
pub api_requests: i64,
pub successful_requests: i64,
pub failed_requests: i64,
pub cache_read_input_tokens: i64,
pub cache_creation_input_tokens: i64,
pub total_response_time_ms: i64,
pub timed_requests: i64,
pub spend: f64,
pub date: String,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
}
#[macro_rules_attribute::apply(crate::db_row)]
pub struct DailyEndUserRow {
pub id: String,
pub end_user_id: Option<String>,
pub api_key: String,
pub model: Option<String>,
pub custom_llm_provider: Option<String>,
pub endpoint: Option<String>,
pub model_group: Option<String>,
pub mcp_namespaced_tool_name: Option<String>,
pub prompt_tokens: i64,
pub completion_tokens: i64,
pub api_requests: i64,
pub successful_requests: i64,
pub failed_requests: i64,
pub cache_read_input_tokens: i64,
pub cache_creation_input_tokens: i64,
pub total_response_time_ms: i64,
pub timed_requests: i64,
pub spend: f64,
pub date: String,
pub created_at: NaiveDateTime,
pub updated_at: NaiveDateTime,
}

View file

@ -0,0 +1,359 @@
use sqlx::PgExecutor;
use crate::Error;
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
pub struct RateLimits {
pub rpm: Option<i64>,
pub tpm: Option<i64>,
}
#[derive(Clone, Debug, Default, PartialEq)]
pub struct EndUser {
pub spend: Option<f64>,
pub blocked: bool,
pub max_budget: Option<f64>,
pub limits: RateLimits,
}
#[macro_rules_attribute::apply(crate::db_row)]
struct EndUserRow {
spend: Option<f64>,
blocked: Option<bool>,
max_budget: Option<f64>,
rpm_limit: Option<i64>,
tpm_limit: Option<i64>,
}
pub async fn find_end_user(
executor: impl PgExecutor<'_>,
end_user_id: &str,
default_budget_id: Option<&str>,
key_budget_id: Option<&str>,
) -> Result<EndUser, Error> {
let row = sqlx::query_file_as!(
EndUserRow,
"queries/end_user/find_end_user.sql",
end_user_id,
default_budget_id,
key_budget_id,
)
.fetch_one(executor)
.await?;
Ok(EndUser {
spend: row.spend,
blocked: row.blocked.unwrap_or(false),
max_budget: row.max_budget,
limits: RateLimits {
rpm: row.rpm_limit,
tpm: row.tpm_limit,
},
})
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{EndUser, RateLimits, find_end_user};
use crate::test_support::database_pool;
use rstest::rstest;
use sqlx::{PgPool, Postgres, Transaction};
#[derive(Clone, Copy)]
struct BudgetFixture {
budget_id: &'static str,
max_budget: f64,
rpm_limit: i64,
tpm_limit: i64,
}
async fn insert_budget(
tx: &mut Transaction<'_, Postgres>,
budget: BudgetFixture,
) -> Result<(), sqlx::Error> {
#[expect(
clippy::disallowed_methods,
reason = "Fixture inserts are outside the compile-checked query set"
)]
let result = sqlx::query(
r#"
INSERT INTO "LiteLLM_BudgetTable"
(budget_id, max_budget, rpm_limit, tpm_limit, created_by, updated_by)
VALUES ($1, $2, $3, $4, 'db-query-test', 'db-query-test')
"#,
)
.bind(budget.budget_id)
.bind(budget.max_budget)
.bind(budget.rpm_limit)
.bind(budget.tpm_limit)
.execute(&mut **tx)
.await;
result.map(|_| ())
}
async fn insert_end_user(
tx: &mut Transaction<'_, Postgres>,
user_id: &str,
spend: f64,
blocked: bool,
budget_id: Option<&str>,
) -> Result<(), sqlx::Error> {
#[expect(
clippy::disallowed_methods,
reason = "Fixture inserts are outside the compile-checked query set"
)]
let result = sqlx::query(
r#"
INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id)
VALUES ($1, $2, $3, $4)
"#,
)
.bind(user_id)
.bind(spend)
.bind(blocked)
.bind(budget_id)
.execute(&mut **tx)
.await;
result.map(|_| ())
}
#[rstest]
#[case::without_default_budget(None)]
#[case::with_default_budget(Some(BudgetFixture {
budget_id: "missing-user-default-budget",
max_budget: 11.5,
rpm_limit: 12,
tpm_limit: 1300,
}))]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn missing_end_user_maps_to_defaults(
#[case] default_budget: Option<BudgetFixture>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
if let Some(budget) = default_budget {
insert_budget(&mut tx, budget).await?;
}
let actual = find_end_user(
&mut *tx,
"missing-end-user",
default_budget.map(|budget| budget.budget_id),
None,
)
.await?;
assert_eq!(
actual,
EndUser {
spend: None,
blocked: false,
max_budget: default_budget.map(|budget| budget.max_budget),
limits: RateLimits {
rpm: default_budget.map(|budget| budget.rpm_limit),
tpm: default_budget.map(|budget| budget.tpm_limit),
},
}
);
Ok(())
}
#[rstest]
#[case::row_budget_precedes_default(
"row-budget-priority-user",
Some(BudgetFixture {
budget_id: "row-budget-priority-row",
max_budget: 10.0,
rpm_limit: 101,
tpm_limit: 1001,
}),
BudgetFixture {
budget_id: "row-budget-priority-default",
max_budget: 20.0,
rpm_limit: 201,
tpm_limit: 2001,
},
None,
Some(BudgetFixture {
budget_id: "row-budget-priority-row",
max_budget: 10.0,
rpm_limit: 101,
tpm_limit: 1001,
}),
false
)]
#[case::default_applies_without_row_budget(
"default-budget-user",
None,
BudgetFixture {
budget_id: "default-budget-fallback",
max_budget: 30.0,
rpm_limit: 301,
tpm_limit: 3001,
},
None,
Some(BudgetFixture {
budget_id: "default-budget-fallback",
max_budget: 30.0,
rpm_limit: 301,
tpm_limit: 3001,
}),
false
)]
#[case::assigned_budget_precedes_key_when_all_ids_differ(
"override-priority-user",
Some(BudgetFixture {
budget_id: "override-priority-row",
max_budget: 40.0,
rpm_limit: 401,
tpm_limit: 4001,
}),
BudgetFixture {
budget_id: "override-priority-default",
max_budget: 50.0,
rpm_limit: 501,
tpm_limit: 5001,
},
Some(BudgetFixture {
budget_id: "override-priority-override",
max_budget: 60.0,
rpm_limit: 601,
tpm_limit: 6001,
}),
Some(BudgetFixture {
budget_id: "override-priority-row",
max_budget: 40.0,
rpm_limit: 401,
tpm_limit: 4001,
}),
true
)]
#[case::missing_key_budget_keeps_assigned_budget(
"dangling-override-user",
Some(BudgetFixture {
budget_id: "dangling-override-row",
max_budget: 70.0,
rpm_limit: 701,
tpm_limit: 7001,
}),
BudgetFixture {
budget_id: "dangling-override-default",
max_budget: 80.0,
rpm_limit: 801,
tpm_limit: 8001,
},
Some(BudgetFixture {
budget_id: "dangling-override-missing",
max_budget: 90.0,
rpm_limit: 901,
tpm_limit: 9001,
}),
Some(BudgetFixture {
budget_id: "dangling-override-row",
max_budget: 70.0,
rpm_limit: 701,
tpm_limit: 7001,
}),
false
)]
#[case::key_budget_precedes_proxy_default(
"key-default-user",
None,
BudgetFixture {
budget_id: "key-default-proxy",
max_budget: 80.0,
rpm_limit: 801,
tpm_limit: 8001,
},
Some(BudgetFixture {
budget_id: "key-default-key",
max_budget: 90.0,
rpm_limit: 901,
tpm_limit: 9001,
}),
Some(BudgetFixture {
budget_id: "key-default-key",
max_budget: 90.0,
rpm_limit: 901,
tpm_limit: 9001,
}),
true
)]
#[case::missing_key_budget_falls_back_to_proxy_default(
"missing-key-default-user",
None,
BudgetFixture {
budget_id: "missing-key-default-proxy",
max_budget: 80.0,
rpm_limit: 801,
tpm_limit: 8001,
},
Some(BudgetFixture {
budget_id: "missing-key-default-key",
max_budget: 90.0,
rpm_limit: 901,
tpm_limit: 9001,
}),
Some(BudgetFixture {
budget_id: "missing-key-default-proxy",
max_budget: 80.0,
rpm_limit: 801,
tpm_limit: 8001,
}),
false
)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn budget_resolution_matches_python_precedence(
#[case] user_id: &'static str,
#[case] user_budget: Option<BudgetFixture>,
#[case] default_budget: BudgetFixture,
#[case] override_budget: Option<BudgetFixture>,
#[case] expected_budget: Option<BudgetFixture>,
#[case] insert_override_budget: bool,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
insert_budget(&mut tx, default_budget).await?;
if let Some(budget) = user_budget {
insert_budget(&mut tx, budget).await?;
}
if let (Some(budget), true) = (override_budget, insert_override_budget) {
insert_budget(&mut tx, budget).await?;
}
insert_end_user(
&mut tx,
user_id,
7.5,
true,
user_budget.map(|budget| budget.budget_id),
)
.await?;
let actual = find_end_user(
&mut *tx,
user_id,
Some(default_budget.budget_id),
override_budget.map(|budget| budget.budget_id),
)
.await?;
assert_eq!(
actual,
EndUser {
spend: Some(7.5),
blocked: true,
max_budget: expected_budget.map(|budget| budget.max_budget),
limits: RateLimits {
rpm: expected_budget.map(|budget| budget.rpm_limit),
tpm: expected_budget.map(|budget| budget.tpm_limit),
},
}
);
Ok(())
}
}

View file

@ -0,0 +1,79 @@
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("database query failed")]
Query(#[from] sqlx::Error),
#[error("spend batch has {ids} ids and {costs} costs")]
InvalidSpendBatch { ids: usize, costs: usize },
#[error("invalid spend timestamp")]
InvalidTimestamp(#[from] chrono::ParseError),
}
impl Error {
pub fn is_retryable(&self) -> bool {
match self {
Self::Query(sqlx::Error::Database(error)) => {
matches!(
error.code().as_deref(),
Some("40P01" | "55P03" | "40001" | "53300")
)
}
Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true,
_ => false,
}
}
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use rstest::rstest;
use sqlx::PgPool;
use super::Error;
use crate::test_support::database_pool;
#[rstest]
#[case::closed_pool(sqlx::Error::PoolClosed, false)]
#[case::pool_timeout(sqlx::Error::PoolTimedOut, true)]
#[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)]
#[case::missing_row(sqlx::Error::RowNotFound, false)]
fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) {
assert_eq!(Error::from(error).is_retryable(), retryable);
}
#[rstest]
#[case::connection_capacity("53300", true)]
#[case::deadlock("40P01", true)]
#[case::lock_unavailable("55P03", true)]
#[case::serialization_failure("40001", true)]
#[case::unique_violation("23505", false)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn classifies_server_errors(
#[case] code: &str,
#[case] retryable: bool,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test query must raise the requested SQLSTATE"
)]
let error = sqlx::query(sqlx::AssertSqlSafe(format!(
"DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$"
)))
.execute(&mut *tx)
.await
.expect_err("PostgreSQL must raise the requested error");
assert_eq!(
error.as_database_error().and_then(|error| error.code()),
Some(code.into())
);
assert_eq!(Error::from(error).is_retryable(), retryable);
tx.rollback().await?;
Ok(())
}
}

View file

@ -1,3 +1,26 @@
mod failure;
macro_rules_attribute::attribute_alias! {
#[apply(db_input)] =
#[derive(Clone, Debug, PartialEq)];
#[apply(db_row)] =
#[derive(Clone, Debug, PartialEq, sqlx::FromRow)];
}
mod daily_spend;
mod end_user;
mod error;
mod failure;
mod spend;
#[cfg(test)]
mod test_support;
pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow};
pub use end_user::{EndUser, RateLimits, find_end_user};
pub use error::Error;
pub use failure::DbFailure;
pub use spend::{
DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend,
add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend,
begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend,
upsert_daily_team_spend, upsert_daily_user_spend,
};

View file

@ -0,0 +1,584 @@
use sqlx::PgExecutor;
use super::{DailySpend, upsert_daily_team_spend, upsert_daily_user_spend};
use crate::Error;
pub async fn upsert_daily_end_user_spend(
executor: impl PgExecutor<'_>,
rows: &[DailySpend],
) -> Result<(), Error> {
if rows.is_empty() {
return Ok(());
}
macro_rules! column {
($field:ident) => {
rows.iter()
.map(|row| row.$field.clone())
.collect::<Vec<_>>()
};
}
sqlx::query_file!(
"queries/spend/upsert_daily_end_user_spend.sql",
&column!(entity_id),
&column!(date),
&column!(api_key),
&column!(model),
&column!(custom_llm_provider),
&column!(endpoint),
&column!(model_group) as &[Option<String>],
&column!(prompt_tokens),
&column!(completion_tokens),
&column!(api_requests),
&column!(successful_requests),
&column!(failed_requests),
&column!(cache_read_input_tokens),
&column!(cache_creation_input_tokens),
&column!(total_response_time_ms),
&column!(timed_requests),
&column!(spend),
&column!(mcp_namespaced_tool_name),
)
.execute(executor)
.await?;
Ok(())
}
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, strum::VariantArray)]
pub enum DailyTable {
User,
Team,
EndUser,
}
pub async fn upsert_daily_spend(
executor: impl PgExecutor<'_>,
table: DailyTable,
rows: &[DailySpend],
) -> Result<(), Error> {
match table {
DailyTable::User => upsert_daily_user_spend(executor, rows).await,
DailyTable::Team => upsert_daily_team_spend(executor, rows).await,
DailyTable::EndUser => upsert_daily_end_user_spend(executor, rows).await,
}
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{DailySpend, DailyTable, upsert_daily_end_user_spend, upsert_daily_spend};
use crate::{DailyEndUserRow, test_support::database_pool};
use rstest::rstest;
use sqlx::{PgPool, types::chrono::NaiveDateTime};
#[rstest]
#[case::full_keys(
"daily-end-user-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
Some("first-group")
)]
#[case::initial_model_group_none(
"daily-end-user-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
None
)]
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_end_user_spend_upsert_preserves_keys_and_accumulates(
#[case] end_user_id: &'static str,
#[case] api_key: &'static str,
#[case] model: &'static str,
#[case] custom_llm_provider: &'static str,
#[case] endpoint: &'static str,
#[case] first_model_group: Option<&'static str>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
)]
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
.execute(&mut *tx)
.await?;
let first_batch = [
DailySpend {
entity_id: end_user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: first_model_group.map(str::to_owned),
prompt_tokens: 1,
completion_tokens: 2,
api_requests: 3,
successful_requests: 4,
failed_requests: 5,
cache_read_input_tokens: 6,
cache_creation_input_tokens: 7,
total_response_time_ms: 8,
timed_requests: 9,
spend: 0.125,
},
DailySpend {
entity_id: end_user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: "endpoint-sentinel".to_owned(),
model_group: Some("sentinel-endpoint-group".to_owned()),
prompt_tokens: 101,
completion_tokens: 102,
api_requests: 103,
successful_requests: 104,
failed_requests: 105,
cache_read_input_tokens: 106,
cache_creation_input_tokens: 107,
total_response_time_ms: 108,
timed_requests: 109,
spend: 1.25,
},
DailySpend {
entity_id: end_user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: "api-key-sentinel".to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-key-group".to_owned()),
prompt_tokens: 201,
completion_tokens: 202,
api_requests: 203,
successful_requests: 204,
failed_requests: 205,
cache_read_input_tokens: 206,
cache_creation_input_tokens: 207,
total_response_time_ms: 208,
timed_requests: 209,
spend: 2.5,
},
DailySpend {
entity_id: "entity-sentinel".to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-entity-group".to_owned()),
prompt_tokens: 301,
completion_tokens: 302,
api_requests: 303,
successful_requests: 304,
failed_requests: 305,
cache_read_input_tokens: 306,
cache_creation_input_tokens: 307,
total_response_time_ms: 308,
timed_requests: 309,
spend: 3.75,
},
];
upsert_daily_end_user_spend(&mut *tx, &first_batch).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the inserted row identity before conflict handling"
)]
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
r#"SELECT id, created_at
FROM "LiteLLM_DailyEndUserSpend"
WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4
AND custom_llm_provider = $5 AND endpoint = $6"#,
)
.bind(end_user_id)
.bind("2026-01-15")
.bind(api_key)
.bind(model)
.bind(custom_llm_provider)
.bind(endpoint)
.fetch_one(&mut *tx)
.await?;
assert!(!initial_id.is_empty());
#[expect(
clippy::disallowed_methods,
reason = "The integration test seeds updated_at before exercising the conflict update"
)]
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
r#"UPDATE "LiteLLM_DailyEndUserSpend"
SET updated_at = TIMESTAMP '2000-01-01'
WHERE id = $1
RETURNING updated_at"#,
)
.bind(&initial_id)
.fetch_one(&mut *tx)
.await?;
let increment = [DailySpend {
entity_id: end_user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("second-group".to_owned()),
prompt_tokens: 10,
completion_tokens: 20,
api_requests: 30,
successful_requests: 40,
failed_requests: 50,
cache_read_input_tokens: 60,
cache_creation_input_tokens: 70,
total_response_time_ms: 80,
timed_requests: 90,
spend: 0.625,
}];
upsert_daily_end_user_spend(&mut *tx, &increment).await?;
upsert_daily_end_user_spend(&mut *tx, &[]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the transaction timestamp for assertions"
)]
let expected_updated_at: NaiveDateTime =
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
.fetch_one(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads all rows to verify attribution and accumulation"
)]
let rows: Vec<DailyEndUserRow> = sqlx::query_as(
r#"SELECT id, end_user_id, api_key, model, custom_llm_provider, endpoint,
model_group, mcp_namespaced_tool_name, prompt_tokens,
completion_tokens, api_requests, successful_requests,
failed_requests, cache_read_input_tokens,
cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, date, created_at, updated_at
FROM "LiteLLM_DailyEndUserSpend"
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
AND mcp_namespaced_tool_name = '' AND end_user_id = ANY($4)
ORDER BY CASE
WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
WHEN end_user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
WHEN end_user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
ELSE 4
END"#,
)
.bind("2026-01-15")
.bind(model)
.bind(custom_llm_provider)
.bind(vec![end_user_id, "entity-sentinel"])
.bind(end_user_id)
.bind(api_key)
.bind(endpoint)
.bind("api-key-sentinel")
.bind("endpoint-sentinel")
.bind("entity-sentinel")
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 4);
assert_eq!(rows[0].id, initial_id);
assert_eq!(rows[0].end_user_id.as_deref(), Some(end_user_id));
assert_eq!(rows[0].api_key, api_key);
assert_eq!(rows[0].model.as_deref(), Some(model));
assert_eq!(
rows[0].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[0].prompt_tokens,
rows[0].completion_tokens,
rows[0].api_requests,
rows[0].successful_requests,
rows[0].failed_requests,
rows[0].cache_read_input_tokens,
rows[0].cache_creation_input_tokens,
rows[0].total_response_time_ms,
rows[0].timed_requests,
rows[0].spend
),
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
);
assert_eq!(rows[0].date, "2026-01-15");
assert_eq!(rows[0].created_at, initial_created_at);
assert!(rows[0].updated_at > seeded_updated_at);
assert_eq!(rows[0].updated_at, expected_updated_at);
assert!(!rows[1].id.is_empty());
assert_eq!(rows[1].end_user_id.as_deref(), Some(end_user_id));
assert_eq!(rows[1].api_key, api_key);
assert_eq!(rows[1].model.as_deref(), Some(model));
assert_eq!(
rows[1].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
assert_eq!(
rows[1].model_group.as_deref(),
Some("sentinel-endpoint-group")
);
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[1].prompt_tokens,
rows[1].completion_tokens,
rows[1].api_requests,
rows[1].successful_requests,
rows[1].failed_requests,
rows[1].cache_read_input_tokens,
rows[1].cache_creation_input_tokens,
rows[1].total_response_time_ms,
rows[1].timed_requests,
rows[1].spend
),
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
);
assert_eq!(rows[1].date, "2026-01-15");
assert!(rows[1].updated_at > seeded_updated_at);
assert_eq!(rows[1].updated_at, expected_updated_at);
assert!(!rows[2].id.is_empty());
assert_eq!(rows[2].end_user_id.as_deref(), Some(end_user_id));
assert_eq!(rows[2].api_key, "api-key-sentinel");
assert_eq!(rows[2].model.as_deref(), Some(model));
assert_eq!(
rows[2].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[2].prompt_tokens,
rows[2].completion_tokens,
rows[2].api_requests,
rows[2].successful_requests,
rows[2].failed_requests,
rows[2].cache_read_input_tokens,
rows[2].cache_creation_input_tokens,
rows[2].total_response_time_ms,
rows[2].timed_requests,
rows[2].spend
),
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
);
assert_eq!(rows[2].date, "2026-01-15");
assert!(rows[2].updated_at > seeded_updated_at);
assert_eq!(rows[2].updated_at, expected_updated_at);
assert!(!rows[3].id.is_empty());
assert_eq!(rows[3].end_user_id.as_deref(), Some("entity-sentinel"));
assert_eq!(rows[3].api_key, api_key);
assert_eq!(rows[3].model.as_deref(), Some(model));
assert_eq!(
rows[3].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
assert_eq!(
rows[3].model_group.as_deref(),
Some("sentinel-entity-group")
);
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[3].prompt_tokens,
rows[3].completion_tokens,
rows[3].api_requests,
rows[3].successful_requests,
rows[3].failed_requests,
rows[3].cache_read_input_tokens,
rows[3].cache_creation_input_tokens,
rows[3].total_response_time_ms,
rows[3].timed_requests,
rows[3].spend
),
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
);
assert_eq!(rows[3].date, "2026-01-15");
assert!(rows[3].updated_at > seeded_updated_at);
assert_eq!(rows[3].updated_at, expected_updated_at);
Ok(())
}
#[rstest]
#[case::distinct_tools("server/tool-a", "server/tool-b")]
#[case::empty_and_named_tool("", "server/tool-b")]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_end_user_spend_preserves_tool_attribution(
#[case] first_tool: &str,
#[case] second_tool: &str,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
let first = DailySpend {
entity_id: "daily-end-user-tools".to_owned(),
date: "2026-02-21".to_owned(),
api_key: "tool-key".to_owned(),
model: "tool-model".to_owned(),
custom_llm_provider: "tool-provider".to_owned(),
endpoint: "/mcp".to_owned(),
mcp_namespaced_tool_name: first_tool.to_owned(),
prompt_tokens: 2,
spend: 0.25,
..Default::default()
};
let second = DailySpend {
mcp_namespaced_tool_name: second_tool.to_owned(),
prompt_tokens: 5,
spend: 0.5,
..first.clone()
};
upsert_daily_end_user_spend(&mut *tx, &[second, first.clone()]).await?;
upsert_daily_end_user_spend(&mut *tx, &[first]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads stored tool attribution and counters"
)]
let rows: Vec<DailyEndUserRow> = sqlx::query_as(
r#"SELECT * FROM "LiteLLM_DailyEndUserSpend"
WHERE end_user_id = $1 AND date = $2 AND api_key = $3
ORDER BY mcp_namespaced_tool_name"#,
)
.bind("daily-end-user-tools")
.bind("2026-02-21")
.bind("tool-key")
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 2);
assert_eq!(
rows[0].mcp_namespaced_tool_name.as_deref(),
Some(first_tool)
);
assert_eq!(
rows[1].mcp_namespaced_tool_name.as_deref(),
Some(second_tool)
);
assert_eq!((rows[0].prompt_tokens, rows[0].spend), (4, 0.5));
assert_eq!((rows[1].prompt_tokens, rows[1].spend), (5, 0.5));
assert_ne!(rows[0].id, rows[1].id);
Ok(())
}
#[rstest]
#[case::user(DailyTable::User, "daily-dispatch-user")]
#[case::team(DailyTable::Team, "daily-dispatch-team")]
#[case::end_user(DailyTable::EndUser, "daily-dispatch-end-user")]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_spend_dispatches_to_selected_table(
#[case] table: DailyTable,
#[case] entity_id: &'static str,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
let api_key = format!("{entity_id}-key");
let rows = [DailySpend {
entity_id: entity_id.to_owned(),
date: "2026-02-20".to_owned(),
api_key: api_key.clone(),
model: "dispatcher-model".to_owned(),
custom_llm_provider: "dispatcher-provider".to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: "/dispatcher".to_owned(),
model_group: Some("dispatcher-group".to_owned()),
prompt_tokens: 1,
completion_tokens: 2,
api_requests: 3,
successful_requests: 4,
failed_requests: 5,
cache_read_input_tokens: 6,
cache_creation_input_tokens: 7,
total_response_time_ms: 8,
timed_requests: 9,
spend: 0.125,
}];
upsert_daily_spend(&mut *tx, table, &rows).await?;
let count: i64 = match table {
DailyTable::User =>
{
#[expect(
clippy::disallowed_methods,
reason = "The integration test checks the user-table dispatcher route"
)]
sqlx::query_scalar(
r#"SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend"
WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
)
.bind(entity_id)
.bind("2026-02-20")
.bind(&api_key)
.bind("dispatcher-model")
.fetch_one(&mut *tx)
.await?
}
DailyTable::Team =>
{
#[expect(
clippy::disallowed_methods,
reason = "The integration test checks the team-table dispatcher route"
)]
sqlx::query_scalar(
r#"SELECT COUNT(*) FROM "LiteLLM_DailyTeamSpend"
WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
)
.bind(entity_id)
.bind("2026-02-20")
.bind(&api_key)
.bind("dispatcher-model")
.fetch_one(&mut *tx)
.await?
}
DailyTable::EndUser =>
{
#[expect(
clippy::disallowed_methods,
reason = "The integration test checks the end-user-table dispatcher route"
)]
sqlx::query_scalar(
r#"SELECT COUNT(*) FROM "LiteLLM_DailyEndUserSpend"
WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
)
.bind(entity_id)
.bind("2026-02-20")
.bind(&api_key)
.bind("dispatcher-model")
.fetch_one(&mut *tx)
.await?
}
};
assert_eq!(count, 1);
Ok(())
}
}

View file

@ -0,0 +1,485 @@
use sqlx::PgExecutor;
use super::DailySpend;
use crate::Error;
pub async fn upsert_daily_team_spend(
executor: impl PgExecutor<'_>,
rows: &[DailySpend],
) -> Result<(), Error> {
if rows.is_empty() {
return Ok(());
}
macro_rules! column {
($field:ident) => {
rows.iter()
.map(|row| row.$field.clone())
.collect::<Vec<_>>()
};
}
sqlx::query_file!(
"queries/spend/upsert_daily_team_spend.sql",
&column!(entity_id),
&column!(date),
&column!(api_key),
&column!(model),
&column!(custom_llm_provider),
&column!(endpoint),
&column!(model_group) as &[Option<String>],
&column!(prompt_tokens),
&column!(completion_tokens),
&column!(api_requests),
&column!(successful_requests),
&column!(failed_requests),
&column!(cache_read_input_tokens),
&column!(cache_creation_input_tokens),
&column!(total_response_time_ms),
&column!(timed_requests),
&column!(spend),
&column!(mcp_namespaced_tool_name),
)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{DailySpend, upsert_daily_team_spend};
use crate::{DailyTeamRow, test_support::database_pool};
use rstest::rstest;
use sqlx::{PgPool, types::chrono::NaiveDateTime};
#[rstest]
#[case::distinct_tools("server.fetch", "server.search")]
#[case::default_and_named_tool("", "server.search")]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_team_spend_separates_tools_and_accumulates(
#[case] first_tool: &str,
#[case] second_tool: &str,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
let first = DailySpend {
entity_id: "daily-team-tools".to_owned(),
date: "2026-01-15".to_owned(),
api_key: "daily-key-tools".to_owned(),
model: "daily-model".to_owned(),
custom_llm_provider: "daily-provider".to_owned(),
endpoint: "/mcp".to_owned(),
mcp_namespaced_tool_name: first_tool.to_owned(),
api_requests: 1,
spend: 0.125,
..Default::default()
};
let second = DailySpend {
mcp_namespaced_tool_name: second_tool.to_owned(),
api_requests: 2,
spend: 0.25,
..first.clone()
};
upsert_daily_team_spend(&mut *tx, &[second.clone(), first.clone()]).await?;
upsert_daily_team_spend(
&mut *tx,
&[
DailySpend {
api_requests: 3,
spend: 0.5,
..first
},
DailySpend {
api_requests: 5,
spend: 1.0,
..second
},
],
)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads stored tool attribution and counters"
)]
let rows = sqlx::query_as::<_, DailyTeamRow>(
r#"SELECT * FROM "LiteLLM_DailyTeamSpend"
WHERE team_id = $1 AND date = $2
ORDER BY mcp_namespaced_tool_name"#,
)
.bind("daily-team-tools")
.bind("2026-01-15")
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 2);
assert_eq!(
rows[0].mcp_namespaced_tool_name.as_deref(),
Some(first_tool)
);
assert_eq!(
rows[1].mcp_namespaced_tool_name.as_deref(),
Some(second_tool)
);
assert_eq!((rows[0].api_requests, rows[0].spend), (4, 0.625));
assert_eq!((rows[1].api_requests, rows[1].spend), (7, 1.25));
Ok(())
}
#[rstest]
#[case::full_keys(
"daily-team-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
Some("first-group")
)]
#[case::initial_model_group_none(
"daily-team-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
None
)]
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_team_spend_upsert_preserves_keys_and_accumulates(
#[case] team_id: &'static str,
#[case] api_key: &'static str,
#[case] model: &'static str,
#[case] custom_llm_provider: &'static str,
#[case] endpoint: &'static str,
#[case] first_model_group: Option<&'static str>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
)]
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
.execute(&mut *tx)
.await?;
let first_batch = [
DailySpend {
entity_id: team_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: first_model_group.map(str::to_owned),
prompt_tokens: 1,
completion_tokens: 2,
api_requests: 3,
successful_requests: 4,
failed_requests: 5,
cache_read_input_tokens: 6,
cache_creation_input_tokens: 7,
total_response_time_ms: 8,
timed_requests: 9,
spend: 0.125,
},
DailySpend {
entity_id: team_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: "endpoint-sentinel".to_owned(),
model_group: Some("sentinel-endpoint-group".to_owned()),
prompt_tokens: 101,
completion_tokens: 102,
api_requests: 103,
successful_requests: 104,
failed_requests: 105,
cache_read_input_tokens: 106,
cache_creation_input_tokens: 107,
total_response_time_ms: 108,
timed_requests: 109,
spend: 1.25,
},
DailySpend {
entity_id: team_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: "api-key-sentinel".to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-key-group".to_owned()),
prompt_tokens: 201,
completion_tokens: 202,
api_requests: 203,
successful_requests: 204,
failed_requests: 205,
cache_read_input_tokens: 206,
cache_creation_input_tokens: 207,
total_response_time_ms: 208,
timed_requests: 209,
spend: 2.5,
},
DailySpend {
entity_id: "entity-sentinel".to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-entity-group".to_owned()),
prompt_tokens: 301,
completion_tokens: 302,
api_requests: 303,
successful_requests: 304,
failed_requests: 305,
cache_read_input_tokens: 306,
cache_creation_input_tokens: 307,
total_response_time_ms: 308,
timed_requests: 309,
spend: 3.75,
},
];
upsert_daily_team_spend(&mut *tx, &first_batch).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the inserted row identity before conflict handling"
)]
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
r#"SELECT id, created_at
FROM "LiteLLM_DailyTeamSpend"
WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4
AND custom_llm_provider = $5 AND endpoint = $6"#,
)
.bind(team_id)
.bind("2026-01-15")
.bind(api_key)
.bind(model)
.bind(custom_llm_provider)
.bind(endpoint)
.fetch_one(&mut *tx)
.await?;
assert!(!initial_id.is_empty());
#[expect(
clippy::disallowed_methods,
reason = "The integration test seeds updated_at before exercising the conflict update"
)]
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
r#"UPDATE "LiteLLM_DailyTeamSpend"
SET updated_at = TIMESTAMP '2000-01-01'
WHERE id = $1
RETURNING updated_at"#,
)
.bind(&initial_id)
.fetch_one(&mut *tx)
.await?;
let increment = [DailySpend {
entity_id: team_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("second-group".to_owned()),
prompt_tokens: 10,
completion_tokens: 20,
api_requests: 30,
successful_requests: 40,
failed_requests: 50,
cache_read_input_tokens: 60,
cache_creation_input_tokens: 70,
total_response_time_ms: 80,
timed_requests: 90,
spend: 0.625,
}];
upsert_daily_team_spend(&mut *tx, &increment).await?;
upsert_daily_team_spend(&mut *tx, &[]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the transaction timestamp for assertions"
)]
let expected_updated_at: NaiveDateTime =
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
.fetch_one(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads all rows to verify attribution and accumulation"
)]
let rows: Vec<DailyTeamRow> = sqlx::query_as(
r#"SELECT id, team_id, api_key, model, custom_llm_provider, endpoint,
model_group, mcp_namespaced_tool_name, prompt_tokens,
completion_tokens, api_requests, successful_requests,
failed_requests, cache_read_input_tokens,
cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, date, created_at, updated_at
FROM "LiteLLM_DailyTeamSpend"
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
AND mcp_namespaced_tool_name = '' AND team_id = ANY($4)
ORDER BY CASE
WHEN team_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
WHEN team_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
WHEN team_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
WHEN team_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
ELSE 4
END"#,
)
.bind("2026-01-15")
.bind(model)
.bind(custom_llm_provider)
.bind(vec![team_id, "entity-sentinel"])
.bind(team_id)
.bind(api_key)
.bind(endpoint)
.bind("api-key-sentinel")
.bind("endpoint-sentinel")
.bind("entity-sentinel")
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 4);
assert_eq!(rows[0].id, initial_id);
assert_eq!(rows[0].team_id.as_deref(), Some(team_id));
assert_eq!(rows[0].api_key, api_key);
assert_eq!(rows[0].model.as_deref(), Some(model));
assert_eq!(
rows[0].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[0].prompt_tokens,
rows[0].completion_tokens,
rows[0].api_requests,
rows[0].successful_requests,
rows[0].failed_requests,
rows[0].cache_read_input_tokens,
rows[0].cache_creation_input_tokens,
rows[0].total_response_time_ms,
rows[0].timed_requests,
rows[0].spend
),
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
);
assert_eq!(rows[0].date, "2026-01-15");
assert_eq!(rows[0].created_at, initial_created_at);
assert!(rows[0].updated_at > seeded_updated_at);
assert_eq!(rows[0].updated_at, expected_updated_at);
assert!(!rows[1].id.is_empty());
assert_eq!(rows[1].team_id.as_deref(), Some(team_id));
assert_eq!(rows[1].api_key, api_key);
assert_eq!(rows[1].model.as_deref(), Some(model));
assert_eq!(
rows[1].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
assert_eq!(
rows[1].model_group.as_deref(),
Some("sentinel-endpoint-group")
);
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[1].prompt_tokens,
rows[1].completion_tokens,
rows[1].api_requests,
rows[1].successful_requests,
rows[1].failed_requests,
rows[1].cache_read_input_tokens,
rows[1].cache_creation_input_tokens,
rows[1].total_response_time_ms,
rows[1].timed_requests,
rows[1].spend
),
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
);
assert_eq!(rows[1].date, "2026-01-15");
assert!(rows[1].updated_at > seeded_updated_at);
assert_eq!(rows[1].updated_at, expected_updated_at);
assert!(!rows[2].id.is_empty());
assert_eq!(rows[2].team_id.as_deref(), Some(team_id));
assert_eq!(rows[2].api_key, "api-key-sentinel");
assert_eq!(rows[2].model.as_deref(), Some(model));
assert_eq!(
rows[2].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[2].prompt_tokens,
rows[2].completion_tokens,
rows[2].api_requests,
rows[2].successful_requests,
rows[2].failed_requests,
rows[2].cache_read_input_tokens,
rows[2].cache_creation_input_tokens,
rows[2].total_response_time_ms,
rows[2].timed_requests,
rows[2].spend
),
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
);
assert_eq!(rows[2].date, "2026-01-15");
assert!(rows[2].updated_at > seeded_updated_at);
assert_eq!(rows[2].updated_at, expected_updated_at);
assert!(!rows[3].id.is_empty());
assert_eq!(rows[3].team_id.as_deref(), Some("entity-sentinel"));
assert_eq!(rows[3].api_key, api_key);
assert_eq!(rows[3].model.as_deref(), Some(model));
assert_eq!(
rows[3].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
assert_eq!(
rows[3].model_group.as_deref(),
Some("sentinel-entity-group")
);
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[3].prompt_tokens,
rows[3].completion_tokens,
rows[3].api_requests,
rows[3].successful_requests,
rows[3].failed_requests,
rows[3].cache_read_input_tokens,
rows[3].cache_creation_input_tokens,
rows[3].total_response_time_ms,
rows[3].timed_requests,
rows[3].spend
),
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
);
assert_eq!(rows[3].date, "2026-01-15");
assert!(rows[3].updated_at > seeded_updated_at);
assert_eq!(rows[3].updated_at, expected_updated_at);
Ok(())
}
}

View file

@ -0,0 +1,500 @@
use sqlx::PgExecutor;
use crate::Error;
#[macro_rules_attribute::apply(crate::db_input)]
#[derive(Default)]
pub struct DailySpend {
pub entity_id: String,
pub date: String,
pub api_key: String,
pub model: String,
pub custom_llm_provider: String,
pub mcp_namespaced_tool_name: String,
pub endpoint: String,
pub model_group: Option<String>,
pub prompt_tokens: i64,
pub completion_tokens: i64,
pub api_requests: i64,
pub successful_requests: i64,
pub failed_requests: i64,
pub cache_read_input_tokens: i64,
pub cache_creation_input_tokens: i64,
pub total_response_time_ms: i64,
pub timed_requests: i64,
pub spend: f64,
}
pub async fn upsert_daily_user_spend(
executor: impl PgExecutor<'_>,
rows: &[DailySpend],
) -> Result<(), Error> {
if rows.is_empty() {
return Ok(());
}
macro_rules! column {
($field:ident) => {
rows.iter()
.map(|row| row.$field.clone())
.collect::<Vec<_>>()
};
}
sqlx::query_file!(
"queries/spend/upsert_daily_user_spend.sql",
&column!(entity_id),
&column!(date),
&column!(api_key),
&column!(model),
&column!(custom_llm_provider),
&column!(endpoint),
&column!(model_group) as &[Option<String>],
&column!(prompt_tokens),
&column!(completion_tokens),
&column!(api_requests),
&column!(successful_requests),
&column!(failed_requests),
&column!(cache_read_input_tokens),
&column!(cache_creation_input_tokens),
&column!(total_response_time_ms),
&column!(timed_requests),
&column!(spend),
&column!(mcp_namespaced_tool_name),
)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{DailySpend, upsert_daily_user_spend};
use crate::{DailyUserRow, test_support::database_pool};
use rstest::rstest;
use sqlx::{PgPool, types::chrono::NaiveDateTime};
#[rstest]
#[case::distinct_tools("server/search", "server/write")]
#[case::default_empty_tool("", "server/search")]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_user_spend_separates_tools_and_accumulates(
#[case] first_tool: &str,
#[case] second_tool: &str,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
let first = DailySpend {
entity_id: format!("daily-user-tools-{first_tool}"),
date: "2026-01-15".to_owned(),
api_key: "tool-key".to_owned(),
model: "tool-model".to_owned(),
custom_llm_provider: "tool-provider".to_owned(),
endpoint: "/mcp".to_owned(),
model_group: Some("tool-group".to_owned()),
mcp_namespaced_tool_name: first_tool.to_owned(),
prompt_tokens: 3,
api_requests: 1,
spend: 0.25,
..Default::default()
};
let second = DailySpend {
mcp_namespaced_tool_name: second_tool.to_owned(),
prompt_tokens: 7,
api_requests: 2,
spend: 0.5,
..first.clone()
};
upsert_daily_user_spend(&mut *tx, &[second.clone(), first.clone()]).await?;
upsert_daily_user_spend(&mut *tx, &[first.clone(), second]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads stored rows to verify tool attribution"
)]
let rows: Vec<DailyUserRow> = sqlx::query_as(
r#"SELECT * FROM "LiteLLM_DailyUserSpend"
WHERE user_id = $1
ORDER BY mcp_namespaced_tool_name, endpoint"#,
)
.bind(&first.entity_id)
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 2);
assert_eq!(
rows[0].mcp_namespaced_tool_name.as_deref(),
Some(first_tool)
);
assert_eq!(
rows[1].mcp_namespaced_tool_name.as_deref(),
Some(second_tool)
);
assert_eq!(
(rows[0].prompt_tokens, rows[0].api_requests, rows[0].spend),
(6, 2, 0.5)
);
assert_eq!(
(rows[1].prompt_tokens, rows[1].api_requests, rows[1].spend),
(14, 4, 1.0)
);
Ok(())
}
#[rstest]
#[case::full_keys(
"daily-user-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
Some("first-group")
)]
#[case::initial_model_group_none(
"daily-user-base",
"daily-key-base",
"daily-model",
"daily-provider",
"/chat/completions",
None
)]
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn daily_user_spend_upsert_preserves_keys_and_accumulates(
#[case] user_id: &'static str,
#[case] api_key: &'static str,
#[case] model: &'static str,
#[case] custom_llm_provider: &'static str,
#[case] endpoint: &'static str,
#[case] first_model_group: Option<&'static str>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
)]
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
.execute(&mut *tx)
.await?;
let first_batch = [
DailySpend {
entity_id: user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: first_model_group.map(str::to_owned),
prompt_tokens: 1,
completion_tokens: 2,
api_requests: 3,
successful_requests: 4,
failed_requests: 5,
cache_read_input_tokens: 6,
cache_creation_input_tokens: 7,
total_response_time_ms: 8,
timed_requests: 9,
spend: 0.125,
},
DailySpend {
entity_id: user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: "endpoint-sentinel".to_owned(),
model_group: Some("sentinel-endpoint-group".to_owned()),
prompt_tokens: 101,
completion_tokens: 102,
api_requests: 103,
successful_requests: 104,
failed_requests: 105,
cache_read_input_tokens: 106,
cache_creation_input_tokens: 107,
total_response_time_ms: 108,
timed_requests: 109,
spend: 1.25,
},
DailySpend {
entity_id: user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: "api-key-sentinel".to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-key-group".to_owned()),
prompt_tokens: 201,
completion_tokens: 202,
api_requests: 203,
successful_requests: 204,
failed_requests: 205,
cache_read_input_tokens: 206,
cache_creation_input_tokens: 207,
total_response_time_ms: 208,
timed_requests: 209,
spend: 2.5,
},
DailySpend {
entity_id: "entity-sentinel".to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("sentinel-entity-group".to_owned()),
prompt_tokens: 301,
completion_tokens: 302,
api_requests: 303,
successful_requests: 304,
failed_requests: 305,
cache_read_input_tokens: 306,
cache_creation_input_tokens: 307,
total_response_time_ms: 308,
timed_requests: 309,
spend: 3.75,
},
];
upsert_daily_user_spend(&mut *tx, &first_batch).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the inserted row identity before conflict handling"
)]
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
r#"SELECT id, created_at
FROM "LiteLLM_DailyUserSpend"
WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4
AND custom_llm_provider = $5 AND endpoint = $6"#,
)
.bind(user_id)
.bind("2026-01-15")
.bind(api_key)
.bind(model)
.bind(custom_llm_provider)
.bind(endpoint)
.fetch_one(&mut *tx)
.await?;
assert!(!initial_id.is_empty());
#[expect(
clippy::disallowed_methods,
reason = "The integration test seeds updated_at before exercising the conflict update"
)]
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
r#"UPDATE "LiteLLM_DailyUserSpend"
SET updated_at = TIMESTAMP '2000-01-01'
WHERE id = $1
RETURNING updated_at"#,
)
.bind(&initial_id)
.fetch_one(&mut *tx)
.await?;
let increment = [DailySpend {
entity_id: user_id.to_owned(),
date: "2026-01-15".to_owned(),
api_key: api_key.to_owned(),
model: model.to_owned(),
custom_llm_provider: custom_llm_provider.to_owned(),
mcp_namespaced_tool_name: String::new(),
endpoint: endpoint.to_owned(),
model_group: Some("second-group".to_owned()),
prompt_tokens: 10,
completion_tokens: 20,
api_requests: 30,
successful_requests: 40,
failed_requests: 50,
cache_read_input_tokens: 60,
cache_creation_input_tokens: 70,
total_response_time_ms: 80,
timed_requests: 90,
spend: 0.625,
}];
upsert_daily_user_spend(&mut *tx, &increment).await?;
upsert_daily_user_spend(&mut *tx, &[]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test captures the transaction timestamp for assertions"
)]
let expected_updated_at: NaiveDateTime =
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
.fetch_one(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "The integration test reads all rows to verify attribution and accumulation"
)]
let rows: Vec<DailyUserRow> = sqlx::query_as(
r#"SELECT id, user_id, api_key, model, custom_llm_provider, endpoint,
model_group, mcp_namespaced_tool_name, prompt_tokens,
completion_tokens, api_requests, successful_requests,
failed_requests, cache_read_input_tokens,
cache_creation_input_tokens, total_response_time_ms,
timed_requests, spend, date, created_at, updated_at
FROM "LiteLLM_DailyUserSpend"
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
AND mcp_namespaced_tool_name = '' AND user_id = ANY($4)
ORDER BY CASE
WHEN user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
WHEN user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
WHEN user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
WHEN user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
ELSE 4
END"#,
)
.bind("2026-01-15")
.bind(model)
.bind(custom_llm_provider)
.bind(vec![user_id, "entity-sentinel"])
.bind(user_id)
.bind(api_key)
.bind(endpoint)
.bind("api-key-sentinel")
.bind("endpoint-sentinel")
.bind("entity-sentinel")
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows.len(), 4);
assert_eq!(rows[0].id, initial_id);
assert_eq!(rows[0].user_id.as_deref(), Some(user_id));
assert_eq!(rows[0].api_key, api_key);
assert_eq!(rows[0].model.as_deref(), Some(model));
assert_eq!(
rows[0].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[0].prompt_tokens,
rows[0].completion_tokens,
rows[0].api_requests,
rows[0].successful_requests,
rows[0].failed_requests,
rows[0].cache_read_input_tokens,
rows[0].cache_creation_input_tokens,
rows[0].total_response_time_ms,
rows[0].timed_requests,
rows[0].spend
),
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
);
assert_eq!(rows[0].date, "2026-01-15");
assert_eq!(rows[0].created_at, initial_created_at);
assert!(rows[0].updated_at > seeded_updated_at);
assert_eq!(rows[0].updated_at, expected_updated_at);
assert!(!rows[1].id.is_empty());
assert_eq!(rows[1].user_id.as_deref(), Some(user_id));
assert_eq!(rows[1].api_key, api_key);
assert_eq!(rows[1].model.as_deref(), Some(model));
assert_eq!(
rows[1].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
assert_eq!(
rows[1].model_group.as_deref(),
Some("sentinel-endpoint-group")
);
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[1].prompt_tokens,
rows[1].completion_tokens,
rows[1].api_requests,
rows[1].successful_requests,
rows[1].failed_requests,
rows[1].cache_read_input_tokens,
rows[1].cache_creation_input_tokens,
rows[1].total_response_time_ms,
rows[1].timed_requests,
rows[1].spend
),
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
);
assert_eq!(rows[1].date, "2026-01-15");
assert!(rows[1].updated_at > seeded_updated_at);
assert_eq!(rows[1].updated_at, expected_updated_at);
assert!(!rows[2].id.is_empty());
assert_eq!(rows[2].user_id.as_deref(), Some(user_id));
assert_eq!(rows[2].api_key, "api-key-sentinel");
assert_eq!(rows[2].model.as_deref(), Some(model));
assert_eq!(
rows[2].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[2].prompt_tokens,
rows[2].completion_tokens,
rows[2].api_requests,
rows[2].successful_requests,
rows[2].failed_requests,
rows[2].cache_read_input_tokens,
rows[2].cache_creation_input_tokens,
rows[2].total_response_time_ms,
rows[2].timed_requests,
rows[2].spend
),
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
);
assert_eq!(rows[2].date, "2026-01-15");
assert!(rows[2].updated_at > seeded_updated_at);
assert_eq!(rows[2].updated_at, expected_updated_at);
assert!(!rows[3].id.is_empty());
assert_eq!(rows[3].user_id.as_deref(), Some("entity-sentinel"));
assert_eq!(rows[3].api_key, api_key);
assert_eq!(rows[3].model.as_deref(), Some(model));
assert_eq!(
rows[3].custom_llm_provider.as_deref(),
Some(custom_llm_provider)
);
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
assert_eq!(
rows[3].model_group.as_deref(),
Some("sentinel-entity-group")
);
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
assert_eq!(
(
rows[3].prompt_tokens,
rows[3].completion_tokens,
rows[3].api_requests,
rows[3].successful_requests,
rows[3].failed_requests,
rows[3].cache_read_input_tokens,
rows[3].cache_creation_input_tokens,
rows[3].total_response_time_ms,
rows[3].timed_requests,
rows[3].spend
),
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
);
assert_eq!(rows[3].date, "2026-01-15");
assert!(rows[3].updated_at > seeded_updated_at);
assert_eq!(rows[3].updated_at, expected_updated_at);
Ok(())
}
}

View file

@ -0,0 +1,197 @@
use sqlx::PgExecutor;
use super::EntitySpend;
use crate::Error;
pub async fn add_end_user_spend(
executor: impl PgExecutor<'_>,
spend: &EntitySpend,
) -> Result<(), Error> {
spend.validate()?;
if spend.is_empty() {
return Ok(());
}
sqlx::query_file!(
"queries/spend/add_end_user_spend.sql",
&spend.ids,
&spend.costs
)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{EntitySpend, add_end_user_spend};
use crate::test_support::database_pool;
use rstest::rstest;
use sqlx::PgPool;
#[rstest]
#[case::fractional_costs(0.25, 0.75, false, 2.5, 1.5)]
#[case::zero_costs(0.0, 0.0, false, 2.0, 0.0)]
#[case::duplicate_ids(0.25, 0.75, true, 3.0, 3.0)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn end_user_spend_matches_python_upsert(
#[case] existing_cost: f64,
#[case] new_cost: f64,
#[case] duplicate_ids: bool,
#[case] expected_existing_spend: f64,
#[case] expected_new_spend: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _budget = sqlx::query(
r#"INSERT INTO "LiteLLM_BudgetTable" (budget_id, created_by, updated_by)
VALUES ('end-user-spend-budget', 'db-query-test', 'db-query-test')"#,
)
.execute(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _users = sqlx::query(
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id)
VALUES ('end-user-spend-z', 2, true, 'end-user-spend-budget'),
('end-user-spend-untouched', 100, true, NULL)"#,
)
.execute(&mut *tx)
.await?;
let spend = if duplicate_ids {
EntitySpend {
ids: vec![
"end-user-spend-z".into(),
"end-user-spend-a".into(),
"end-user-spend-z".into(),
"end-user-spend-a".into(),
],
costs: vec![existing_cost, new_cost, existing_cost, new_cost],
}
} else {
EntitySpend {
ids: vec!["end-user-spend-z".into(), "end-user-spend-a".into()],
costs: vec![existing_cost, new_cost],
}
};
add_end_user_spend(&mut *tx, &spend).await?;
add_end_user_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64, bool, Option<String>)> = sqlx::query_as(
r#"SELECT user_id, spend, blocked, budget_id FROM "LiteLLM_EndUserTable"
WHERE user_id = ANY($1) ORDER BY user_id"#,
)
.bind(vec![
"end-user-spend-a",
"end-user-spend-z",
"end-user-spend-untouched",
])
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
("end-user-spend-a".into(), expected_new_spend, false, None),
("end-user-spend-untouched".into(), 100.0, true, None),
(
"end-user-spend-z".into(),
expected_existing_spend,
true,
Some("end-user-spend-budget".into()),
),
]
);
Ok(())
}
#[rstest]
#[case::missing_cost(vec!["end-user-invalid-existing".into(), "end-user-invalid-new".into()], vec![1.0])]
#[case::extra_cost(vec!["end-user-invalid-existing".into()], vec![1.0, 2.0])]
#[case::empty_ids(vec![], vec![1.0])]
#[case::empty_costs(vec!["end-user-invalid-existing".into()], vec![])]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn malformed_end_user_spend_is_rejected_before_writes(
#[case] ids: Vec<String>,
#[case] costs: Vec<f64>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _user = sqlx::query(
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
VALUES ('end-user-invalid-existing', 8.5, true)"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend { ids, costs };
let result = add_end_user_spend(&mut *tx, &spend).await;
assert!(matches!(
result,
Err(crate::Error::InvalidSpendBatch { ids, costs })
if ids == spend.ids.len() && costs == spend.costs.len()
));
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
r#"SELECT user_id, spend, blocked FROM "LiteLLM_EndUserTable"
WHERE user_id IN ('end-user-invalid-existing', 'end-user-invalid-new')"#,
)
.fetch_all(&mut *tx)
.await?;
assert_eq!(rows, vec![("end-user-invalid-existing".into(), 8.5, true)]);
Ok(())
}
#[rstest]
#[case::empty_batch()]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn empty_end_user_spend_is_a_noop(
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _user = sqlx::query(
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
VALUES ('end-user-empty-spend', 8.5, true)"#,
)
.execute(&mut *tx)
.await?;
add_end_user_spend(&mut *tx, &EntitySpend::default()).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (f64, bool, Option<String>) = sqlx::query_as(
r#"SELECT spend, blocked, budget_id FROM "LiteLLM_EndUserTable"
WHERE user_id = 'end-user-empty-spend'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(row, (8.5, true, None));
Ok(())
}
}

View file

@ -0,0 +1,309 @@
use sqlx::PgExecutor;
use std::collections::BTreeMap;
use crate::Error;
#[macro_rules_attribute::apply(crate::db_input)]
#[derive(Default)]
pub struct EntitySpend {
pub ids: Vec<String>,
pub costs: Vec<f64>,
}
impl EntitySpend {
pub(crate) fn validate(&self) -> Result<(), Error> {
if self.ids.len() != self.costs.len() {
return Err(Error::InvalidSpendBatch {
ids: self.ids.len(),
costs: self.costs.len(),
});
}
Ok(())
}
pub fn is_empty(&self) -> bool {
self.ids.is_empty()
}
}
impl FromIterator<(String, f64)> for EntitySpend {
fn from_iter<I: IntoIterator<Item = (String, f64)>>(entries: I) -> Self {
let entries = entries
.into_iter()
.fold(BTreeMap::new(), |mut sums, (id, cost)| {
*sums.entry(id).or_insert(0.0) += cost;
sums
});
let (ids, costs) = entries.into_iter().unzip();
Self { ids, costs }
}
}
pub async fn add_key_spend(
executor: impl PgExecutor<'_>,
spend: &EntitySpend,
) -> Result<(), Error> {
spend.validate()?;
if spend.is_empty() {
return Ok(());
}
sqlx::query_file!("queries/spend/add_key_spend.sql", &spend.ids, &spend.costs)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{EntitySpend, add_key_spend};
use crate::{Error, test_support::database_pool};
use rstest::rstest;
use sqlx::PgPool;
#[rstest]
#[case::direct(false)]
#[case::collected(true)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn duplicate_key_spend_accumulates(
#[case] collected: bool,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
sqlx::query(
r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at)
VALUES ('key-spend-duplicate', 2, 7, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = if collected {
[
("key-spend-duplicate".into(), 0.25),
("key-spend-duplicate".into(), 0.75),
]
.into_iter()
.collect()
} else {
EntitySpend {
ids: vec!["key-spend-duplicate".into(), "key-spend-duplicate".into()],
costs: vec![0.25, 0.75],
}
};
add_key_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (f64, f64) = sqlx::query_as(
r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken"
WHERE token = 'key-spend-duplicate'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(row, (3.0, 8.0));
Ok(())
}
#[rstest]
#[case::extra_id(vec!["key-spend-invalid".into(), "key-spend-invalid".into()], vec![0.25])]
#[case::extra_cost(vec!["key-spend-invalid".into()], vec![0.25, 0.75])]
#[case::empty_ids(vec![], vec![0.25])]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn malformed_key_spend_leaves_transaction_usable(
#[case] ids: Vec<String>,
#[case] costs: Vec<f64>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
sqlx::query(
r#"INSERT INTO "LiteLLM_VerificationToken"
(token, spend, total_spend, last_active, updated_at)
VALUES ('key-spend-invalid', 2, 7, NULL, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend { ids, costs };
let result = add_key_spend(&mut *tx, &spend).await;
assert!(matches!(
result,
Err(Error::InvalidSpendBatch { ids, costs })
if ids == spend.ids.len() && costs == spend.costs.len()
));
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let unchanged: (f64, f64, bool, bool) = sqlx::query_as(
r#"SELECT spend, total_spend, last_active IS NULL,
updated_at = TIMESTAMP '2000-01-01'
FROM "LiteLLM_VerificationToken" WHERE token = 'key-spend-invalid'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(unchanged, (2.0, 7.0, true, true));
let valid = EntitySpend {
ids: vec!["key-spend-invalid".into()],
costs: vec![0.5],
};
add_key_spend(&mut *tx, &valid).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let updated: (f64, f64) = sqlx::query_as(
r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken"
WHERE token = 'key-spend-invalid'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(updated, (2.5, 7.5));
Ok(())
}
#[rstest]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn key_activity_uses_update_time_in_old_transaction(
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
sqlx::query(
r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at)
VALUES ('key-spend-old-transaction', 2, 7, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "Transaction timing setup is outside the compile-checked query set"
)]
sqlx::query("SELECT pg_sleep(0.02)")
.execute(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let boundary: chrono::NaiveDateTime = sqlx::query_scalar(
"SELECT date_trunc('milliseconds', clock_timestamp() AT TIME ZONE 'UTC')",
)
.fetch_one(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec!["key-spend-old-transaction".into()],
costs: vec![0.25],
};
add_key_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (chrono::NaiveDateTime, chrono::NaiveDateTime) = sqlx::query_as(
r#"SELECT last_active, updated_at FROM "LiteLLM_VerificationToken"
WHERE token = 'key-spend-old-transaction'"#,
)
.fetch_one(&mut *tx)
.await?;
assert!(row.0 >= boundary, "last_active predates update boundary");
assert!(row.1 >= boundary, "updated_at predates update boundary");
Ok(())
}
#[rstest]
#[case::fractional_costs(0.25, 2.5, 7.5, 4.0, 12.0)]
#[case::zero_costs(0.0, 2.0, 7.0, 3.0, 11.0)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn key_spend_matches_python_update_many(
#[case] cost: f64,
#[case] expected_a_spend: f64,
#[case] expected_a_total: f64,
#[case] expected_z_spend: f64,
#[case] expected_z_total: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_VerificationToken"
(token, spend, total_spend, last_active, updated_at)
VALUES ('key-spend-a', 2, 7, NULL, '2000-01-01'),
('key-spend-z', 3, 11, NULL, '2000-01-01'),
('key-spend-untouched', 100, 200, NULL, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec![
"key-spend-z".into(),
"key-spend-a".into(),
"key-spend-missing".into(),
],
costs: vec![2.0 * cost, cost, 50.0],
};
add_key_spend(&mut *tx, &EntitySpend::default()).await?;
add_key_spend(&mut *tx, &spend).await?;
add_key_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64, f64, bool, bool)> = sqlx::query_as(
r#"SELECT token, spend, total_spend, last_active IS NOT NULL,
updated_at > TIMESTAMP '2000-01-01'
FROM "LiteLLM_VerificationToken"
WHERE token = ANY($1) ORDER BY token"#,
)
.bind(vec![
"key-spend-a",
"key-spend-z",
"key-spend-missing",
"key-spend-untouched",
])
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
(
"key-spend-a".into(),
expected_a_spend,
expected_a_total,
true,
true
),
("key-spend-untouched".into(), 100.0, 200.0, false, false),
(
"key-spend-z".into(),
expected_z_spend,
expected_z_total,
true,
true
),
]
);
Ok(())
}
}

View file

@ -0,0 +1,21 @@
mod daily_end_user_spend;
mod daily_team_spend;
mod daily_user_spend;
mod end_user_spend;
mod key_spend;
mod spend_logs;
mod team_lock;
mod team_member_spend;
mod team_spend;
mod user_spend;
pub use daily_end_user_spend::{DailyTable, upsert_daily_end_user_spend, upsert_daily_spend};
pub use daily_team_spend::upsert_daily_team_spend;
pub use daily_user_spend::{DailySpend, upsert_daily_user_spend};
pub use end_user_spend::add_end_user_spend;
pub use key_spend::{EntitySpend, add_key_spend};
pub use spend_logs::{SpendLog, SpendLogRow, insert_spend_logs};
pub use team_lock::{begin, lock_team};
pub use team_member_spend::{TeamMemberSpend, add_team_member_spend};
pub use team_spend::add_team_spend;
pub use user_spend::add_user_spend;

View file

@ -0,0 +1,517 @@
use chrono::{DateTime, NaiveDateTime};
use sqlx::PgExecutor;
use crate::Error;
#[macro_rules_attribute::apply(crate::db_input)]
#[derive(Default)]
pub struct SpendLog {
pub request_id: String,
pub call_type: String,
pub api_key: String,
pub spend: f64,
pub total_tokens: i32,
pub prompt_tokens: i32,
pub completion_tokens: i32,
pub start_time: String,
pub end_time: String,
pub completion_start_time: Option<String>,
pub request_duration_ms: Option<i32>,
pub model: String,
pub model_id: String,
pub model_group: String,
pub custom_llm_provider: String,
pub api_base: String,
pub user: String,
pub metadata: serde_json::Value,
pub cache_hit: String,
pub cache_key: String,
pub team_id: String,
pub organization_id: String,
pub end_user: String,
pub requester_ip_address: Option<String>,
pub session_id: Option<String>,
pub status: String,
pub litellm_call_id: Option<String>,
}
#[macro_rules_attribute::apply(crate::db_row)]
pub struct SpendLogRow {
pub request_id: String,
pub call_type: String,
pub api_key: String,
pub spend: f64,
pub total_tokens: i32,
pub prompt_tokens: i32,
pub completion_tokens: i32,
#[sqlx(rename = "startTime")]
pub start_time: NaiveDateTime,
#[sqlx(rename = "endTime")]
pub end_time: NaiveDateTime,
#[sqlx(rename = "completionStartTime")]
pub completion_start_time: Option<NaiveDateTime>,
pub request_duration_ms: Option<i32>,
pub model: String,
pub model_id: Option<String>,
pub model_group: Option<String>,
pub custom_llm_provider: Option<String>,
pub api_base: Option<String>,
pub user: Option<String>,
pub metadata: Option<serde_json::Value>,
pub cache_hit: Option<String>,
pub cache_key: Option<String>,
pub request_tags: Option<serde_json::Value>,
pub team_id: Option<String>,
pub organization_id: Option<String>,
pub end_user: Option<String>,
pub requester_ip_address: Option<String>,
pub messages: Option<serde_json::Value>,
pub response: Option<serde_json::Value>,
pub proxy_server_request: Option<serde_json::Value>,
pub session_id: Option<String>,
pub status: Option<String>,
pub litellm_call_id: Option<String>,
}
fn parse_timestamp(value: &str) -> Result<NaiveDateTime, chrono::ParseError> {
DateTime::parse_from_rfc3339(value)
.map(|timestamp| timestamp.naive_utc())
.or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f"))
.or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f"))
}
pub async fn insert_spend_logs(
executor: impl PgExecutor<'_>,
logs: &[SpendLog],
) -> Result<(), Error> {
if logs.is_empty() {
return Ok(());
}
let start_times = logs
.iter()
.map(|log| parse_timestamp(&log.start_time))
.collect::<Result<Vec<_>, _>>()?;
let end_times = logs
.iter()
.map(|log| parse_timestamp(&log.end_time))
.collect::<Result<Vec<_>, _>>()?;
let completion_start_times = logs
.iter()
.map(|log| {
log.completion_start_time
.as_deref()
.map(parse_timestamp)
.transpose()
})
.collect::<Result<Vec<_>, _>>()?;
macro_rules! column {
($field:ident) => {
logs.iter()
.map(|log| log.$field.clone())
.collect::<Vec<_>>()
};
}
sqlx::query_file!(
"queries/spend/insert_spend_logs.sql",
&column!(request_id),
&column!(call_type),
&column!(api_key),
&column!(spend),
&column!(total_tokens),
&column!(prompt_tokens),
&column!(completion_tokens),
&start_times,
&end_times,
&completion_start_times as &[Option<NaiveDateTime>],
&column!(request_duration_ms) as &[Option<i32>],
&column!(model),
&column!(model_id),
&column!(model_group),
&column!(custom_llm_provider),
&column!(api_base),
&column!(user),
&column!(metadata),
&column!(cache_hit),
&column!(cache_key),
&column!(team_id),
&column!(organization_id),
&column!(end_user),
&column!(requester_ip_address) as &[Option<String>],
&column!(session_id) as &[Option<String>],
&column!(status),
&column!(litellm_call_id) as &[Option<String>],
)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{SpendLog, SpendLogRow, insert_spend_logs};
use crate::test_support::database_pool;
use rstest::{fixture, rstest};
use serde_json::json;
use sqlx::{Acquire, Connection, PgExecutor, PgPool, types::chrono::NaiveDateTime};
#[fixture]
fn logs() -> [SpendLog; 2] {
[
SpendLog {
request_id: "spend-logs-a".into(),
call_type: "acompletion".into(),
api_key: "key-a".into(),
spend: 0.125,
total_tokens: 17,
prompt_tokens: 11,
completion_tokens: 6,
start_time: "2026-01-02 03:04:05.123".into(),
end_time: "2026-01-02 03:04:06.789".into(),
completion_start_time: Some("2026-01-02 03:04:05.456".into()),
request_duration_ms: Some(1666),
model: "model-a".into(),
model_id: "deployment-a".into(),
model_group: "group-a".into(),
custom_llm_provider: "provider-a".into(),
api_base: "https://a.example.test".into(),
user: "user-a".into(),
metadata: json!({"tenant": "a", "nested": {"tokens": 11}, "flag": false}),
cache_hit: "False".into(),
cache_key: "cache-a".into(),
team_id: "team-a".into(),
organization_id: "org-a".into(),
end_user: "end-user-a".into(),
requester_ip_address: Some("192.0.2.1".into()),
session_id: Some("session-a".into()),
status: "success".into(),
litellm_call_id: Some("call-a".into()),
},
SpendLog {
request_id: "spend-logs-b".into(),
call_type: "aembedding".into(),
api_key: "key-b".into(),
spend: 2.75,
total_tokens: 43,
prompt_tokens: 29,
completion_tokens: 14,
start_time: "2026-02-03 14:15:16.234".into(),
end_time: "2026-02-03 14:15:18.567".into(),
completion_start_time: None,
request_duration_ms: None,
model: "model-b".into(),
model_id: "deployment-b".into(),
model_group: "group-b".into(),
custom_llm_provider: "provider-b".into(),
api_base: "https://b.example.test".into(),
user: "user-b".into(),
metadata: json!({"tenant": "b", "tags": ["distinct", 7], "nullable": null}),
cache_hit: "True".into(),
cache_key: "cache-b".into(),
team_id: "team-b".into(),
organization_id: "org-b".into(),
end_user: "end-user-b".into(),
requester_ip_address: None,
session_id: None,
status: "failure".into(),
litellm_call_id: None,
},
]
}
async fn read_log(
executor: impl PgExecutor<'_>,
request_id: &str,
) -> Result<SpendLogRow, sqlx::Error> {
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
sqlx::query_as::<_, SpendLogRow>(
r#"SELECT * FROM "LiteLLM_SpendLogs" WHERE request_id = $1"#,
)
.bind(request_id)
.fetch_one(executor)
.await
}
fn expected_log(log: &SpendLog) -> Result<SpendLogRow, chrono::ParseError> {
Ok(SpendLogRow {
request_id: log.request_id.clone(),
call_type: log.call_type.clone(),
api_key: log.api_key.clone(),
spend: log.spend,
total_tokens: log.total_tokens,
prompt_tokens: log.prompt_tokens,
completion_tokens: log.completion_tokens,
start_time: log.start_time.replace(' ', "T").parse()?,
end_time: log.end_time.replace(' ', "T").parse()?,
completion_start_time: log
.completion_start_time
.as_ref()
.map(|value| value.replace(' ', "T").parse())
.transpose()?,
request_duration_ms: log.request_duration_ms,
model: log.model.clone(),
model_id: Some(log.model_id.clone()),
model_group: Some(log.model_group.clone()),
custom_llm_provider: Some(log.custom_llm_provider.clone()),
api_base: Some(log.api_base.clone()),
user: Some(log.user.clone()),
metadata: Some(log.metadata.clone()),
cache_hit: Some(log.cache_hit.clone()),
cache_key: Some(log.cache_key.clone()),
request_tags: Some(json!([])),
team_id: Some(log.team_id.clone()),
organization_id: Some(log.organization_id.clone()),
end_user: Some(log.end_user.clone()),
requester_ip_address: log.requester_ip_address.clone(),
messages: Some(json!({})),
response: Some(json!({})),
proxy_server_request: Some(json!({})),
session_id: log.session_id.clone(),
status: Some(log.status.clone()),
litellm_call_id: log.litellm_call_id.clone(),
})
}
#[rstest]
#[case::all_columns_and_duplicate()]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn inserts_distinct_rows_without_replacing_duplicates(
logs: [SpendLog; 2],
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
insert_spend_logs(&mut *tx, &logs).await?;
let replacement = SpendLog {
request_id: logs[0].request_id.clone(),
..logs[1].clone()
};
insert_spend_logs(&mut *tx, &[replacement]).await?;
assert_eq!(
read_log(&mut *tx, &logs[0].request_id).await?,
expected_log(&logs[0])?
);
assert_eq!(
read_log(&mut *tx, &logs[1].request_id).await?,
expected_log(&logs[1])?
);
tx.rollback().await?;
Ok(())
}
#[rstest]
#[case::partitioned_duplicate()]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn inserts_into_partitioned_logs_without_replacing_duplicates(
logs: [SpendLog; 2],
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "The test creates a temporary partitioned table at runtime"
)]
sqlx::raw_sql(
r#"CREATE TEMP TABLE "LiteLLM_SpendLogs" (
LIKE public."LiteLLM_SpendLogs" INCLUDING DEFAULTS INCLUDING GENERATED,
PRIMARY KEY (request_id, "startTime")
) PARTITION BY RANGE ("startTime");
CREATE TEMP TABLE spend_logs_default
PARTITION OF pg_temp."LiteLLM_SpendLogs" DEFAULT;
SET LOCAL search_path TO pg_temp, public;"#,
)
.execute(&mut *tx)
.await?;
tx.clear_cached_statements().await?;
insert_spend_logs(&mut *tx, &logs).await?;
let replacement = SpendLog {
spend: 99.0,
..logs[0].clone()
};
insert_spend_logs(&mut *tx, &[replacement]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The partitioned test fixture is created at runtime"
)]
let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#)
.fetch_one(&mut *tx)
.await?;
assert_eq!(count.0, 2);
assert_eq!(
read_log(&mut *tx, &logs[0].request_id).await?,
expected_log(&logs[0])?
);
assert_eq!(
read_log(&mut *tx, &logs[1].request_id).await?,
expected_log(&logs[1])?
);
let distinct_partition_key = SpendLog {
request_id: logs[0].request_id.clone(),
..logs[1].clone()
};
insert_spend_logs(&mut *tx, &[distinct_partition_key]).await?;
#[expect(
clippy::disallowed_methods,
reason = "The partitioned test fixture is created at runtime"
)]
let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#)
.fetch_one(&mut *tx)
.await?;
assert_eq!(count.0, 3);
tx.rollback().await?;
Ok(())
}
#[rstest]
#[case::positive_offset_across_midnight(
["2026-01-02T00:04:05.1236+02:00", "2026-01-02T00:04:06.7894+02:00", "2026-01-02T00:04:05.4567+02:00"],
["2026-01-01T22:04:05.124", "2026-01-01T22:04:06.789", "2026-01-01T22:04:05.457"]
)]
#[case::negative_offset_across_midnight(
["2026-01-01T23:04:05-02:00", "2026-01-01T23:04:06-02:00", "2026-01-01T23:04:05.456-02:00"],
["2026-01-02T01:04:05", "2026-01-02T01:04:06", "2026-01-02T01:04:05.456"]
)]
#[case::utc(
["2026-01-02T03:04:05Z", "2026-01-02T03:04:06Z", "2026-01-02T03:04:05.456Z"],
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
)]
#[case::naive_iso(
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"],
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
)]
#[case::naive_space(
["2026-01-02 03:04:05", "2026-01-02 03:04:06", "2026-01-02 03:04:05.456"],
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn stores_utc_timestamps(
#[case] input: [&str; 3],
#[case] expected: [&str; 3],
logs: [SpendLog; 2],
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "The test sets a transaction-local timezone to verify timestamp handling"
)]
sqlx::raw_sql("SET LOCAL TIME ZONE 'America/Los_Angeles'")
.execute(&mut *tx)
.await?;
let log = SpendLog {
start_time: input[0].into(),
end_time: input[1].into(),
completion_start_time: Some(input[2].into()),
..logs[0].clone()
};
insert_spend_logs(&mut *tx, std::slice::from_ref(&log)).await?;
let row = read_log(&mut *tx, &log.request_id).await?;
assert_eq!(row.start_time, expected[0].parse::<NaiveDateTime>()?);
assert_eq!(row.end_time, expected[1].parse::<NaiveDateTime>()?);
assert_eq!(
row.completion_start_time,
Some(expected[2].parse::<NaiveDateTime>()?)
);
tx.rollback().await?;
Ok(())
}
#[rstest]
#[case::empty_batch()]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn empty_input_leaves_rows_unchanged(
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let before: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#)
.fetch_one(&mut *tx)
.await?;
insert_spend_logs(&mut *tx, &[]).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let after: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#)
.fetch_one(&mut *tx)
.await?;
assert_eq!(before, after);
tx.rollback().await?;
Ok(())
}
#[rstest]
#[case::invalid_timestamp("not-a-timestamp", "2026-01-02T03:04:06", None)]
#[case::invalid_end("2026-01-02T03:04:05", "not-a-timestamp", None)]
#[case::invalid_completion(
"2026-01-02T03:04:05",
"2026-01-02T03:04:06",
Some("not-a-timestamp")
)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn invalid_timestamp_allows_caller_rollback(
#[case] start_time: &str,
#[case] end_time: &str,
#[case] completion_start_time: Option<&str>,
logs: [SpendLog; 2],
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut outer = pool.begin().await?;
let mut tx = outer.begin().await?;
let valid = SpendLog {
request_id: "spend-logs-rollback-valid".into(),
..logs[0].clone()
};
let invalid = SpendLog {
request_id: "spend-logs-rollback-invalid".into(),
start_time: start_time.into(),
end_time: end_time.into(),
completion_start_time: completion_start_time.map(str::to_owned),
..logs[1].clone()
};
insert_spend_logs(&mut *tx, std::slice::from_ref(&valid)).await?;
let result = insert_spend_logs(&mut *tx, &[logs[0].clone(), invalid]).await;
assert!(matches!(result, Err(crate::Error::InvalidTimestamp(_))));
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let partial_batch: (i64,) = sqlx::query_as(
r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#,
)
.bind([logs[0].request_id.as_str(), "spend-logs-rollback-invalid"].as_slice())
.fetch_one(&mut *tx)
.await?;
assert_eq!(partial_batch.0, 0);
tx.rollback().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let remaining: (i64,) = sqlx::query_as(
r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#,
)
.bind([valid.request_id.as_str(), "spend-logs-rollback-invalid"].as_slice())
.fetch_one(&mut *outer)
.await?;
assert_eq!(remaining.0, 0);
outer.rollback().await?;
Ok(())
}
}

View file

@ -0,0 +1,98 @@
use sqlx::{PgPool, Postgres, Transaction};
use crate::Error;
pub async fn begin(pool: &PgPool) -> Result<Transaction<'static, Postgres>, Error> {
let mut transaction = pool.begin().await?;
sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)")
.fetch_one(&mut *transaction)
.await?;
Ok(transaction)
}
pub async fn lock_team(
transaction: &mut Transaction<'_, Postgres>,
team_id: &str,
) -> Result<(), Error> {
sqlx::query_file!("queries/spend/lock_team.sql", team_id)
.fetch_one(&mut **transaction)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{begin, lock_team};
use crate::test_support::database_pool;
use rstest::rstest;
use sqlx::{PgPool, postgres::PgPoolOptions};
#[rstest]
#[case::transaction_scoped("team-lock-scope")]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn team_lock_is_held_until_rollback(
#[case] team_id: &str,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let database_pool = database_pool?;
let pool = PgPoolOptions::new()
.max_connections(2)
.connect_with(database_pool.connect_options().as_ref().clone())
.await?;
let mut first = begin(&pool).await?;
let mut second = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')")
.fetch_one(&mut *second)
.await?;
lock_team(&mut first, team_id).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let (backend, timeout): (i32, String) =
sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')")
.fetch_one(&mut *first)
.await?;
assert_eq!(timeout, "5s");
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let held: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))")
.bind(team_id)
.fetch_one(&mut *second)
.await?;
assert!(!held);
first.rollback().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let released: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))")
.bind(team_id)
.fetch_one(&mut *second)
.await?;
assert!(released);
let mut next = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let (next_backend, next_timeout): (i32, String) =
sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')")
.fetch_one(&mut *next)
.await?;
assert_eq!(next_backend, backend);
assert_eq!(next_timeout, original_timeout);
next.rollback().await?;
second.rollback().await?;
Ok(())
}
}

View file

@ -0,0 +1,162 @@
use sqlx::{Postgres, Transaction};
use super::lock_team;
use crate::Error;
#[macro_rules_attribute::apply(crate::db_input)]
pub struct TeamMemberSpend {
pub team_id: String,
pub user_id: String,
pub cost: f64,
}
pub async fn add_team_member_spend(
transaction: &mut Transaction<'_, Postgres>,
members: &[TeamMemberSpend],
) -> Result<(), Error> {
if members.is_empty() {
return Ok(());
}
let mut members = members.to_vec();
members.sort_by(|left, right| {
(&left.team_id, &left.user_id).cmp(&(&right.team_id, &right.user_id))
});
let mut teams: Vec<&str> = members
.iter()
.map(|member| member.team_id.as_str())
.collect();
teams.dedup();
for team in teams {
lock_team(transaction, team).await?;
}
let rows = serde_json::Value::Array(
members
.iter()
.map(|member| {
serde_json::json!({
"user_id": member.user_id,
"team_id": member.team_id,
"cost": member.cost,
})
})
.collect(),
);
sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows)
.execute(&mut **transaction)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::super::begin;
use super::{TeamMemberSpend, add_team_member_spend};
use crate::test_support::database_pool;
use rstest::rstest;
use sqlx::PgPool;
fn members(cost: f64) -> [TeamMemberSpend; 3] {
[
TeamMemberSpend {
team_id: "member-spend-z".into(),
user_id: "off-roster-existing".into(),
cost: 2.0 * cost,
},
TeamMemberSpend {
team_id: "member-spend-a".into(),
user_id: "on-roster-missing".into(),
cost,
},
TeamMemberSpend {
team_id: "member-spend-a".into(),
user_id: "off-roster-missing".into(),
cost: 50.0 * cost,
},
]
}
#[rstest]
#[case::zero_then_quarter(0.25, 0.5, 3.0, 8.0)]
#[case::zero_then_eighth(0.125, 0.25, 2.5, 7.5)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn membership_spend_preserves_roster_guard_and_fractional_costs(
#[case] cost: f64,
#[case] expected_created: f64,
#[case] expected_spend: f64,
#[case] expected_total: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = begin(&pool).await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _teams = sqlx::query(
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, members_with_roles)
VALUES ('member-spend-a', '[{"user_id":"on-roster-missing","role":"user"}]'),
('member-spend-z', '[]')"#,
)
.execute(&mut *tx)
.await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _members = sqlx::query(
r#"INSERT INTO "LiteLLM_TeamMembership" (team_id, user_id, spend, total_spend)
VALUES ('member-spend-z', 'off-roster-existing', 2, 7),
('member-spend-a', 'untouched', 100, 200)"#,
)
.execute(&mut *tx)
.await?;
add_team_member_spend(&mut tx, &[]).await?;
add_team_member_spend(&mut tx, &members(0.0)).await?;
add_team_member_spend(&mut tx, &members(cost)).await?;
add_team_member_spend(&mut tx, &members(cost)).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, String, f64, f64)> = sqlx::query_as(
r#"SELECT team_id, user_id, spend, total_spend FROM "LiteLLM_TeamMembership"
WHERE team_id IN ('member-spend-a', 'member-spend-z') ORDER BY team_id, user_id"#,
)
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
(
"member-spend-a".into(),
"on-roster-missing".into(),
expected_created,
expected_created
),
("member-spend-a".into(), "untouched".into(), 100.0, 200.0),
(
"member-spend-z".into(),
"off-roster-existing".into(),
expected_spend,
expected_total
),
]
);
tx.rollback().await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let remaining: i64 = sqlx::query_scalar(
r#"SELECT COUNT(*) FROM "LiteLLM_TeamMembership"
WHERE team_id IN ('member-spend-a', 'member-spend-z')"#,
)
.fetch_one(&pool)
.await?;
assert_eq!(remaining, 0);
Ok(())
}
}

View file

@ -0,0 +1,182 @@
use sqlx::PgExecutor;
use super::EntitySpend;
use crate::Error;
pub async fn add_team_spend(
executor: impl PgExecutor<'_>,
spend: &EntitySpend,
) -> Result<(), Error> {
spend.validate()?;
if spend.is_empty() {
return Ok(());
}
sqlx::query_file!("queries/spend/add_team_spend.sql", &spend.ids, &spend.costs)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{EntitySpend, add_team_spend};
use crate::Error;
use crate::test_support::database_pool;
use rstest::rstest;
use sqlx::PgPool;
#[rstest]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn duplicate_team_ids_add_every_cost(
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
VALUES ('team-spend-duplicate-a', 2, '2000-01-01', '2000-01-01'),
('team-spend-duplicate-z', 3, '2000-01-01', '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec![
"team-spend-duplicate-z".into(),
"team-spend-duplicate-a".into(),
"team-spend-duplicate-z".into(),
"team-spend-duplicate-a".into(),
],
costs: vec![0.5, 0.25, 1.0, 0.5],
};
add_team_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64)> = sqlx::query_as(
r#"SELECT team_id, spend FROM "LiteLLM_TeamTable"
WHERE team_id = ANY($1) ORDER BY team_id"#,
)
.bind(&spend.ids)
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
("team-spend-duplicate-a".into(), 2.75),
("team-spend-duplicate-z".into(), 4.5),
]
);
Ok(())
}
#[rstest]
#[case::missing_cost(vec!["team-spend-malformed".into()], vec![])]
#[case::extra_cost(vec!["team-spend-malformed".into()], vec![0.25, 0.5])]
#[case::empty_ids(vec![], vec![0.25])]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn malformed_team_batch_does_not_write(
#[case] ids: Vec<String>,
#[case] costs: Vec<f64>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
VALUES ('team-spend-malformed', 2, '2000-01-01', '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let result = add_team_spend(&mut *tx, &EntitySpend { ids, costs }).await;
assert!(matches!(result, Err(Error::InvalidSpendBatch { .. })));
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (f64, bool) = sqlx::query_as(
r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01'
FROM "LiteLLM_TeamTable" WHERE team_id = 'team-spend-malformed'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(row, (2.0, true));
Ok(())
}
#[rstest]
#[case::fractional_costs(0.25, 2.5, 4.0)]
#[case::zero_costs(0.0, 2.0, 3.0)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn team_spend_matches_python_update_many(
#[case] cost: f64,
#[case] expected_a_spend: f64,
#[case] expected_z_spend: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
VALUES ('team-spend-a', 2, '2000-01-01', '2000-01-01'),
('team-spend-z', 3, '2000-01-01', '2000-01-01'),
('team-spend-untouched', 100, '2000-01-01', '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec![
"team-spend-z".into(),
"team-spend-a".into(),
"team-spend-missing".into(),
],
costs: vec![2.0 * cost, cost, 50.0],
};
add_team_spend(&mut *tx, &EntitySpend::default()).await?;
add_team_spend(&mut *tx, &spend).await?;
add_team_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
r#"SELECT team_id, spend, updated_at > TIMESTAMP '2000-01-01'
FROM "LiteLLM_TeamTable"
WHERE team_id = ANY($1) ORDER BY team_id"#,
)
.bind(vec![
"team-spend-a",
"team-spend-z",
"team-spend-missing",
"team-spend-untouched",
])
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
("team-spend-a".into(), expected_a_spend, true),
("team-spend-untouched".into(), 100.0, false),
("team-spend-z".into(), expected_z_spend, true),
]
);
Ok(())
}
}

View file

@ -0,0 +1,173 @@
use sqlx::PgExecutor;
use super::EntitySpend;
use crate::Error;
pub async fn add_user_spend(
executor: impl PgExecutor<'_>,
spend: &EntitySpend,
) -> Result<(), Error> {
spend.validate()?;
if spend.is_empty() {
return Ok(());
}
sqlx::query_file!("queries/spend/add_user_spend.sql", &spend.ids, &spend.costs)
.execute(executor)
.await?;
Ok(())
}
#[cfg(test)]
mod tests {
use std::error::Error as StdError;
use super::{EntitySpend, add_user_spend};
use crate::{Error, test_support::database_pool};
use rstest::rstest;
use sqlx::PgPool;
#[rstest]
#[case::fractional_costs(0.25, 2.5, 4.0)]
#[case::zero_costs(0.0, 2.0, 3.0)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn user_spend_matches_python_update_many(
#[case] cost: f64,
#[case] expected_a_spend: f64,
#[case] expected_z_spend: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
VALUES ('user-spend-a', 2, '2000-01-01'),
('user-spend-z', 3, '2000-01-01'),
('user-spend-untouched', 100, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec![
"user-spend-z".into(),
"user-spend-a".into(),
"user-spend-missing".into(),
],
costs: vec![2.0 * cost, cost, 50.0],
};
add_user_spend(&mut *tx, &EntitySpend::default()).await?;
add_user_spend(&mut *tx, &spend).await?;
add_user_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
r#"SELECT user_id, spend, updated_at > TIMESTAMP '2000-01-01'
FROM "LiteLLM_UserTable"
WHERE user_id = ANY($1) ORDER BY user_id"#,
)
.bind(vec![
"user-spend-a",
"user-spend-z",
"user-spend-missing",
"user-spend-untouched",
])
.fetch_all(&mut *tx)
.await?;
assert_eq!(
rows,
vec![
("user-spend-a".into(), expected_a_spend, true),
("user-spend-untouched".into(), 100.0, false),
("user-spend-z".into(), expected_z_spend, true),
]
);
Ok(())
}
#[rstest]
#[case::direct_duplicate_batch(vec![0.25, 0.75], 3.0)]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn user_spend_sums_duplicate_ids(
#[case] costs: Vec<f64>,
#[case] expected_spend: f64,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
VALUES ('user-spend-duplicate', 2, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec!["user-spend-duplicate".into(), "user-spend-duplicate".into()],
costs,
};
add_user_spend(&mut *tx, &spend).await?;
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (f64, bool) = sqlx::query_as(
r#"SELECT spend, updated_at > TIMESTAMP '2000-01-01'
FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-duplicate'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(row, (expected_spend, true));
Ok(())
}
#[rstest]
#[case::missing_cost(vec![])]
#[case::extra_cost(vec![0.25, 0.75])]
#[tokio::test]
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
async fn user_spend_rejects_malformed_batch_without_writes(
#[case] costs: Vec<f64>,
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
) -> Result<(), Box<dyn StdError>> {
let pool = database_pool?;
let mut tx = pool.begin().await?;
#[expect(
clippy::disallowed_methods,
reason = "Fixture writes are outside the compile-checked query set"
)]
let _inserted = sqlx::query(
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
VALUES ('user-spend-malformed', 2, '2000-01-01')"#,
)
.execute(&mut *tx)
.await?;
let spend = EntitySpend {
ids: vec!["user-spend-malformed".into()],
costs,
};
let result = add_user_spend(&mut *tx, &spend).await;
assert!(matches!(result, Err(Error::InvalidSpendBatch { .. })));
#[expect(
clippy::disallowed_methods,
reason = "Test assertions are outside the compile-checked query set"
)]
let row: (f64, bool) = sqlx::query_as(
r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01'
FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-malformed'"#,
)
.fetch_one(&mut *tx)
.await?;
assert_eq!(row, (2.0, true));
Ok(())
}
}

View file

@ -0,0 +1,10 @@
use std::error::Error as StdError;
use rstest::fixture;
use sqlx::PgPool;
#[fixture]
pub async fn database_pool() -> Result<PgPool, Box<dyn StdError>> {
let database_url = std::env::var("DATABASE_URL")?;
Ok(PgPool::connect(&database_url).await?)
}

View file

@ -88,20 +88,23 @@ fn aws_context(
}
fn read_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<Duration>> {
let Some(value) = value.filter(|value| !value.is_none()) else {
return Ok(None);
};
let seconds = match value.extract::<f64>() {
Ok(value) => Some(value),
Err(_) => value.getattr("read")?.extract::<Option<f64>>()?,
};
seconds
read_timeout_seconds(value)?
.map(|value| {
Duration::try_from_secs_f64(value).map_err(|_| PyValueError::new_err("invalid timeout"))
})
.transpose()
}
pub(super) fn read_timeout_seconds(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<f64>> {
let Some(value) = value.filter(|value| !value.is_none()) else {
return Ok(None);
};
match value.extract::<f64>() {
Ok(value) => Ok(Some(value)),
Err(_) => value.getattr("read")?.extract::<Option<f64>>(),
}
}
fn vault_context(params: Option<&Bound<'_, PyAny>>) -> PyResult<HashicorpOperationContext> {
let params = params.and_then(|value| value.cast::<PyDict>().ok());
let nested = params

View file

@ -2,10 +2,12 @@ mod operation;
pub(super) use operation::{Failure, FailureKind, FailureStage, delete, rotate, write};
use litellm_core_utils::exception_mapping_utils::{TimeoutPhase, timeout_message};
use litellm_secrets::hashicorp::{Error, RawOperationError};
use pyo3::prelude::*;
use super::mutation::{error_value, http_message, json_value};
use super::provider::read_timeout_seconds;
const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport";
@ -69,21 +71,17 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P
}
FailureKind::Native(RawOperationError::Timeout { method, elapsed }) => {
if method == "POST" {
let elapsed = py
.import("builtins")?
.call_method1("round", (elapsed.as_secs_f64(), 3))?;
let kwargs = pyo3::types::PyDict::new(py);
kwargs.set_item(
"message",
format!(
"Connection timed out. Timeout passed={}, time taken={} seconds",
context.timeout.as_deref().unwrap_or("None"),
elapsed.str()?
timeout_message(
TimeoutPhase::Read,
context.timeout,
Some(elapsed.as_secs_f64()),
),
)?;
kwargs.set_item("model", "default-model-name")?;
kwargs.set_item("llm_provider", "litellm-httpx-handler")?;
kwargs.set_item("headers", pyo3::types::PyDict::new(py))?;
py.import("litellm.exceptions")?
.getattr("Timeout")?
.call((), Some(&kwargs))?
@ -159,7 +157,7 @@ fn response_text(py: Python<'_>, body: &[u8]) -> PyResult<String> {
#[derive(Default)]
pub(super) struct ErrorContext {
timeout: Option<String>,
timeout: Option<f64>,
aiohttp: bool,
}
@ -173,7 +171,7 @@ impl ErrorContext {
return Ok(Self::default());
}
Ok(Self {
timeout: timeout.map(|value| value.str()?.extract()).transpose()?,
timeout: read_timeout_seconds(timeout)?,
aiohttp: py
.import(HTTP_TRANSPORT_MODULE)?
.getattr("uses_aiohttp_transport")?

View file

@ -1,10 +1,14 @@
"""
Internal unified UUID helper.
Always uses fastuuid for performance.
Uses fastuuid where supported and the standard library on Python 3.15+.
"""
import fastuuid as _uuid
import sys
import uuid as _uuid
if sys.version_info < (3, 15):
import fastuuid as _uuid
# Expose a module-like alias so callers can use: uuid.uuid4()
uuid = _uuid

View file

@ -42,7 +42,7 @@ class OtelDestination(LiteLLMBaseModel):
success_sampling_rate: float | None = Field(
default=None,
description=(
"Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; "
"Share of the request trees forwarded, 0.0..1.0, picked by a hash of the trace id; "
"``None`` forwards every one."
),
)

View file

@ -1,13 +1,14 @@
"""Provider / exporter factory + the Baggage span processor."""
import queue
import random
import threading
import time
import weakref
from collections import OrderedDict
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from hashlib import sha256
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, Literal
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
from opentelemetry import _logs, baggage, metrics, trace
from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider
@ -486,15 +487,15 @@ def is_llm_call_span(span: ReadableSpan) -> bool:
return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes
#: Request trees held back for a sampled destination until the last span of the trace
#: Request trees held back for a sampled destination until the last span of the request
#: still open ends, the verdicts kept for the spans that start after that (a post-call
#: database write), and the spans in flight per trace. All bounded, so a span that never
#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided
#: on what it has.
#: database write), the spans in flight per request, and the request each open span
#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot
#: hold spans for ever: the oldest tree is decided on what it has.
_MAX_PENDING_TREES: Final = 1024
_MAX_PENDING_SPANS_PER_TREE: Final = 512
_MAX_REMEMBERED_VERDICTS: Final = 4096
_MAX_OPEN_TRACES: Final = 16384
_MAX_OPEN_REQUESTS: Final = 16384
class _PendingTree:
@ -512,25 +513,35 @@ def _is_sampled(destination: "OtelDestination") -> bool:
return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None
#: A verdict is one request tree's draw for one destination at its rates: the exporter
#: A request is its trace id and the id of its root, the span of it whose parent is
#: remote or missing. Calls sharing a caller's trace are separate requests, each decided
#: on its own outcome.
_RequestKey: TypeAlias = tuple[int, int]
#: A verdict is one request's keep or drop for one destination at its rates: the exporter
#: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with
#: different rates would otherwise share one draw.
_VerdictKey = tuple[int, object, float | None, float | None]
#: different rates would otherwise share one verdict.
_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None]
def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey:
return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey:
return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool:
"""Whether one draw against the destination's rate keeps a request tree, read the
way the legacy Arize callback reads ``arize_success_sampling_rate`` /
``arize_error_sampling_rate``: a tree with a failed span answers to the error
rate, an unset rate keeps everything and ``0.0`` keeps nothing."""
def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool:
"""Whether the destination's rate keeps a request tree of ``trace_id``: a tree with
a failed span answers to ``arize_error_sampling_rate``, the rest to
``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps
nothing.
The first eight bytes of a SHA-256 of the trace id are read against the rate, so
every process deciding a request of the trace with the same outcome reaches the same
verdict."""
rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate
if rate is None:
return True
return rate > 0.0 and draw() <= rate
digest: Final = sha256(trace_id.to_bytes(16, "big")).digest()
return int.from_bytes(digest[:8], "big") < round(rate * 2**64)
def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool:
@ -667,10 +678,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
pending_drains: int = _MAX_PENDING_DRAINS,
drain_pool: _DrainPool | None = None,
sampling_draw: Callable[[], float] | None = None,
) -> None:
self._operator_sinks: Final = operator_sinks
self._draw: Final = sampling_draw if sampling_draw is not None else random.random
self._excluded_db_systems: Final = excluded_db_systems
self._default_capture: Final = default_capture
self._default_capture_by_backend: Final = default_capture_by_backend
@ -682,24 +691,32 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU
self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id
self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request
self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
context: Final = span.context
trace_id: Final = context.trace_id if context is not None else 0
with self._lock:
self._open[trace_id] = self._open.get(trace_id, 0) + 1
while len(self._open) > _MAX_OPEN_TRACES:
self._open.popitem(last=False)
request: Final = self._request_of(span, trace.get_current_span(parent_context))
if context is not None:
self._span_roots[span] = request[1]
self._roots[context.span_id] = request
self._open.setdefault(request, set()).add(context.span_id)
while len(self._open) > _MAX_OPEN_REQUESTS:
for forgotten in self._open.popitem(last=False)[1]:
self._roots.pop(forgotten, None)
def on_end(self, span: ReadableSpan) -> None:
suppressed: Final = suppressed_backends()
attributes: Final = span.attributes or _NO_ATTRIBUTES
context: Final = span.context
trace_id: Final = context.trace_id if context is not None else 0
with self._lock:
request: Final = self._request_of(span)
if span.context is not None:
self._roots.pop(span.context.span_id, None)
failed: Final = span.status.status_code is StatusCode.ERROR
for destination in request_destinations():
if (
@ -716,11 +733,32 @@ class TenantFanOutSpanProcessor(SpanProcessor):
if not _is_sampled(destination):
self._forward(span, destination)
continue
for held_span, held_destination in self._route(trace_id, span, destination, failed):
for held_span, held_destination in self._route(request, span, destination, failed):
self._forward(held_span, held_destination)
for held_span, held_destination in self._settle(trace_id, failed):
for held_span, held_destination in self._settle(request, span, failed):
self._forward(held_span, held_destination)
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
"""The request ``span`` belongs to, read under ``_lock``: the request recorded while
it is open, else the span itself when its parent is remote or missing, else its
parent's request, recorded while the parent is open or on ``parent_span`` for a
child starting under a parent that has ended, else the parent itself."""
context: Final = span.context
if context is None:
return (0, 0)
recorded: Final = self._roots.get(context.span_id)
if recorded is not None:
return recorded
parent: Final = span.parent
if parent is None or parent.is_remote:
return (context.trace_id, context.span_id)
parent_request: Final = self._roots.get(parent.span_id)
if parent_request is not None:
return parent_request
if isinstance(parent_span, SDKSpan):
return (context.trace_id, self._span_roots.get(parent_span, parent.span_id))
return (context.trace_id, parent.span_id)
def _default_capture_for(self, destination: "OtelDestination") -> str:
backend: Final = destination.callback_name
return (
@ -743,14 +781,14 @@ class TenantFanOutSpanProcessor(SpanProcessor):
self._release(processor)
def _route(
self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool
self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""The spans to forward now for ``span`` on a sampled ``destination``: the span
itself when its tree's verdict is already known and keeps it, nothing while the
tree is held for the rest of its trace, or what the bounds made the fan-out decide
tree is held for the rest of its request, or what the bounds made the fan-out decide
early (a tree that outgrew its cap is decided on what it has, and so is the oldest
tree once too many are waiting, so a span that never ends holds nothing back for
ever). A failed span marks its tree before any such early draw. Once shutdown
ever). A failed span marks its tree before any such early decision. Once shutdown
has started flushing, nothing is held: a span ending then is decided at once,
since no later flush would reach it.
@ -758,58 +796,62 @@ class TenantFanOutSpanProcessor(SpanProcessor):
another thread between the two would leave this span in a fresh tree.
"""
with self._lock:
verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination))
verdict: Final = self._verdicts.get(_verdict_key(request, destination))
if verdict is not None:
return ((span, destination),) if verdict else ()
tree: Final = self._pending.setdefault(trace_id, _PendingTree())
tree: Final = self._pending.setdefault(request, _PendingTree())
tree.held.append((span, destination))
if failed:
tree.failed = True
if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE:
return self._decide_locked(trace_id)
return self._decide_locked(request)
if len(self._pending) > _MAX_PENDING_TREES:
oldest: Final = next(iter(self._pending))
return self._decide_locked(oldest)
return ()
def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
with self._lock:
return self._decide_locked(trace_id)
return self._decide_locked(request)
def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""What the trace's held tree exports once the span that just ended was the last
of its trace still open. The model-call span ends in the post-call callback,
after the server span, so a tree is decided when its trace goes quiet rather than
def _settle(
self, request: _RequestKey, span: ReadableSpan, failed: bool
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""What the request's held tree exports once the span that just ended was the last
of its request still open. The model-call span ends in the post-call callback,
after the server span, so a tree is decided when its request goes quiet rather than
when its root ends, and a model call that fails late still answers to the error
rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten
by the bound) is decided as each of its spans ends.
rate. A request whose starts this fan-out never saw (attached mid-flight, or
forgotten by the bound) is decided as each of its spans ends.
"""
with self._lock:
tree: Final = self._pending.get(trace_id)
tree: Final = self._pending.get(request)
if tree is not None and failed:
tree.failed = True
still_open: Final = self._open.pop(trace_id, 1) - 1
if still_open > 0:
self._open[trace_id] = still_open
still_open: Final = self._open.pop(request, set())
if span.context is not None:
still_open.discard(span.context.span_id)
if still_open:
self._open[request] = still_open
return ()
return self._decide_locked(trace_id)
return self._decide_locked(request)
def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""Draw once per destination for the held tree; the spans the draws keep, and a
verdict remembered for the spans of this trace that start later."""
tree: Final = self._pending.pop(trace_id, None)
def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
"""Decide once per destination for the held tree; the spans the verdicts keep, and
a verdict remembered for the spans of this request that start later."""
tree: Final = self._pending.pop(request, None)
if tree is None:
return ()
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree
for _, destination in tree.held:
key = _verdict_key(trace_id, destination)
key = _verdict_key(request, destination)
if key not in verdicts:
verdicts[key] = _keeps_tree(destination, tree.failed, self._draw)
verdicts[key] = _keeps_tree(destination, tree.failed, request[0])
self._verdicts[key] = verdicts[key]
while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS:
self._verdicts.popitem(last=False)
return tuple(
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)]
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)]
)
def _operator_already_writes(
@ -850,8 +892,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
with self._lock:
self._draining = True
undecided: Final = tuple(self._pending)
for trace_id in undecided:
for held_span, held_destination in self._decide(trace_id):
for request in undecided:
for held_span, held_destination in self._decide(request):
self._forward(held_span, held_destination)
deadline: Final = time.monotonic() + self._drain_seconds
with self._lock:

View file

@ -22,7 +22,7 @@ _BUDGET_DURATION_WORD_ALIASES: Final[dict[str, str]] = {
}
def _normalize_duration(duration: str) -> str:
def normalize_duration(duration: str) -> str:
return _BUDGET_DURATION_WORD_ALIASES.get(duration.strip().lower(), duration)
@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int:
Returns time in seconds till when budget needs to be reset
"""
value, unit = _extract_from_regex(duration=_normalize_duration(duration))
value, unit = _extract_from_regex(duration=normalize_duration(duration))
if unit == "s":
return value
@ -89,7 +89,7 @@ def duration_in_seconds_from(start: datetime, duration: str) -> int:
month of a different length needs its own start to know when it ends. Every other
unit is a fixed length and ignores `start`.
"""
value, unit = _extract_from_regex(duration=_normalize_duration(duration))
value, unit = _extract_from_regex(duration=normalize_duration(duration))
if unit == "mo":
return _months_in_seconds(start=start, months=value)
return duration_in_seconds(duration)
@ -129,7 +129,7 @@ def get_next_standardized_reset_time(
current_time, _ = _setup_timezone(current_time, timezone_str)
# Parse duration
value, unit = _parse_duration(_normalize_duration(duration))
value, unit = _parse_duration(normalize_duration(duration))
if value is None:
verbose_logger.warning(
"Unrecognized budget_duration %r; falling back to a next-midnight reset. "
@ -171,7 +171,7 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
"""Start of the budget period that ends at `reset_at`, under the same rules
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
months and a duration it does not recognize resets at the next midnight."""
value, unit = _parse_duration(_normalize_duration(duration))
value, unit = _parse_duration(normalize_duration(duration))
if value is None:
return reset_at - timedelta(days=1)
match unit:

View file

@ -76,6 +76,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
"itpm",
"otpm",
"use_xai_oauth",
"chatgpt_auth_file",
"github_copilot_auth_type",
"github_copilot_host",
"github_copilot_user_session",

View file

@ -158,6 +158,14 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]:
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
if declared is not None:
return model.removeprefix(f"{declared}/"), declared
resolved_model, resolved_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
return resolved_model, resolved_provider
def inferred_provider(model: str | None) -> str | None:
if not model:
return None
@ -842,7 +850,9 @@ def _get_openai_compatible_provider_info(
api_base,
dynamic_api_key,
custom_llm_provider,
) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(
model, api_base, api_key, custom_llm_provider, litellm_params
)
elif custom_llm_provider == "novita":
api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai"
dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")

View file

@ -1,10 +1,23 @@
from collections.abc import Mapping, Sequence
from typing import Final
from pydantic import TypeAdapter
from litellm.types.utils import ProviderSpecificHeader
_PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter(
ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None
)
class ProviderSpecificHeaderUtils:
@staticmethod
def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None:
headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
_PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider
)
return next((value for key, value in headers.items() if key.lower() == "authorization"), None)
@staticmethod
def get_provider_specific_headers(
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,

View file

@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result(
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
last_message_with_tool_calls: dict | None,
forward_function_call_id: bool = False,
supports_multimodal_function_responses: bool = True,
) -> VertexPartType | list[VertexPartType]:
"""
OpenAI message with a tool result looks like:
@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result(
_part: Final[VertexPartType] = {"function_response": _function_response}
# For multimodal function responses, Gemini expects media parts nested
# inside functionResponse.parts instead of sibling content parts.
if inline_data_list:
if not supports_multimodal_function_responses:
return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)]
_function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list]
return [_part]

View file

@ -1,9 +1,9 @@
import base64
import json
import os
import time
from collections.abc import Mapping
from typing import Final, TypeAlias
from functools import lru_cache
from typing import TYPE_CHECKING, Final
import httpx
from pydantic import JsonValue, TypeAdapter, ValidationError
@ -24,19 +24,21 @@ from .common_utils import (
CHATGPT_OAUTH_TOKEN_URL,
GetAccessTokenError,
GetDeviceCodeError,
JsonObject,
RefreshAccessTokenError,
decode_jwt_claims,
extract_chatgpt_account_id,
)
if TYPE_CHECKING:
from litellm.types.router import GenericLiteLLMParams
TOKEN_EXPIRY_SKEW_SECONDS: Final = 60
TOKEN_REFRESH_TIMEOUT_SECONDS: Final = 30
DEVICE_CODE_TIMEOUT_SECONDS: Final = 15 * 60
DEVICE_CODE_COOLDOWN_SECONDS: Final = 5 * 60
DEVICE_CODE_POLL_SLEEP_SECONDS: Final = 5
OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth"
JsonObject: TypeAlias = Mapping[str, JsonValue]
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject)
@ -44,6 +46,26 @@ def _optional_str(value: JsonValue | None) -> str | None:
return value if isinstance(value, str) else None
def get_chatgpt_auth_file(
litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None",
) -> str | None:
if litellm_params is None:
return None
value: Final = (
litellm_params.get("chatgpt_auth_file")
if isinstance(litellm_params, Mapping)
else litellm_params.chatgpt_auth_file
)
if not isinstance(value, str) or not value:
return None
return os.path.abspath(os.path.expanduser(value))
@lru_cache(maxsize=128)
def get_cached_authenticator(auth_file: str) -> "Authenticator":
return Authenticator(auth_file=auth_file)
def _token_refresh_timeout() -> httpx.Timeout:
configured: Final = get_configured_request_timeout()
seconds: Final = (
@ -53,13 +75,13 @@ def _token_refresh_timeout() -> httpx.Timeout:
class Authenticator:
def __init__(self) -> None:
self.token_dir = os.getenv(
"CHATGPT_TOKEN_DIR",
os.path.expanduser("~/.config/litellm/chatgpt"),
def __init__(self, auth_file: str | None = None) -> None:
default_auth_file: Final = os.path.join(
os.getenv("CHATGPT_TOKEN_DIR", os.path.expanduser("~/.config/litellm/chatgpt")),
os.getenv("CHATGPT_AUTH_FILE", "auth.json"),
)
self.auth_file = os.path.join(self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json"))
self._ensure_token_dir()
self.auth_file = os.path.abspath(os.path.expanduser(auth_file or default_auth_file))
self.token_dir = os.path.dirname(self.auth_file)
def get_api_base(self) -> str:
return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE
@ -99,16 +121,18 @@ class Authenticator:
tokens: Final = self._login_device_code()
return tokens["access_token"]
def get_account_id(self) -> str | None:
def get_account_id(self, access_token: str | None = None) -> str | None:
auth_data: Final = self._read_auth_file()
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
return extract_chatgpt_account_id(access_token)
if not auth_data:
return None
account_id: Final = _optional_str(auth_data.get("account_id"))
if account_id:
return account_id
id_token: Final = auth_data.get("id_token")
access_token: Final = auth_data.get("access_token")
derived: Final = self._extract_account_id(_optional_str(id_token or access_token))
saved_access_token: Final = auth_data.get("access_token")
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
if derived:
self._write_auth_file({**auth_data, "account_id": derived})
return derived
@ -129,6 +153,7 @@ class Authenticator:
def _write_auth_file(self, data: JsonObject) -> None:
try:
self._ensure_token_dir()
with open(self.auth_file, "w") as f:
json.dump(data, f)
except OSError as exc:
@ -146,34 +171,11 @@ class Authenticator:
return time.time() >= float(derived_expires_at) - TOKEN_EXPIRY_SKEW_SECONDS
def _get_expires_at(self, token: str) -> int | None:
claims: Final = self._decode_jwt_claims(token)
exp: Final = claims.get("exp")
exp: Final = decode_jwt_claims(token).get("exp")
if isinstance(exp, (int, float)):
return int(exp)
return None
def _decode_jwt_claims(self, token: str) -> JsonObject:
try:
parts: Final = token.split(".")
if len(parts) < 2:
return {}
payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4)
payload_bytes: Final = base64.urlsafe_b64decode(payload_b64)
return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8")))
except Exception:
return {}
def _extract_account_id(self, token: str | None) -> str | None:
if not token:
return None
claims: Final = self._decode_jwt_claims(token)
auth_claims: Final = claims.get(OPENAI_AUTH_CLAIM_KEY)
if isinstance(auth_claims, dict):
account_id: Final = auth_claims.get("chatgpt_account_id")
if isinstance(account_id, str) and account_id:
return account_id
return None
def _login_device_code(self) -> dict[str, str]:
cooldown_remaining: Final = self._get_device_code_cooldown_remaining(self._read_auth_file())
if cooldown_remaining > 0:
@ -361,15 +363,16 @@ class Authenticator:
"refresh_token": _optional_str(data.get("refresh_token")) or refresh_token,
"id_token": id_token,
}
account_id: Final = self.get_account_id()
auth_data: Final = self._build_auth_record(refreshed)
self._write_auth_file(auth_data)
self._write_auth_file({**auth_data, "account_id": account_id} if account_id else auth_data)
return refreshed
def _build_auth_record(self, tokens: dict[str, str]) -> JsonObject:
access_token: Final = tokens.get("access_token")
id_token: Final = tokens.get("id_token")
expires_at: Final = self._get_expires_at(access_token) if access_token else None
account_id: Final = self._extract_account_id(id_token or access_token)
account_id: Final = extract_chatgpt_account_id(id_token or access_token)
return {
"access_token": access_token,
"refresh_token": tokens.get("refresh_token"),

View file

@ -1,14 +1,20 @@
from collections.abc import Mapping
from typing import Any, Final
from litellm.exceptions import AuthenticationError
from litellm.llms.openai.openai import OpenAIConfig
from litellm.types.llms.openai import AllMessageValues
from litellm.types.router import GenericLiteLLMParams
from ..authenticator import Authenticator
from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
from ..common_utils import (
GetAccessTokenError,
chatgpt_client_api_base,
ensure_chatgpt_session_id,
get_chatgpt_client_credential,
get_chatgpt_default_headers,
get_chatgpt_request_credential,
strip_chatgpt_client_credential_headers,
)
from .streaming_utils import ChatGPTToolCallNormalizer
@ -19,9 +25,27 @@ class ChatGPTConfig(OpenAIConfig):
api_key: str | None = None,
api_base: str | None = None,
custom_llm_provider: str = "openai",
authenticator: Authenticator | None = None,
) -> None:
super().__init__()
self.authenticator = Authenticator()
self.authenticator = authenticator if authenticator is not None else Authenticator()
def _resolve_authenticator(
self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None
) -> Authenticator:
auth_file: Final = get_chatgpt_auth_file(litellm_params)
return get_cached_authenticator(auth_file) if auth_file else self.authenticator
@staticmethod
def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str:
try:
return authenticator.get_access_token()
except GetAccessTokenError as e:
raise AuthenticationError(
model=model,
llm_provider=llm_provider,
message=str(e),
)
def api_base_without_login(self) -> str:
return self.authenticator.get_api_base()
@ -32,16 +56,16 @@ class ChatGPTConfig(OpenAIConfig):
api_base: str | None,
api_key: str | None,
custom_llm_provider: str,
litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None,
) -> tuple[str | None, str | None, str]:
dynamic_api_base: Final = self.api_base_without_login()
try:
dynamic_api_key: Final = self.authenticator.get_access_token()
except GetAccessTokenError as e:
raise AuthenticationError(
model=model,
llm_provider=custom_llm_provider,
message=str(e),
)
client_credential: Final = get_chatgpt_request_credential(litellm_params)
if client_credential is not None:
return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider
if api_key:
return api_base or self.api_base_without_login(), api_key, custom_llm_provider
authenticator: Final = self._resolve_authenticator(litellm_params)
dynamic_api_base: Final = authenticator.get_api_base()
dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider)
return dynamic_api_base, dynamic_api_key, custom_llm_provider
def get_openai_compatible_provider_info(
@ -50,26 +74,49 @@ class ChatGPTConfig(OpenAIConfig):
api_base: str | None,
api_key: str | None,
custom_llm_provider: str,
litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None,
) -> tuple[str | None, str | None, str]:
return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider, litellm_params)
def validate_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
messages: list[AllMessageValues],
optional_params: dict,
litellm_params: dict,
litellm_params: dict[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict:
) -> dict[str, str]:
client_credential: Final = get_chatgpt_client_credential(headers)
if client_credential is not None:
return dict(
get_chatgpt_default_headers(
client_credential.access_token,
client_credential.account_id,
ensure_chatgpt_session_id(litellm_params),
),
**strip_chatgpt_client_credential_headers(headers),
)
authenticator: Final = self._resolve_authenticator(litellm_params)
resolved_api_key: Final = (
self._get_access_token_or_raise(authenticator, model, "chatgpt")
if get_chatgpt_auth_file(litellm_params)
else api_key
)
validated_headers: Final = super().validate_environment(
headers, model, messages, optional_params, litellm_params, api_key, api_base
dict(strip_chatgpt_client_credential_headers(headers)),
model,
messages,
optional_params,
litellm_params,
resolved_api_key,
api_base,
)
account_id: Final = self.authenticator.get_account_id()
account_id: Final = authenticator.get_account_id(resolved_api_key)
session_id: Final = ensure_chatgpt_session_id(litellm_params)
default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)
default_headers: Final = get_chatgpt_default_headers(resolved_api_key or "", account_id, session_id)
return {**default_headers, **validated_headers}
def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:

View file

@ -2,14 +2,32 @@
Constants and helpers for ChatGPT subscription OAuth.
"""
import base64
import json
import os
import platform
from typing import Any, Final
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeAlias
from urllib.parse import urlsplit
from uuid import uuid4
import httpx
from pydantic import JsonValue, TypeAdapter
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.utils import ProviderSpecificHeader
if TYPE_CHECKING:
from litellm.types.router import GenericLiteLLMParams
_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter(
ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None
)
_STRING_HEADERS: Final = TypeAdapter(Mapping[str, str])
_PARAMS: Final = TypeAdapter(Mapping[str, object])
# OAuth + API constants (derived from openai/codex)
CHATGPT_AUTH_BASE: Final = "https://auth.openai.com"
@ -19,6 +37,12 @@ CHATGPT_OAUTH_TOKEN_URL: Final = f"{CHATGPT_AUTH_BASE}/oauth/token"
CHATGPT_DEVICE_VERIFY_URL: Final = f"{CHATGPT_AUTH_BASE}/codex/device"
CHATGPT_API_BASE: Final = "https://chatgpt.com/backend-api/codex"
CHATGPT_CLIENT_ID: Final = "app_EMoamEEZ73f0CkXaXp7hrann"
CHATGPT_ACCOUNT_ID_HEADER: Final = "ChatGPT-Account-Id"
OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth"
JsonObject: TypeAlias = Mapping[str, JsonValue]
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject)
_EMPTY_CLAIMS: Final[JsonObject] = MappingProxyType({})
DEFAULT_ORIGINATOR: Final = "codex_cli_rs"
DEFAULT_USER_AGENT: Final = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown"
@ -229,7 +253,7 @@ def get_chatgpt_default_headers(
access_token: str,
account_id: str | None,
session_id: str | None = None,
) -> dict:
) -> dict[str, str]:
originator: Final = get_chatgpt_originator()
user_agent: Final = get_chatgpt_user_agent(originator)
headers: Final = {
@ -242,30 +266,133 @@ def get_chatgpt_default_headers(
if session_id:
headers["session_id"] = session_id
if account_id:
headers["ChatGPT-Account-Id"] = account_id
headers[CHATGPT_ACCOUNT_ID_HEADER] = account_id
return headers
def decode_jwt_claims(token: str) -> JsonObject:
try:
parts: Final = token.split(".")
if len(parts) < 2:
return _EMPTY_CLAIMS
payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4)
payload_bytes: Final = base64.urlsafe_b64decode(payload_b64)
return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8")))
except Exception:
return _EMPTY_CLAIMS
def extract_chatgpt_account_id(token: str | None) -> str | None:
if not token:
return None
auth_claims: Final = decode_jwt_claims(token).get(OPENAI_AUTH_CLAIM_KEY)
if not isinstance(auth_claims, dict):
return None
account_id: Final = auth_claims.get("chatgpt_account_id")
return account_id if isinstance(account_id, str) and account_id else None
def _bearer_token(value: str) -> str:
scheme, separator, token = value.strip().partition(" ")
if not separator:
return scheme
return token.strip() if scheme.lower() == "bearer" else ""
def is_chatgpt_oauth_key(value: str | None) -> bool:
if value is None:
return False
return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(_bearer_token(value))
@dataclass(frozen=True, slots=True)
class ChatGPTClientCredential:
access_token: str
account_id: str | None
def _header_value(headers: Mapping[str, str], name: str) -> str | None:
return next((value for key, value in headers.items() if key.lower() == name), None)
def get_chatgpt_client_credential(headers: Mapping[str, str]) -> ChatGPTClientCredential | None:
auth_header: Final = _header_value(headers, "authorization")
if auth_header is None or not is_chatgpt_oauth_key(auth_header):
return None
access_token: Final = _bearer_token(auth_header)
account_id: Final = _header_value(headers, CHATGPT_ACCOUNT_ID_HEADER.lower()) or extract_chatgpt_account_id(
access_token
)
return ChatGPTClientCredential(access_token=access_token, account_id=account_id)
def select_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
if get_chatgpt_client_credential(headers) is None:
return MappingProxyType({})
return MappingProxyType(
{
key: value
for key, value in headers.items()
if (key.lower() == "authorization" and is_chatgpt_oauth_key(value))
or key.lower() == CHATGPT_ACCOUNT_ID_HEADER.lower()
}
)
def strip_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
return MappingProxyType(
{
key: value
for key, value in headers.items()
if key.lower() not in ("authorization", CHATGPT_ACCOUNT_ID_HEADER.lower())
}
)
def get_chatgpt_request_credential(
params: "Mapping[str, object] | GenericLiteLLMParams | None",
) -> ChatGPTClientCredential | None:
normalized: Final = _normalize_litellm_params(params)
scoped: Final = normalized.get("provider_specific_header")
extra: Final = normalized.get("extra_headers")
headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt"
)
return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential(
_STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({})
)
def chatgpt_client_api_base(api_base: str | None) -> str:
base: Final = api_base or CHATGPT_API_BASE
parsed: Final = urlsplit(base)
if (
parsed.scheme != "https"
or parsed.hostname != "chatgpt.com"
or parsed.port not in (None, 443)
or parsed.username
or parsed.password
):
raise ChatGPTAuthError(
status_code=400, message="Client ChatGPT credentials require the official HTTPS ChatGPT endpoint"
)
return base
def get_chatgpt_default_instructions() -> str:
return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS
def _normalize_litellm_params(litellm_params: Any | None) -> dict:
if litellm_params is None:
return {}
if isinstance(litellm_params, dict):
return litellm_params
if hasattr(litellm_params, "model_dump"):
try:
return litellm_params.model_dump()
except Exception:
return {}
if hasattr(litellm_params, "dict"):
try:
return litellm_params.dict()
except Exception:
return {}
return {}
def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]:
if isinstance(litellm_params, Mapping):
return _PARAMS.validate_python(litellm_params)
serializer: Final = getattr(litellm_params, "model_dump", None) or getattr(litellm_params, "dict", None)
if not callable(serializer):
return MappingProxyType({})
try:
return _PARAMS.validate_python(serializer())
except Exception:
return MappingProxyType({})
def get_chatgpt_session_id(litellm_params: object) -> str | None:
@ -274,8 +401,8 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None:
value = params.get(key)
if value:
return str(value)
metadata: Final = params.get("metadata")
if isinstance(metadata, dict):
metadata: Final = _normalize_litellm_params(params.get("metadata"))
if metadata:
value = metadata.get("session_id")
if value:
return str(value)

View file

@ -25,13 +25,15 @@ from litellm.types.llms.openai import (
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from ..authenticator import Authenticator
from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
from ..common_utils import (
CHATGPT_API_BASE,
GetAccessTokenError,
ensure_chatgpt_session_id,
get_chatgpt_client_credential,
get_chatgpt_default_headers,
get_chatgpt_default_instructions,
strip_chatgpt_client_credential_headers,
)
if TYPE_CHECKING:
@ -49,14 +51,35 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.CHATGPT
def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator:
auth_file: Final = get_chatgpt_auth_file(litellm_params)
return get_cached_authenticator(auth_file) if auth_file else self.authenticator
def validate_environment(
self,
headers: dict,
headers: dict[str, str],
model: str,
litellm_params: GenericLiteLLMParams | None,
) -> dict:
) -> dict[str, str]:
session_id: Final = ensure_chatgpt_session_id(litellm_params)
client_credential: Final = get_chatgpt_client_credential(headers)
if client_credential is not None:
return {
**get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id),
**strip_chatgpt_client_credential_headers(headers),
}
authenticator: Final = self._resolve_authenticator(litellm_params)
safe_headers: Final = strip_chatgpt_client_credential_headers(headers)
configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None
if configured_api_key and not get_chatgpt_auth_file(litellm_params):
return {
**get_chatgpt_default_headers(
configured_api_key, authenticator.get_account_id(configured_api_key), session_id
),
**safe_headers,
}
try:
access_token: Final = self.authenticator.get_access_token()
access_token: Final = authenticator.get_access_token()
except GetAccessTokenError as e:
raise AuthenticationError(
model=model,
@ -64,10 +87,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
message=str(e),
)
account_id: Final = self.authenticator.get_account_id()
session_id: Final = ensure_chatgpt_session_id(litellm_params)
account_id: Final = authenticator.get_account_id()
default_headers: Final = get_chatgpt_default_headers(access_token, account_id, session_id)
return {**default_headers, **headers}
return {**default_headers, **safe_headers}
def transform_responses_api_request(
self,
@ -79,7 +101,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
) -> dict:
request: Final = super().transform_responses_api_request(
model,
input,
[{"role": "user", "content": input}] if isinstance(input, str) else input,
response_api_optional_request_params,
litellm_params,
headers,

View file

@ -20,6 +20,7 @@ from aiohttp import ClientSession, DummyCookieJar, TCPConnector
from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
from httpx._types import CertTypes, RequestFiles
from httpx._utils import get_environment_proxies
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_logger
@ -566,6 +567,34 @@ async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> N
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
_TIMEOUT_PHASES: Final[Mapping[type[httpx.TimeoutException], tuple[str, str]]] = MappingProxyType(
{
httpx.ConnectTimeout: ("Connection", "connect"),
httpx.ReadTimeout: ("Read", "read"),
httpx.WriteTimeout: ("Write", "write"),
httpx.PoolTimeout: ("Connection pool", "pool"),
}
)
_UNKNOWN_TIMEOUT_PHASE: Final = ("Request", "read")
_TIMEOUT_EXTENSION: Final = TypeAdapter(dict[str, float | None])
def _timeout_phase_seconds(error: httpx.TimeoutException, phase: str) -> float | None:
"""The limit httpx enforced for ``phase`` on the request that raised ``error``, from the request itself."""
return _TIMEOUT_EXTENSION.validate_python(error.request.extensions["timeout"]).get(phase)
def _raise_timeout(error: httpx.TimeoutException, started_at: float) -> NoReturn:
label, phase = _TIMEOUT_PHASES.get(type(error), _UNKNOWN_TIMEOUT_PHASE)
seconds: Final = _timeout_phase_seconds(error, phase)
time_taken: Final = round(time.time() - started_at, 3)
raise litellm.Timeout(
message=f"{label} timed out. Timeout passed={seconds}, time taken={time_taken} seconds",
model="default-model-name",
llm_provider="litellm-httpx-handler",
)
class HTTPResponseLimitError(ValueError):
pass
@ -870,20 +899,7 @@ class AsyncHTTPHandler:
finally:
await new_client.aclose()
except httpx.TimeoutException as e:
end_time: Final = time.time()
time_delta: Final = round(end_time - start_time, 3)
headers = {}
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
raise litellm.Timeout(
message=f"Connection timed out. Timeout passed={timeout}, time taken={time_delta} seconds",
model="default-model-name",
llm_provider="litellm-httpx-handler",
headers=headers,
)
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
await _raise_masked_async_error(e, stream)
except Exception as e:
@ -902,6 +918,7 @@ class AsyncHTTPHandler:
follow_redirects: bool | None = None,
):
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
start_time: Final = time.time()
try:
if timeout is None:
timeout = self.timeout
@ -946,18 +963,7 @@ class AsyncHTTPHandler:
finally:
await new_client.aclose()
except httpx.TimeoutException as e:
headers = {}
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
headers=headers,
)
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
await _raise_masked_async_error(e, stream)
except Exception as e:
@ -974,6 +980,7 @@ class AsyncHTTPHandler:
stream: bool = False,
content: _RequestContent | None = None,
):
start_time: Final = time.time()
try:
if timeout is None:
timeout = self.timeout
@ -1012,18 +1019,7 @@ class AsyncHTTPHandler:
finally:
await new_client.aclose()
except httpx.TimeoutException as e:
headers = {}
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
if error_response is not None:
for key, value in error_response.headers.items():
headers[f"response_headers-{key}"] = value
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
headers=headers,
)
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
await _raise_masked_async_error(e, stream)
except Exception as e:
@ -1514,6 +1510,7 @@ class HTTPHandler:
content: _RequestContent | None = None,
logging_obj: LiteLLMLoggingObject | None = None,
):
start_time: Final = time.time()
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
request_data, request_content = _prepare_request_data_and_content(data, content)
@ -1546,12 +1543,8 @@ class HTTPHandler:
_anchor_handler_to(response, self)
response.raise_for_status()
return response
except httpx.TimeoutException:
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
)
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
_raise_masked_sync_error(e, stream)
except Exception as e:
@ -1568,6 +1561,7 @@ class HTTPHandler:
timeout: float | httpx.Timeout | None = None,
content: _RequestContent | None = None,
):
start_time: Final = time.time()
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
request_data, request_content = _prepare_request_data_and_content(data, content)
@ -1598,12 +1592,8 @@ class HTTPHandler:
_anchor_handler_to(response, self)
response.raise_for_status()
return response
except httpx.TimeoutException:
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
)
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
_raise_masked_sync_error(e, stream)
except Exception as e:
@ -1620,6 +1610,7 @@ class HTTPHandler:
timeout: float | httpx.Timeout | None = None,
content: _RequestContent | None = None,
):
start_time: Final = time.time()
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
request_data, request_content = _prepare_request_data_and_content(data, content)
@ -1649,12 +1640,8 @@ class HTTPHandler:
if stream:
_anchor_handler_to(response, self)
return response
except httpx.TimeoutException:
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
)
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
_raise_masked_sync_error(e, stream)
except Exception as e:
@ -1671,6 +1658,7 @@ class HTTPHandler:
stream: bool = False,
content: _RequestContent | None = None,
):
start_time: Final = time.time()
try:
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
request_data, request_content = _prepare_request_data_and_content(data, content)
@ -1701,12 +1689,8 @@ class HTTPHandler:
_anchor_handler_to(response, self)
response.raise_for_status()
return response
except httpx.TimeoutException:
raise litellm.Timeout(
message=f"Connection timed out after {timeout} seconds.",
model="default-model-name",
llm_provider="litellm-httpx-handler",
)
except httpx.TimeoutException as e:
_raise_timeout(e, start_time)
except httpx.HTTPStatusError as e:
_raise_masked_sync_error(e, stream)
except Exception as e:

View file

@ -198,6 +198,26 @@ from litellm.utils import (
)
def _merge_headers_case_insensitive(
headers: Mapping[str, object], extra_headers: Mapping[str, object] | None
) -> dict[str, str]: # mutable-ok: legacy HTTP handler methods require a dict of headers
overridden: Final = frozenset(name.lower() for name in extra_headers or {})
return {
**{name: str(value) for name, value in headers.items() if name.lower() not in overridden},
**{name: str(value) for name, value in (extra_headers or {}).items()},
}
def _client_request_headers(
response_api_optional_request_params: Mapping[str, object],
extra_headers: Mapping[str, object] | None,
) -> dict[str, str]: # mutable-ok: provider validate_environment methods require a dict
from litellm.litellm_core_utils.core_helpers import as_str_mapping
optional_headers: Final = as_str_mapping(response_api_optional_request_params.get("extra_headers"))
return _merge_headers_case_insensitive(optional_headers or {}, extra_headers)
def _rust_responses_websocket_enabled(
custom_llm_provider: str | None,
) -> bool:
@ -2906,14 +2926,18 @@ class BaseLLMHTTPHandler:
else:
sync_httpx_client = client
headers = responses_api_provider_config.validate_environment(
headers=response_api_optional_request_params.get("extra_headers", {}) or {},
model=model,
litellm_params=litellm_params,
)
if extra_headers:
headers.update(extra_headers)
headers = { # rebind-ok: validation replaces the method's incoming headers
**responses_api_provider_config.validate_environment(
headers=(
_client_request_headers(response_api_optional_request_params, extra_headers)
if custom_llm_provider == "chatgpt"
else response_api_optional_request_params.get("extra_headers", {}) or {}
),
model=model,
litellm_params=litellm_params,
),
**((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}),
}
# Check if streaming is requested
stream = response_api_optional_request_params.get("stream", False)
@ -3104,11 +3128,15 @@ class BaseLLMHTTPHandler:
validated_headers: Final = {
**await asyncio.to_thread(
responses_api_provider_config.validate_environment,
headers=response_api_optional_request_params.get("extra_headers", {}) or {},
headers=(
_client_request_headers(response_api_optional_request_params, extra_headers)
if custom_llm_provider == "chatgpt"
else response_api_optional_request_params.get("extra_headers", {}) or {}
),
model=model,
litellm_params=litellm_params,
),
**(extra_headers or {}),
**((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}),
}
# Check if streaming is requested

View file

@ -56,9 +56,9 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
# Response format mappings from OpenAI to ElevenLabs
FORMAT_MAPPINGS = {
"mp3": "mp3_44100_128",
"pcm": "pcm_44100",
"pcm": "pcm_24000",
"opus": "opus_48000_128",
# ElevenLabs does not support WAV, AAC, or FLAC formats.
"wav": "wav_24000",
}
ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__"

View file

@ -90,7 +90,7 @@ class PassThroughEndpointHandler(BaseTranslation):
payload_to_check: Final = {
k: v
for k, v in without_server_streaming_classification(data).items()
if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj")
if not k.startswith("_") and k not in ("metadata", "litellm_metadata", "litellm_logging_obj")
}
verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail")
return safe_dumps(payload_to_check)

View file

@ -726,6 +726,9 @@ def gemini_convert_messages_with_history(
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "")
supports_multimodal_function_responses: Final = model is None or (
VertexGeminiConfig.supports_multimodal_function_responses(model)
)
try:
while msg_i < len(messages):
@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history(
messages[msg_i],
last_message_with_tool_calls,
forward_function_call_id=forward_function_call_id,
supports_multimodal_function_responses=supports_multimodal_function_responses,
)
msg_i += 1
# Handle both single part and list of parts (for Computer Use with images)

View file

@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
) -> bool:
return cls._forward_gemini_function_call_id(model)
@classmethod
def supports_multimodal_function_responses(cls, model: str) -> bool:
return cls.is_gemini_3_or_newer(model)
def _supports_penalty_parameters(self, model: str) -> bool:
# Gemini 3 models do not support penalty parameters
if VertexGeminiConfig._is_gemini_3_or_newer(model):

View file

@ -7,12 +7,14 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S
import asyncio
import json
import threading
from collections import OrderedDict
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
from urllib.parse import urlparse
import litellm
from litellm._logging import verbose_logger
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
@ -70,16 +72,20 @@ class _VertexCredentialsObject(Protocol):
def refresh(self, request: object) -> None: ...
_VertexCredentialCacheKey = tuple[str | None, str | None]
class VertexBase:
def __init__(self) -> None:
super().__init__()
self.access_token: str | None = None
self.refresh_token: str | None = None
self._credentials: GoogleCredentialsObject | None = None
self._credentials_project_mapping: dict[
tuple[VERTEX_CREDENTIALS_TYPES | None, str | None],
self._credentials_project_mapping: OrderedDict[
_VertexCredentialCacheKey,
tuple[_VertexCredentialsObject, str | None],
] = {}
] = OrderedDict()
self._credentials_cache_lock = threading.RLock()
self.project_id: str | None = None
self.async_handler: AsyncHTTPHandler | None = None
# Per-credential-key asyncio.Lock for single-flight async refresh.
@ -91,10 +97,10 @@ class VertexBase:
# each lock; the entry is pruned when the count reaches zero, so the
# dict stays bounded even in long-running high-cardinality deployments
# without depending on any private asyncio internals.
self._async_refresh_locks: dict[tuple, asyncio.Lock] = {}
self._async_refresh_lock_refcounts: dict[tuple, int] = {}
self._async_refresh_locks: dict[_VertexCredentialCacheKey, asyncio.Lock] = {}
self._async_refresh_lock_refcounts: dict[_VertexCredentialCacheKey, int] = {}
# Tracks in-flight background refresh tasks to avoid duplicate refreshes.
self._background_refresh_tasks: dict[tuple, asyncio.Task] = {}
self._background_refresh_tasks: dict[_VertexCredentialCacheKey, asyncio.Task] = {}
# Protects the sync get_access_token refresh path.
# Use RLock so that the reauthentication retry path (which calls
# back into get_access_token while still holding the lock) can
@ -378,7 +384,7 @@ class VertexBase:
with self._sync_refresh_lock:
credentials.refresh(Request())
def _acquire_async_refresh_lock(self, credential_cache_key: tuple) -> asyncio.Lock:
def _acquire_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey) -> asyncio.Lock:
"""Increment the refcount and return the lock for ``credential_cache_key``.
Every call must be paired with ``_release_async_refresh_lock`` once the
@ -391,7 +397,7 @@ class VertexBase:
)
return lock
def _release_async_refresh_lock(self, credential_cache_key: tuple, lock: asyncio.Lock) -> None:
def _release_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey, lock: asyncio.Lock) -> None:
"""Decrement the refcount and drop the lock entry when it reaches zero.
Must be called only after the caller has released ``lock`` (i.e. once
@ -409,7 +415,7 @@ class VertexBase:
def _try_get_cached_token(
self,
credential_cache_key: tuple,
credential_cache_key: _VertexCredentialCacheKey,
project_id: str | None,
) -> tuple[str, str] | None:
"""
@ -432,7 +438,7 @@ class VertexBase:
def _try_get_usable_cached_token(
self,
credential_cache_key: tuple,
credential_cache_key: _VertexCredentialCacheKey,
project_id: str | None,
) -> tuple[str, str, "TokenState", _VertexCredentialsObject, str | None] | None:
"""
@ -458,19 +464,37 @@ class VertexBase:
return creds.token, resolved_project, token_state, creds, cached_project_id
def _unpack_cached_credentials(
self, credential_cache_key: tuple
self, credential_cache_key: _VertexCredentialCacheKey
) -> tuple[_VertexCredentialsObject | None, str | None]:
"""
Return (credentials, project_id) from the cache, or (None, None) if
not cached. Handles both tuple and legacy cache formats.
"""
if credential_cache_key not in self._credentials_project_mapping:
return None, None
cached_entry: Final = self._credentials_project_mapping[credential_cache_key]
with self._credentials_cache_lock:
cached_entry: Final = self._credentials_project_mapping.get(credential_cache_key)
if cached_entry is None:
return None, None
self._credentials_project_mapping.move_to_end(credential_cache_key)
if isinstance(cached_entry, tuple):
return cached_entry
return cached_entry, cached_entry.quota_project_id or getattr(cached_entry, "project_id", None)
def _cache_credentials(
self,
credential_cache_key: _VertexCredentialCacheKey,
credentials: _VertexCredentialsObject,
project_id: str | None,
*,
replace: bool = True,
) -> None:
with self._credentials_cache_lock:
if not replace and credential_cache_key in self._credentials_project_mapping:
return
self._credentials_project_mapping[credential_cache_key] = (credentials, project_id)
self._credentials_project_mapping.move_to_end(credential_cache_key)
while len(self._credentials_project_mapping) > max(0, DEFAULT_MAX_LRU_CACHE_SIZE):
self._credentials_project_mapping.popitem(last=False)
def _get_token_state(self, credentials: _VertexCredentialsObject) -> "TokenState":
"""
Return the token state using google-auth's TokenState enum.
@ -494,7 +518,7 @@ class VertexBase:
self,
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
credential_cache_key: tuple,
credential_cache_key: _VertexCredentialCacheKey,
) -> tuple[_VertexCredentialsObject, str | None]:
"""Load credentials via load_auth (in thread) and cache the result."""
try:
@ -507,17 +531,12 @@ class VertexBase:
raise
if _credentials is None:
raise ValueError("Could not resolve credentials")
self._credentials_project_mapping[credential_cache_key] = (
_credentials,
credential_project_id,
)
self._cache_credentials(credential_cache_key, _credentials, credential_project_id)
return _credentials, credential_project_id
async def _background_refresh_credentials(
self,
credentials: _VertexCredentialsObject,
credential_cache_key: tuple,
credential_project_id: str | None,
) -> None:
"""
Refresh credentials in the background without blocking the calling request.
@ -528,25 +547,13 @@ class VertexBase:
try:
verbose_logger.debug("Background proactive credential refresh")
await asyncify(self.refresh_auth)(credentials)
# Only update the cache if it still points at the credentials
# object we just refreshed. The per-key async lock is not held
# here, so a concurrent INVALID path may have already replaced
# this entry (e.g. via _handle_reauthentication_async, which
# creates a fresh credentials object). In that case our write
# would clobber the newer entry with a stale reference.
cached_creds, _ = self._unpack_cached_credentials(credential_cache_key)
if cached_creds is credentials:
self._credentials_project_mapping[credential_cache_key] = (
credentials,
credential_project_id,
)
except Exception:
verbose_logger.debug(
"Background credential refresh failed, will retry on next request",
exc_info=True,
)
async def _await_in_flight_background_refresh(self, credential_cache_key: tuple) -> None:
async def _await_in_flight_background_refresh(self, credential_cache_key: _VertexCredentialCacheKey) -> None:
"""Wait for an in-flight background refresh to finish, if any.
google-auth's ``Credentials.refresh()`` is not safe to invoke
@ -568,8 +575,7 @@ class VertexBase:
def _schedule_background_refresh(
self,
credentials: _VertexCredentialsObject,
credential_cache_key: tuple,
credential_project_id: str | None,
credential_cache_key: _VertexCredentialCacheKey,
) -> None:
"""Kick off a single background refresh for ``credential_cache_key``.
@ -581,9 +587,7 @@ class VertexBase:
if existing is not None and not existing.done():
return
self._background_refresh_tasks.pop(credential_cache_key, None)
task: Final = asyncio.create_task(
self._background_refresh_credentials(credentials, credential_cache_key, credential_project_id)
)
task: Final = asyncio.create_task(self._background_refresh_credentials(credentials))
def _drop_background_refresh_task(_fut: asyncio.Future[None]) -> None:
if self._background_refresh_tasks.get(credential_cache_key) is _fut:
@ -796,7 +800,7 @@ class VertexBase:
self,
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
credential_cache_key: tuple,
credential_cache_key: _VertexCredentialCacheKey,
error: Exception,
) -> tuple[str, str]:
"""
@ -822,8 +826,8 @@ class VertexBase:
)
# Clear the cached credentials
if credential_cache_key in self._credentials_project_mapping:
del self._credentials_project_mapping[credential_cache_key]
with self._credentials_cache_lock:
self._credentials_project_mapping.pop(credential_cache_key, None)
# Retry once with _retry_reauth=True to prevent infinite recursion
try:
@ -846,7 +850,7 @@ class VertexBase:
self,
credentials: VERTEX_CREDENTIALS_TYPES | None,
project_id: str | None,
credential_cache_key: tuple,
credential_cache_key: _VertexCredentialCacheKey,
error: Exception,
) -> tuple[str, str]:
"""
@ -856,7 +860,8 @@ class VertexBase:
"Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id
)
self._credentials_project_mapping.pop(credential_cache_key, None)
with self._credentials_cache_lock:
self._credentials_project_mapping.pop(credential_cache_key, None)
try:
(
@ -875,10 +880,7 @@ class VertexBase:
# references the OLD credentials object we just replaced, and
# leaving it would force the next request to do a redundant
# refresh/reauth before realizing the cached creds are stale.
self._credentials_project_mapping[resolved_cache_key] = (
_credentials,
credential_project_id,
)
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id)
if _credentials.token is None or not isinstance(_credentials.token, str):
raise ValueError(
@ -924,21 +926,11 @@ class VertexBase:
# Convert dict credentials to string for caching
cache_credentials: Final = json.dumps(credentials) if isinstance(credentials, dict) else credentials
credential_cache_key: Final = (cache_credentials, project_id)
_credentials: _VertexCredentialsObject | None = None
verbose_logger.debug("Checking cached credentials for project_id: %s", project_id)
if credential_cache_key in self._credentials_project_mapping:
_credentials, credential_project_id = self._unpack_cached_credentials(credential_cache_key)
if _credentials is not None:
verbose_logger.debug("Cached credentials found for project_id: %s.", project_id)
# Retrieve both credentials and cached project_id
cached_entry: Final = self._credentials_project_mapping[credential_cache_key]
verbose_logger.debug("cached_entry: %s", cached_entry)
if isinstance(cached_entry, tuple):
_credentials, credential_project_id = cached_entry
else:
# Backward compatibility with old cache format
_credentials = cached_entry
credential_project_id = _credentials.quota_project_id or getattr(_credentials, "project_id", None)
verbose_logger.debug(
"Using cached credentials for project_id: %s",
credential_project_id,
@ -963,10 +955,7 @@ class VertexBase:
f"Could not resolve credentials - either dynamically or from environment, for project_id: {project_id}"
)
# Cache the project_id and credentials from load_auth result (resolved project_id)
self._credentials_project_mapping[credential_cache_key] = (
_credentials,
credential_project_id,
)
self._cache_credentials(credential_cache_key, _credentials, credential_project_id)
## VALIDATE CREDENTIALS
verbose_logger.debug("Validating credentials")
@ -974,11 +963,7 @@ class VertexBase:
project_id = credential_project_id
# Update cache with resolved project_id for future lookups
resolved_cache_key: Final = (cache_credentials, project_id)
if resolved_cache_key not in self._credentials_project_mapping:
self._credentials_project_mapping[resolved_cache_key] = (
_credentials,
credential_project_id,
)
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id, replace=False)
# Check if credentials are None before accessing attributes
if _credentials is None:
@ -991,10 +976,6 @@ class VertexBase:
try:
verbose_logger.debug("Credentials expired, refreshing")
self.refresh_auth(_credentials)
self._credentials_project_mapping[credential_cache_key] = (
_credentials,
credential_project_id,
)
except Exception as e:
# if refresh fails, it's possible the user has re-authenticated via `gcloud auth application-default login`
# in this case, we should try to reload the credentials by clearing the cache and retrying
@ -1043,9 +1024,9 @@ class VertexBase:
# callers on the lock just to schedule that refresh.
usable: Final = self._try_get_usable_cached_token(credential_cache_key, project_id)
if usable is not None:
cached_token, resolved_project, token_state, creds, cached_project_id = usable
cached_token, resolved_project, token_state, creds, _cached_project_id = usable
if token_state == TokenState.STALE:
self._schedule_background_refresh(creds, credential_cache_key, cached_project_id)
self._schedule_background_refresh(creds, credential_cache_key)
return cached_token, resolved_project
# === SLOW PATH (per-key lock) ===
@ -1075,10 +1056,7 @@ class VertexBase:
# reauth that only repopulated the unresolved key), which
# would force the next request through an unnecessary
# refresh/reauth cycle.
self._credentials_project_mapping[resolved_cache_key] = (
_credentials,
credential_project_id,
)
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id)
# Use google-auth's token_state to decide refresh strategy:
# - STALE: token is usable but within REFRESH_THRESHOLD (3:45) of
@ -1098,7 +1076,6 @@ class VertexBase:
self._schedule_background_refresh(
_credentials,
credential_cache_key,
credential_project_id,
)
return current_token, project_id
@ -1117,10 +1094,6 @@ class VertexBase:
try:
verbose_logger.debug("Credentials expired, refreshing")
await asyncify(self.refresh_auth)(_credentials)
self._credentials_project_mapping[credential_cache_key] = (
_credentials,
credential_project_id,
)
except Exception as e:
if "Reauthentication is needed" in str(e):
verbose_logger.debug("Reauthentication needed, clearing cache and retrying")

View file

@ -677,7 +677,11 @@ async def acompletion(
}
if custom_llm_provider is None:
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
"dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs}
"dict[str, object]",
{
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
},
)
_, custom_llm_provider, _, _ = get_llm_provider(
model=model,
@ -5678,7 +5682,10 @@ def completion(
if deployment_id is not None: # azure llms
model = deployment_id
custom_llm_provider = "azure"
_supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs}
_supplemental_provider_params: Final = {
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
"extra_headers": extra_headers,
}
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
model=model,
custom_llm_provider=custom_llm_provider,
@ -5998,6 +6005,7 @@ def completion(
tpm=kwargs.get("tpm"),
rpm=kwargs.get("rpm"),
use_xai_oauth=kwargs.get("use_xai_oauth", False),
chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"),
gigachat_scope=kwargs.get("gigachat_scope"),
gigachat_auth_url=kwargs.get("gigachat_auth_url"),
gigachat_access_token=kwargs.get("gigachat_access_token"),
@ -9140,6 +9148,7 @@ async def ahealth_check(
custom_llm_provider=custom_llm_provider_from_params,
api_base=api_base_from_params,
api_key=api_key_from_params,
litellm_params=GenericLiteLLMParams(**model_params),
)
model_params["cache"] = {"no-cache": True} # don't used cached responses for making health check calls

File diff suppressed because it is too large Load diff

View file

@ -27,6 +27,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import (
get_passthrough_resource_metadata_url,
get_passthrough_www_authenticate,
get_request_base_url,
get_route_relative_request_path,
well_known_root_suffix,
)
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (
@ -387,7 +388,7 @@ def _gateway_dcr_challenge(
"""The RFC 9728 challenge pointing the client at the protected-resource metadata
matching the scope it requested: the per-server document (same URL spelling the
request arrived on) when the single target advertises gateway-owned sign-in,
else the gateway's aggregate document. Either way the client discovers the gateway
else a gateway document containing the public resource. Either way the client discovers the gateway
as its authorization server and starts the same sign-in flow.
``invalid_token`` adds the RFC 6750 error code for a request that DID
@ -397,10 +398,13 @@ def _gateway_dcr_challenge(
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
target: Final = _gateway_dcr_challenge_target(route, mcp_servers, client_ip)
server_scope: Final = gateway_dcr_challenge_scope(route, mcp_servers, client_ip) if target is None else None
public_path: Final = get_route_relative_request_path(request.scope).rstrip("/")
aggregate_resource: Final = public_path == "/mcp" or public_path.startswith("/mcp/")
metadata_suffix: Final = f"{well_known_root_suffix()}/mcp" if aggregate_resource else ""
resource_metadata_url: Final = (
get_passthrough_resource_metadata_url(request.scope, target)
if target is not None
else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp"
else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{metadata_suffix}"
)
error_attr: Final = 'error="invalid_token", ' if invalid_token else ""
scope_attr: Final = f', scope="{server_scope}"' if server_scope is not None else ""

View file

@ -472,15 +472,15 @@ def redeem_passthrough_authorization_code(
return sealed
def _session_cookie_user_id(request: Request) -> str | None:
async def _session_cookie_user_id(request: Request) -> str | None:
"""The signed-in litellm user for a browser request, or ``None``. Thin wrapper so the
aggregate DCR flow's verbs receive the identity as a plain value instead of parsing
cookies themselves."""
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load
user_id_from_session_cookie,
get_authenticated_browser_user_id,
)
return user_id_from_session_cookie(request)
return await get_authenticated_browser_user_id(request)
def _redirect_to_litellm_login(request: Request) -> RedirectResponse:
@ -996,17 +996,13 @@ async def _resolve_oauth_authorization_user(
enforce_binding: bool,
) -> str | RedirectResponse:
"""Resolve the authorization subject without replacing denied credentials with cookie grants."""
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # proxy import cycle
user_id_from_session_cookie,
)
use_gateway_credential: Final = enforce_binding and await oauth_authorization_uses_gateway_credential(request)
request_user_id: Final = (
await authorize_oauth_credential_request(request, mcp_server.server_id) if use_gateway_credential else None
)
if use_gateway_credential and request_user_id is None:
return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
user_id: Final = request_user_id or user_id_from_session_cookie(request)
user_id: Final = request_user_id or await _session_cookie_user_id(request)
if user_id is None:
return _redirect_to_litellm_login(request)
if not await _user_can_reach_mcp_server(user_id, mcp_server.server_id):
@ -1024,7 +1020,7 @@ async def _managed_authorization_scope(
request
)
session_user: Final = (
_session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None
await _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None
)
scope_user_id: Final = authorized_user_id or (
await authorize_oauth_credential_request(request, server.server_id)
@ -2359,7 +2355,7 @@ async def authorize_mcp_session(
code_challenge=code_challenge,
code_challenge_method=code_challenge_method,
response_type=response_type,
session_user_id=_session_cookie_user_id(request),
session_user_id=await _session_cookie_user_id(request),
resource=resource,
scope=scope,
)
@ -2391,7 +2387,7 @@ async def authorize(
code_challenge=code_challenge,
code_challenge_method=code_challenge_method,
response_type=response_type,
session_user_id=_session_cookie_user_id(request),
session_user_id=await _session_cookie_user_id(request),
lookup_consent_teams=lookup_consent_teams,
)
return aggregate_authorize(
@ -2402,7 +2398,7 @@ async def authorize(
code_challenge=code_challenge,
code_challenge_method=code_challenge_method,
response_type=response_type,
session_user_id=_session_cookie_user_id(request),
session_user_id=await _session_cookie_user_id(request),
resource=resource,
scope=scope,
)
@ -2560,7 +2556,7 @@ async def authorize_flow(request: Request, flow: str) -> Response:
return await describe_connect_flow(
request=request,
flow_handle=flow,
session_user_id=_session_cookie_user_id(request),
session_user_id=await _session_cookie_user_id(request),
lookup_vendor_credential=_vendor_credential_state,
lookup_server_reachability=_user_can_reach_mcp_server,
)
@ -2586,7 +2582,7 @@ async def authorize_complete(
return await complete_connect_flow(
request=request,
flow_handle=flow,
session_user_id=_session_cookie_user_id(request),
session_user_id=await _session_cookie_user_id(request),
cache=user_api_key_cache,
delivery=delivery,
team_id=team_id,

View file

@ -3,8 +3,8 @@
The aggregate listing deliberately keeps serving the healthy subset when one server fails, but a
failed server must contribute a classified outcome instead of silently shrinking the list: an empty
contribution with no signal makes a broken upstream indistinguishable from a healthy server with no
tools. Outcomes carry only machine fields (category and status code) so nothing from an upstream
body crosses the trust boundary; classification is total, so any exception out of a server fetch
tools. Outcomes carry machine fields and controlled diagnostics so nothing from an upstream body
crosses the trust boundary; classification is total, so any exception out of a server fetch
becomes an outcome, never a second failure.
"""
@ -21,8 +21,10 @@ from urllib.request import parse_http_list, parse_keqv_list
import anyio
import httpx
import httpx2
from mcp import MCPError
from mcp.types import CONNECTION_CLOSED
from mcp.types import Tool as MCPTool
from pydantic import ConfigDict, TypeAdapter
from pydantic import ConfigDict, PositiveInt, TypeAdapter
from typing_extensions import assert_never
from litellm._logging import verbose_logger
@ -65,6 +67,16 @@ class ServerListFault(LiteLLMBaseModel):
model_config = ConfigDict(frozen=True)
tag: ListFaultCategory
status_code: int | None = None
sse_event_limit_bytes: PositiveInt | None = None
@property
def public_message(self) -> str | None:
if self.sse_event_limit_bytes is None:
return None
return (
f"Upstream MCP SSE event exceeded the {self.sse_event_limit_bytes} byte per-event limit. "
"Paginate tools/list responses so each event stays below the limit."
)
ServerOutcome: TypeAlias = ServerListOk | ServerListFault
@ -155,6 +167,18 @@ def raise_classified_list_failure(
raise MCPServerListError(classify_list_exception(exc), server_name) from exc
def _sse_event_limit(exc: BaseException) -> int | None:
"""The SDK replaces decoder exceptions with CONNECTION_CLOSED and a fixed diagnostic."""
if not isinstance(exc, (httpx2.SSEError, MCPError)):
return None
if isinstance(exc, MCPError) and exc.error.code != CONNECTION_CLOSED:
return None
prefix: Final = "SSE stream failed: " if isinstance(exc, MCPError) else ""
message: Final = exc.error.message if isinstance(exc, MCPError) else str(exc)
matched: Final = re.fullmatch(prefix + r"Server-sent event exceeded the ([1-9][0-9]{0,18}) byte limit\.", message)
return int(matched[1]) if matched is not None else None
def classify_list_exception(exc: BaseException) -> ServerListFault:
"""Classify a per-server listing failure into exactly one outcome. Total: an exception this
function cannot recognize is the gateway's own fault (``internal``), never a re-raise."""
@ -179,6 +203,11 @@ def classify_list_exception(exc: BaseException) -> ServerListFault:
response: Final = _find_upstream_response(exc)
if response is not None:
return ServerListFault(tag="upstream_error", status_code=response.status_code)
sse_event_limit: Final = next(
(limit for error in iter_exception_tree(exc) if (limit := _sse_event_limit(error)) is not None), None
)
if sse_event_limit is not None:
return ServerListFault(tag="upstream_error", sse_event_limit_bytes=sse_event_limit)
if isinstance(exc, (httpx.TimeoutException, httpx2.TimeoutException)):
return ServerListFault(tag="timeout")
if isinstance(exc, (httpx.TransportError, httpx2.TransportError)):
@ -295,7 +324,7 @@ def listing_failed_outright(outcomes: Mapping[str, ServerOutcome]) -> bool:
def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
"""The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST
response: category plus status code only, never upstream prose or URLs."""
response: category, status code and controlled diagnostics, never upstream prose or URLs."""
match outcome.tag:
case "ok":
return {"status": "ok", "tool_count": outcome.tool_count}
@ -303,6 +332,15 @@ def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
return {
"status": outcome.tag,
**({"http_status": outcome.status_code} if outcome.status_code is not None else {}),
**(
{
"reason": "sse_event_too_large",
"limit_bytes": outcome.sse_event_limit_bytes,
"message": outcome.public_message,
}
if outcome.sse_event_limit_bytes is not None
else {}
),
}
case _:
assert_never(outcome.tag)

View file

@ -957,7 +957,7 @@ if MCP_AVAILABLE:
status_code=list_fault_http_status(fault),
detail={
"error": fault.tag,
"message": f"Failed to list tools from server {get_server_prefix(server)}",
"message": fault.public_message or f"Failed to list tools from server {get_server_prefix(server)}",
},
) from e
except Exception as e:

View file

@ -222,7 +222,7 @@ def _select_model_by_priority(
input_cost = info.get("input_cost_per_token") or 0.0
output_cost = info.get("output_cost_per_token") or 0.0
total_cost = input_cost + output_cost
max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0
max_output = info.get("max_output_tokens") or 0
output_tps = info.get("output_tokens_per_second") or 0.0
scored.append(
_ScoredModel(

View file

@ -33,9 +33,8 @@ class OAuthScopeResponse:
self._request_path = request_path
self._ready = asyncio.Event()
self._lock = asyncio.Lock()
self._start: Message | None = None
self._challenge: MCPUpstreamAuthError | None = None
self._refused = False
self._closed = False
@property
def pending(self) -> bool:
@ -50,31 +49,33 @@ class OAuthScopeResponse:
self._ready.set()
async def __call__(self, message: Message) -> None:
if message["type"] == "http.response.start":
self._start = message
if message["status"] != 200:
self.allow()
return
await self._ready.wait()
async with self._lock:
if self._refused:
if self._closed:
return
if self._start is not None:
start: Final = self._start
self._start = None
if self._challenge is not None:
error: Final = self._challenge.to_http_exception(self._base_url, self._request_path)
response: Final = JSONResponse(
status_code=error.status_code, content={"detail": error.detail}, headers=error.headers
)
self._refused = True
await self._send(
{"type": "http.response.start", "status": response.status_code, "headers": response.raw_headers}
)
await self._send({"type": "http.response.body", "body": bytes(response.body)})
return
await self._send(start)
await self._send(message)
try:
if message["type"] == "http.response.start":
if message["status"] != 200:
self.allow()
await self._ready.wait()
if self._challenge is not None:
error: Final = self._challenge.to_http_exception(self._base_url, self._request_path)
response: Final = JSONResponse(
status_code=error.status_code, content={"detail": error.detail}, headers=error.headers
)
self._closed = True
await self._send(
{
"type": "http.response.start",
"status": response.status_code,
"headers": response.raw_headers,
}
)
await self._send({"type": "http.response.body", "body": bytes(response.body)})
return
await self._send(message)
except BaseException:
self._closed = True
raise
def scope_response_from_request(request: object) -> OAuthScopeResponse | None:

View file

@ -207,6 +207,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.management_endpoints.config_override_endpoints",
path_prefixes=("/config_overrides",),
),
LazyFeature(
name="codex_passthrough",
module_path="litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints",
path_prefixes=("/backend-api/codex/",),
),
LazyFeature(
name="llm_passthrough",
module_path="litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints",

View file

@ -8673,6 +8673,184 @@
}
}
},
"codex_passthrough": {
"components": {
"schemas": {
"HTTPValidationError": {
"properties": {
"detail": {
"items": {
"$ref": "#/components/schemas/ValidationError"
},
"title": "Detail",
"type": "array"
}
},
"title": "HTTPValidationError",
"type": "object"
},
"ValidationError": {
"properties": {
"ctx": {
"title": "Context",
"type": "object"
},
"input": {
"title": "Input"
},
"loc": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"title": "Location",
"type": "array"
},
"msg": {
"title": "Message",
"type": "string"
},
"type": {
"title": "Error Type",
"type": "string"
}
},
"required": [
"loc",
"msg",
"type"
],
"title": "ValidationError",
"type": "object"
}
}
},
"paths": {
"/backend-api/codex/{endpoint}": {
"get": {
"description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.",
"operationId": "codex_passthrough_route_backend_api_codex__endpoint__get",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"enum": [
"models",
"responses"
],
"title": "Endpoint",
"type": "string"
}
}
],
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Codex Passthrough Route",
"tags": [
"codex_passthrough"
]
},
"post": {
"description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.",
"operationId": "codex_passthrough_route_backend_api_codex__endpoint__post",
"parameters": [
{
"in": "path",
"name": "endpoint",
"required": true,
"schema": {
"enum": [
"models",
"responses"
],
"title": "Endpoint",
"type": "string"
}
}
],
"requestBody": {
"content": {
"application/json": {
"schema": {
"additionalProperties": true,
"type": "object"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {
"additionalProperties": true,
"type": "object"
}
},
"text/event-stream": {
"schema": {
"type": "string"
}
}
},
"description": "Unmodified Codex response or server-sent events"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Codex Passthrough Route",
"tags": [
"codex_passthrough"
]
}
}
}
},
"compliance": {
"components": {
"schemas": {

View file

@ -391,6 +391,7 @@ class LiteLLMRoutes(enum.Enum):
]
mapped_pass_through_routes = [
"/backend-api/codex",
"/bedrock",
"/comprehendmedical",
"/azure_speech",
@ -1766,8 +1767,10 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
max_parallel_requests: int | None = Field(
default=None, description="Max concurrent requests allowed for this budget id."
)
tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
tpm_limit: int | None = Field(default=None, ge=0, description="Max tokens per minute, allowed for this budget id.")
rpm_limit: int | None = Field(
default=None, ge=0, description="Max requests per minute, allowed for this budget id."
)
tpd_limit: int | None = Field(
default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
)
@ -1878,6 +1881,8 @@ RouterSettingsDict = Annotated[
class NewTeamRequest(TeamBase):
tpm_limit: int | None = Field(default=None, ge=0)
rpm_limit: int | None = Field(default=None, ge=0)
router_settings: RouterSettingsDict | None = None
model_aliases: dict | None = None
model_max_budget: GenericBudgetConfigType | None = Field(
@ -1957,8 +1962,8 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
team_alias: str | None = None
organization_id: str | None = None
metadata: dict | None = None
tpm_limit: int | None = None
rpm_limit: int | None = None
tpm_limit: int | None = Field(default=None, ge=0)
rpm_limit: int | None = Field(default=None, ge=0)
tpd_limit: int | None = None
max_budget: float | None = None
soft_budget: float | None = None
@ -3272,7 +3277,7 @@ class LiteLLM_OrganizationTableWithMembers(LiteLLM_OrganizationTable):
members: list[LiteLLM_OrganizationMembershipTable] = []
teams: list[LiteLLM_TeamTable] = []
litellm_budget_table: LiteLLM_BudgetTable | None = None
litellm_budget_table: LiteLLM_BudgetTableFull | None = None
created_at: datetime
updated_at: datetime

View file

@ -6461,8 +6461,29 @@ def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool
return True
def _is_dashboard_session(valid_token: UserAPIKeyAuth | None) -> bool:
return valid_token is not None and valid_token.team_id == UI_TEAM_ID
def _is_search_tool_strict_identity(valid_token: UserAPIKeyAuth) -> bool:
if not _is_dashboard_session(valid_token):
return _is_strict_grant_identity(valid_token)
return valid_token.user_role not in (
LitellmUserRoles.PROXY_ADMIN,
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
)
def _search_tool_strict_layers(
valid_token: UserAPIKeyAuth, team_object: LiteLLM_TeamTable | None
) -> frozenset[GrantLayer]:
if _is_dashboard_session(valid_token):
return frozenset(("user",))
return _strict_grant_layers(True, valid_token, team_object)
def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool:
return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token)
return _search_tool_deny_by_default(general_settings) and _is_search_tool_strict_identity(valid_token)
def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException:
@ -6506,12 +6527,12 @@ async def resolve_search_tool_grants(
load_team_object: TeamObjectLoader,
) -> SearchToolGrants:
"""
Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions,
Without `general_settings.search_tool_deny_by_default`, or for the master key and admin dashboard sessions,
the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names
must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing
"""
if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
team_object: Final = await load_team_object()
team_object: Final = await load_team_object() if not _is_dashboard_session(valid_token) else None
return SearchToolGrants(
grants=MappingProxyType(
{"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None}
@ -6521,8 +6542,10 @@ async def resolve_search_tool_grants(
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
strict_team_object: Final = await _strict_team_object(load_team_object)
strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object)
strict_team_object: Final = (
await _strict_team_object(load_team_object) if not _is_dashboard_session(valid_token) else None
)
strict_layers: Final = _search_tool_strict_layers(valid_token, strict_team_object)
if prisma_client is None:
return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers)
user_object: Final = (
@ -6607,7 +6630,7 @@ def check_unregistered_search_fallback(
"""The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it"""
if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
return True
layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index)
layer: Final = min(_search_tool_strict_layers(valid_token, None), key=("key", "team", "user").index)
raise _search_tool_denied(
layer,
"No registered search tool is available and search_tool_deny_by_default is enabled",

View file

@ -12,7 +12,7 @@ import litellm
from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger
from litellm.constants import EMPTY_MAPPING
from litellm.integrations.otel.runtime import seed_request_identity
from litellm.litellm_core_utils.core_helpers import is_expected_client_error
from litellm.litellm_core_utils.core_helpers import as_str_mapping, is_expected_client_error
from litellm.proxy._types import (
LitellmUserRoles,
ModelAccessDeniedProxyException,
@ -139,7 +139,7 @@ class UserAPIKeyAuthExceptionHandler:
async def handle_authentication_error(
e: Exception,
request: Request,
request_data: dict[str, object],
request_data: object,
route: str,
parent_otel_span: Span | None,
api_key: str,
@ -190,6 +190,8 @@ class UserAPIKeyAuthExceptionHandler:
request_route=route,
)
else:
request_mapping: Final = as_str_mapping(request_data)
request_context: Final[dict[str, object]] = dict(request_mapping) if request_mapping is not None else {}
# raise the exception to the caller
requester_ip: Final = get_request_ip_address(
request=request,
@ -229,7 +231,7 @@ class UserAPIKeyAuthExceptionHandler:
# without this the failed trace would carry no team/key attributes.
seed_request_identity(
user_api_key_dict,
model=request_data.get("model"),
model=request_context.get("model"),
)
# Budget checks live in tenant-scoped helpers (key / team / org / tag)
@ -242,14 +244,14 @@ class UserAPIKeyAuthExceptionHandler:
resolve_llm_provider_for_rate_limit,
)
budget_model: Final = request_data.get("model")
budget_model: Final = request_context.get("model")
_, e.llm_provider = resolve_llm_provider_for_rate_limit(
budget_model if isinstance(budget_model, str) else None
)
# Allow callbacks to transform the error response
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
request_data=_with_client_context(request_data, requester_ip, user_agent),
request_data=_with_client_context(request_context, requester_ip, user_agent),
original_exception=e,
user_api_key_dict=user_api_key_dict,
error_type=ProxyErrorTypes.auth_error,

View file

@ -736,7 +736,9 @@ def is_request_body_safe(
def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
if manages_deployments:
reject_server_owned_wif_fields(litellm_params)
reject_server_owned_wif_fields(
MappingProxyType({key: value for key, value in litellm_params.items() if key != "chatgpt_auth_file"})
)
else:
reject_server_owned_wif_params(litellm_params)
reject_federated_credential_reference(litellm_params)

View file

@ -119,6 +119,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
)
from litellm.proxy.dd_span_tagger import DDSpanTagger
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
from litellm.proxy.native_compaction import with_proxy_compaction_executor
from litellm.proxy.route_llm_request import (
route_request,
@ -1416,6 +1417,8 @@ async def open_sse_before_first_byte(
if interval is None:
return await produce_response
# Share admission state with stream cleanup before the producer copies this context.
get_or_create_request_stash()
produce_task: Final = asyncio.ensure_future(produce_response)
await asyncio.wait((produce_task,), timeout=interval)
if produce_task.done():

View file

@ -6,7 +6,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin
import orjson
from fastapi import Request, UploadFile, status
from pydantic import TypeAdapter
from pydantic import TypeAdapter, ValidationError
from starlette._utils import get_route_path
from starlette.requests import HTTPConnection
from typing_extensions import NotRequired, ReadOnly, Required, assert_never
@ -162,13 +162,11 @@ def coerce_numeric_form_fields(
}
def _parse_binary_body(body: bytes) -> dict:
def _parse_binary_body(body: bytes) -> dict[str, object]:
"""JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}."""
try:
parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body))
if isinstance(parsed, dict):
return parsed
except orjson.JSONDecodeError:
return dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body)))
except (orjson.JSONDecodeError, ValidationError):
pass
return {}
@ -299,8 +297,9 @@ async def read_request_body(request: Request | None) -> dict:
body_str = re.sub(r"(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]", "", body_str)
try:
parsed_body = json.loads(body_str)
json.dumps(parsed_body, ensure_ascii=False).encode("utf-8")
decoded_body: Final = _DECODED_JSON.validate_python(json.loads(body_str))
json.dumps(decoded_body, ensure_ascii=False).encode("utf-8")
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(decoded_body))
except (json.JSONDecodeError, UnicodeEncodeError):
# json.loads accepts lone surrogate escapes that no provider can encode
verbose_proxy_logger.error("Invalid JSON payload received: %s", e)
@ -315,6 +314,34 @@ async def read_request_body(request: Request | None) -> dict:
safe_set_request_parsed_body(request=request, parsed_body=parsed_body)
return parsed_body
except ValidationError as e:
from fastapi.routing import APIRoute
from litellm.proxy.auth.auth_utils import (
request_dispatched_to_pass_through_endpoint,
request_dispatched_to_provider_pass_through,
)
from litellm.proxy.lens.buffering import LensRoute
route: Final[object] = request.scope.get("route") if request is not None else None
if request is not None and (
request_dispatched_to_pass_through_endpoint(request)
or request_dispatched_to_provider_pass_through(request)
or isinstance(route, LensRoute)
or (
isinstance(route, APIRoute)
and route.body_field is not None
and not route.body_field.field_info.is_required()
and (await request.body()).strip() == b"null"
)
):
return {}
raise ProxyException(
message="Invalid JSON payload: expected an object",
type="invalid_request_error",
param="request_body",
code=status.HTTP_400_BAD_REQUEST,
) from e
except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e:
# Re-raise ProxyException as-is
verbose_proxy_logger.error("Invalid JSON payload received: %s", e)

View file

@ -2304,7 +2304,15 @@ async def test_model_connection(
)
selected_by_id: Final = deployment_by_id is not None
reject_server_owned_wif_fields(request_litellm_params)
reject_server_owned_wif_fields(
MappingProxyType(
{
key: value
for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items()
if key != "chatgpt_auth_file"
}
)
)
# Merge: config params (from proxy config) as base, request params override
litellm_params = {
**_config_base_for_health_check(

View file

@ -1417,13 +1417,14 @@ def clean_headers(
Cleaned headers dict
"""
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
from litellm.llms.chatgpt.common_utils import is_chatgpt_oauth_key
clean_headers: Final = {}
litellm_key_lower: Final = litellm_key_header_name.lower() if litellm_key_header_name is not None else None
for header, value in headers.items():
header_lower = header.lower()
if header_lower == "authorization" and is_anthropic_oauth_key(value):
if header_lower == "authorization" and (is_anthropic_oauth_key(value) or is_chatgpt_oauth_key(value)):
if authenticated_with_header is None or authenticated_with_header.lower() != "authorization":
clean_headers[header] = value
continue
@ -1452,7 +1453,7 @@ def clean_headers(
def _is_credential_header(header: str) -> bool:
"""Whether `header` carries a caller credential rather than request context."""
return header.lower() in _CREDENTIAL_HEADER_NAMES
return header.lower().removeprefix("x-pass-") in _CREDENTIAL_HEADER_NAMES
def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
@ -4282,6 +4283,7 @@ _NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
)
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.CHATGPT.value
def add_provider_specific_headers_to_request(
@ -4289,6 +4291,7 @@ def add_provider_specific_headers_to_request(
headers: Mapping[str, str],
) -> bool:
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
from litellm.llms.chatgpt.common_utils import select_chatgpt_client_credential_headers
anthropic_api_headers: Final = {header: headers[header] for header in ANTHROPIC_API_HEADERS if header in headers}
anthropic_oauth_credential_headers: Final = {
@ -4296,6 +4299,8 @@ def add_provider_specific_headers_to_request(
for header, value in headers.items()
if header.lower() == "authorization" and is_anthropic_oauth_key(value)
}
chatgpt_oauth_credential_headers: Final = dict(select_chatgpt_client_credential_headers(headers))
typesafe_danger_zone_headers: Final = {
header: value
for header, value in headers.items()
@ -4320,13 +4325,17 @@ def add_provider_specific_headers_to_request(
ProviderSpecificHeader(
custom_llm_provider=LlmProviders.TYPESAFE.value, extra_headers=typesafe_danger_zone_headers
),
ProviderSpecificHeader(
custom_llm_provider=_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS,
extra_headers=chatgpt_oauth_credential_headers,
),
)
if scoped_header["extra_headers"]
]
if scoped_headers:
data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers
return bool(anthropic_oauth_credential_headers)
return bool(anthropic_oauth_credential_headers or chatgpt_oauth_credential_headers)
def _add_otel_traceparent_to_data(data: dict, request: Request):

View file

@ -569,8 +569,8 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = (
"temp_budget_expiry",
)
_TEMP_BUDGET_FIELDS: Final = frozenset({"temp_budget_increase", "temp_budget_expiry"})
_BUDGET_COLUMNS: Final = TypeAdapter(dict[str, object])
MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType(
{
@ -653,8 +653,8 @@ async def upsert_budget_and_membership(
``shared_budget_ids`` extends that protection to any other row more than one
membership points at, which a caller patching several members at once has
already counted; a row listed there is cloned rather than written in place.
A patch that only touches the temporary budget pair never copies permanent
limits into a new row, so the member keeps inheriting the live team default.
A patch whose every field already matches the shared row changes nothing and
is a no-op rather than a clone.
"""
ignore_default_model_budget_clear: Final = (
existing_budget_id is not None
@ -685,7 +685,6 @@ async def upsert_budget_and_membership(
is_shared_default: Final = existing_budget_id is not None and (
existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset())
)
temp_only: Final = frozenset(write_data) <= _TEMP_BUDGET_FIELDS
async def _disconnect():
await tx.litellm_teammembership.update(
@ -707,11 +706,15 @@ async def upsert_budget_and_membership(
return
source_row: Final = (
await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id})
if is_shared_default and not temp_only
else None
await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) if is_shared_default else None
)
source: Final[Mapping[str, object]] = source_row.model_dump() if source_row is not None else MappingProxyType({})
source: Final[Mapping[str, object]] = (
MappingProxyType(_BUDGET_COLUMNS.validate_python(source_row.model_dump()))
if source_row is not None
else MappingProxyType({})
)
if is_shared_default and all(source.get(field) == value for field, value in effective_patch.items()):
return
create_data: Final[dict[str, object]] = { # mutable-ok: Prisma create payloads are dict-shaped
"created_by": user_api_key_dict.user_id or "",
@ -740,7 +743,7 @@ async def upsert_budget_and_membership(
create_data.pop("budget_reset_at", None)
if not _has_meaningful_budget_limit(create_data):
if existing_budget_id is not None and not temp_only:
if existing_budget_id is not None:
await _disconnect()
return

View file

@ -9,6 +9,7 @@ from typing import Final, Literal, TypeAlias
from pydantic import TypeAdapter, ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.litellm_core_utils.duration_parser import normalize_duration
from litellm.models.team import BudgetLimitEntry
from litellm.models.verification_token import LiteLLM_VerificationToken
from litellm.proxy._types import UpdateKeyRequest
@ -99,6 +100,30 @@ def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_Verification
return "tightened" if kept_or_lowered else "loosened"
def _reset_schedule(budget_duration: str | None) -> str | None:
return None if budget_duration is None else normalize_duration(budget_duration)
def _changes_budget_duration(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool:
return "budget_duration" in data.model_fields_set and _reset_schedule(data.budget_duration) != _reset_schedule(
existing.budget_duration
)
def _changes_budget_id(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool:
return "budget_id" in data.model_fields_set and data.budget_id != existing.budget_id
def _budget_duration_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange:
"""
Any new `budget_duration` restarts the reset schedule from now, so only clearing it tightens the
stored `max_budget`. A key with no `max_budget` has no cap for the schedule to reset.
"""
if existing.max_budget is None or not _changes_budget_duration(data, existing):
return "unchanged"
return "tightened" if data.budget_duration is None else "loosened"
@dataclass(frozen=True, slots=True)
class EffectiveKeyBudget:
max_budget: float | None
@ -122,12 +147,23 @@ def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationT
def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange:
overrides_linked_budget: Final = (
existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged"
overrides_linked_budget: Final = existing.budget_id is not None and (
_max_budget_change(data, existing) != "unchanged" or _changes_budget_duration(data, existing)
)
if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget:
if (
data.spend is not None
or "soft_budget" in data.model_fields_set
or overrides_linked_budget
or _changes_budget_id(data, existing)
):
return KeyBudgetAdminOnly()
changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing)))
changes: Final = frozenset(
(
_max_budget_change(data, existing),
_budget_limits_change(data, existing),
_budget_duration_change(data, existing),
)
)
if "loosened" in changes:
return KeyBudgetLoosened()
if "tightened" in changes:

View file

@ -504,9 +504,8 @@ async def new_organization(
_budget: Final = await _table(BudgetRepository(prisma_client)).create(
data={
**new_budget,
**build_budget_write_data(new_budget, user_api_key_dict.user_id or litellm_proxy_admin_name),
"created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
"updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
}
)

View file

@ -0,0 +1,87 @@
import hmac
from types import MappingProxyType
from typing import Annotated, Final, Literal
from fastapi import APIRouter, Depends, HTTPException, Request, Response
from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request
router: Final = APIRouter()
_HEADERS: Final = frozenset(
{
"authorization",
"chatgpt-account-id",
"content-type",
"accept",
"user-agent",
"originator",
"openai-beta",
"session-id",
"thread-id",
"session_id",
"conversation_id",
"x-codex-turn-metadata",
"x-codex-turn-state",
}
)
@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"])
@router.post(
"/backend-api/codex/{endpoint}",
tags=["ChatGPT Pass-through", "pass-through"],
openapi_extra={
"requestBody": {
"required": True,
"content": {"application/json": {"schema": {"type": "object", "additionalProperties": True}}},
}
},
responses={
200: {
"description": "Unmodified Codex response or server-sent events",
"content": {
"application/json": {"schema": {"type": "object", "additionalProperties": True}},
"text/event-stream": {"schema": {"type": "string"}},
},
}
},
)
async def codex_passthrough_route(
endpoint: Literal["models", "responses"],
request: Request,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
) -> Response:
"""Forward a client's ChatGPT subscription with separate LiteLLM authentication."""
expected_method: Final = "GET" if endpoint == "models" else "POST"
if request.method != expected_method:
raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method})
authorization: Final = request.headers.get("authorization", "")
scheme, _, token = authorization.partition(" ")
gateway_key: Final = request.headers.get("x-litellm-api-key", "")
gateway_parts: Final = gateway_key.split()
gateway_token: Final = (
gateway_parts[-1]
if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer")
else ""
)
if scheme.lower() != "bearer" or not token.strip() or not gateway_token:
raise HTTPException(
status_code=401,
detail="Send the ChatGPT bearer in Authorization and the LiteLLM key in x-litellm-api-key",
)
if hmac.compare_digest(token.strip().encode(), gateway_token.encode()):
raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate")
headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS})
return await pass_through_request(
request=request,
target=f"{CHATGPT_API_BASE}/{endpoint}",
custom_headers=dict(headers),
user_api_key_dict=user_api_key_dict,
custom_llm_provider="chatgpt",
upstream_headers_filter=lambda _: headers,
)

View file

@ -10,6 +10,7 @@ from typing import Final, Literal, TypeAlias
from urllib.parse import urlparse
import httpx
from pydantic import TypeAdapter
import litellm
from litellm._logging import verbose_proxy_logger
@ -95,6 +96,8 @@ def is_openai_compatible_url(url_route: str | None) -> bool:
hostname: Final = parsed_url.hostname
if not hostname:
return False
if hostname == "chatgpt.com" and parsed_url.path.startswith("/backend-api/codex/"):
return True
if _hostname_matches(hostname, _OPENAI_HOSTNAMES):
return True
if _hostname_matches(hostname, _AZURE_OPENAI_HOSTNAMES):
@ -198,7 +201,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
if not url_route:
return False
parsed_url: Final = urlparse(url_route)
return _is_openai_compatible_host(parsed_url.hostname) and (
return is_openai_compatible_url(url_route) and (
"/v1/responses" in parsed_url.path or "/responses" in parsed_url.path
)
@ -424,7 +427,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
) = None
handler_instance: Final = OpenAIPassthroughLoggingHandler()
custom_llm_provider: Final = kwargs.get("custom_llm_provider", "openai")
custom_llm_provider: Final = TypeAdapter(str).validate_python(
kwargs.get("custom_llm_provider", logging_obj.model_call_details.get("custom_llm_provider", "openai"))
)
if is_chat_completions:
# Handle chat completions with existing logic

View file

@ -118,6 +118,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul
_key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy
_strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs
get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above
redact_credential_headers,
)
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
from litellm.proxy.route_priority import (
@ -409,6 +410,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
@staticmethod
def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType:
parsed_url: Final = urlparse(url)
if custom_llm_provider == "chatgpt":
return EndpointType.OPENAI
if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith(
"/v1/systemone"
):
@ -723,7 +726,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
"url": str(request.url),
"method": request.method,
"body": copy.copy(_parsed_body), # use copy instead of deepcopy
"headers": request.headers,
"headers": redact_credential_headers(request.headers),
},
},
"call_type": "pass_through_endpoint",
@ -1490,7 +1493,7 @@ async def pass_through_request(
additional_args={
"complete_input_dict": _parsed_body,
"api_base": str(logging_url),
"headers": upstream_headers,
"headers": redact_credential_headers(upstream_headers),
},
)
stream = HttpPassThroughEndpointHelpers.update_stream_param_based_on_request_body(
@ -1561,7 +1564,7 @@ async def pass_through_request(
data=_parsed_body or {},
user_api_key_dict=user_api_key_dict,
response=relay_response,
request_headers=dict(request.headers),
request_headers=dict(redact_credential_headers(request.headers)),
)
if callback_headers:
_response_headers.update(callback_headers)
@ -1653,7 +1656,7 @@ async def pass_through_request(
data=_parsed_body or {},
user_api_key_dict=user_api_key_dict,
response=detected_relay_response,
request_headers=dict(request.headers),
request_headers=dict(redact_credential_headers(request.headers)),
)
if callback_headers:
_response_headers.update(callback_headers)
@ -1703,7 +1706,7 @@ async def pass_through_request(
data=_parsed_body or {},
user_api_key_dict=user_api_key_dict,
response=response,
request_headers=dict(request.headers),
request_headers=dict(redact_credential_headers(request.headers)),
)
if relay_callback_headers:
relay_custom_headers.update(relay_callback_headers)
@ -1878,7 +1881,7 @@ async def pass_through_request(
data=_parsed_body or {},
user_api_key_dict=user_api_key_dict,
response=response,
request_headers=dict(request.headers),
request_headers=dict(redact_credential_headers(request.headers)),
)
if callback_headers:
custom_headers.update(callback_headers)

View file

@ -21795,8 +21795,7 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
detail=f"Toolset '{toolset_name}' not found",
)
scope: Final = dict(request.scope)
scope["path"] = "/mcp"
scope: Final = {**request.scope, "_original_path": request.scope.get("path", ""), "path": "/mcp"}
token: Final = mcp_active_toolset_id.set(toolset.toolset_id)
try:

View file

@ -308,10 +308,7 @@ def should_use_polling_for_request(
# Otherwise, check ALL deployments for this model_name in router
elif llm_router is not None:
try:
# Get all deployment indices for this model name
indices: Final = llm_router.model_name_to_deployment_indices.get(model, [])
for idx in indices:
deployment_dict = llm_router.model_list[idx]
for deployment_dict in llm_router.deployments_named(model) or ():
litellm_params = deployment_dict.get("litellm_params", {})
# Check custom_llm_provider first

Some files were not shown because too many files have changed in this diff Show more