mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-18 09:45:23 +00:00
Merge remote-tracking branch 'refs/remotes/origin/litellm_redis_client_scripts' into HEAD
This commit is contained in:
commit
a578ad80e7
237 changed files with 16694 additions and 10774 deletions
52
.agents/skills/database-access/SKILL.md
Normal file
52
.agents/skills/database-access/SKILL.md
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
---
|
||||
name: database-access
|
||||
description: Add database queries, extract existing Prisma calls into repositories, or explicitly port queries to Rust and SQLx without expanding migration scope
|
||||
---
|
||||
|
||||
# Database access
|
||||
|
||||
## Adding new database access
|
||||
|
||||
- Put Python database access behind an operation in `litellm/repositories`
|
||||
- Give callers typed inputs and outputs, not Prisma clients, models, or query dictionaries
|
||||
- Keep query execution and backend selection inside the repository
|
||||
- Prisma is fine for the implementation, adding a query does not require a Rust port
|
||||
- Native Rust callers use `litellm-rust/crates/db` directly
|
||||
|
||||
## Asked to move existing access behind a repository
|
||||
|
||||
- Identify the calls being moved and their transaction boundary
|
||||
- Move the existing Prisma implementation into the relevant repository
|
||||
- Replace those direct database calls with repository operations
|
||||
- Preserve return values, errors, and transaction behavior, with focused regression tests
|
||||
- Keep this an extraction, not a backend migration or unrelated cleanup
|
||||
|
||||
## An equivalent Rust or SQLx query already exists
|
||||
|
||||
- Leave it and the current backend selection unchanged unless integration is explicitly requested
|
||||
- Its existence alone does not justify switching a Prisma caller to Rust
|
||||
- Repository extraction can keep using Prisma even when the Rust query is available
|
||||
|
||||
## Asked to port a Prisma query to Rust or SQLx
|
||||
|
||||
- Identify the existing inputs, outputs, errors, and transaction boundary
|
||||
- Check `litellm-rust/crates/db` for an equivalent query and reuse it if its behavior matches
|
||||
- Otherwise follow `litellm-rust/crates/db/AGENTS.md`
|
||||
- Put static SQL in `queries/` and compile-check it with SQLx
|
||||
- Keep typed inputs and reusable row projections in `src/`
|
||||
- Refresh SQLx query metadata after SQL changes
|
||||
- Keep Python conversion in the Python bridge, not in `litellm-rust/crates/db`
|
||||
- When caller integration is requested, expose the Rust operation through the bridge and call it from the repository
|
||||
- A query-only port can land without switching callers
|
||||
- Verify behavior against real PostgreSQL before switching callers
|
||||
- Check relevant null handling, conflicts, ordering, errors, and transaction semantics
|
||||
- Remove the old implementation only when no callers need it
|
||||
|
||||
## Migration context and boundaries
|
||||
|
||||
- Long-term direction is to centralize Python database access in repositories, then replace their Prisma implementations incrementally with Rust APIs
|
||||
- Mixed Prisma and Rust implementations are expected during the transition, unrelated existing calls can stay as they are
|
||||
- Both backends may have separate pools through PgBouncer, size them together and verify pooling-mode compatibility
|
||||
- PgBouncer does not share transactions between Prisma and SQLx, keep each atomic unit of work on one backend and transaction
|
||||
- Never retry a failed Rust write through Prisma when it may already have committed
|
||||
- Prisma owns schema migrations today, eventual SQLx migration ownership and Prisma removal are separate planned steps, not part of a query port
|
||||
|
|
@ -1,319 +0,0 @@
|
|||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
# Asynchronously fetch data from a given URL
|
||||
async def fetch_data(url):
|
||||
try:
|
||||
# Create an asynchronous session
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Send a GET request to the URL
|
||||
async with session.get(url) as resp:
|
||||
# Raise an error if the response status is not OK
|
||||
resp.raise_for_status()
|
||||
# Parse the response JSON
|
||||
resp_json = await resp.json()
|
||||
print("Fetch the data from URL.")
|
||||
# Return the 'data' field from the JSON response
|
||||
return resp_json['data']
|
||||
except Exception as e:
|
||||
# Print an error message if fetching data fails
|
||||
print("Error fetching data from URL:", e)
|
||||
return None
|
||||
|
||||
|
||||
FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
|
||||
FRIENDLI_PROVIDER = "friendliai"
|
||||
|
||||
INHERITABLE_BASE_KEYS = (
|
||||
"supports_pdf_input",
|
||||
"supports_assistant_prefill",
|
||||
"supports_adaptive_thinking",
|
||||
"supports_output_config",
|
||||
)
|
||||
|
||||
REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
|
||||
|
||||
|
||||
def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
|
||||
if not base_model:
|
||||
return None
|
||||
bm_tail = base_model.split("/")[-1].lower()
|
||||
if base_model in local_data:
|
||||
return base_model
|
||||
for key in local_data:
|
||||
if key.startswith("sample_spec") or key == "fallback_generalizations":
|
||||
continue
|
||||
if key.split("/")[-1].lower() == bm_tail:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _reasoning_effort_levels(reasoning_options: list) -> list:
|
||||
offered = {
|
||||
val
|
||||
for opt in reasoning_options or []
|
||||
if opt.get("type") == "effort"
|
||||
for val in opt.get("values", [])
|
||||
}
|
||||
return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
|
||||
|
||||
|
||||
def _valid_token_price(value: object) -> bool:
|
||||
try:
|
||||
price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return math.isfinite(price) and price >= 0
|
||||
|
||||
|
||||
def _has_valid_token_prices(pricing: dict | None) -> bool:
|
||||
prices = pricing or {}
|
||||
return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
|
||||
|
||||
|
||||
def _pricing(pricing: dict) -> dict:
|
||||
out: dict[str, Any] = {}
|
||||
if not pricing:
|
||||
return out
|
||||
if "input" in pricing:
|
||||
out["input_cost_per_token"] = float(pricing["input"])
|
||||
if "output" in pricing:
|
||||
out["output_cost_per_token"] = float(pricing["output"])
|
||||
if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
|
||||
out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
|
||||
return out
|
||||
|
||||
|
||||
def _modality_flags(input_mods: list) -> dict:
|
||||
mods = input_mods or []
|
||||
has_image = "image" in mods
|
||||
return {
|
||||
"supports_vision": has_image,
|
||||
"supports_image_input": has_image,
|
||||
"supports_video_input": "video" in mods,
|
||||
}
|
||||
|
||||
|
||||
def transform_friendli_data(data: list, local_data: dict) -> dict:
|
||||
transformed: dict[str, dict] = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for model in data:
|
||||
# An unpriced row must never wholesale-replace an already priced local entry:
|
||||
# missing prices cost-calculate as zero, silently zeroing tracked spend
|
||||
if not _has_valid_token_prices(model.get("pricing")):
|
||||
continue
|
||||
model_id = model["id"]
|
||||
base_model = model.get("base_model") or ""
|
||||
entry: dict[str, Any] = {
|
||||
"litellm_provider": FRIENDLI_PROVIDER,
|
||||
}
|
||||
|
||||
base_key = _find_base_model_entry(base_model, local_data)
|
||||
if base_key:
|
||||
base_entry = local_data[base_key]
|
||||
for k in INHERITABLE_BASE_KEYS:
|
||||
if k in base_entry:
|
||||
entry[k] = base_entry[k]
|
||||
|
||||
ctx = model.get("context_length")
|
||||
if ctx is not None:
|
||||
entry["max_input_tokens"] = int(ctx)
|
||||
max_out = model.get("max_completion_tokens")
|
||||
if max_out is not None:
|
||||
entry["max_output_tokens"] = int(max_out)
|
||||
entry["max_tokens"] = int(max_out)
|
||||
|
||||
pricing = _pricing(model.get("pricing", {}))
|
||||
entry.update(pricing)
|
||||
entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
|
||||
|
||||
reasoning = model.get("reasoning") is True
|
||||
entry["supports_reasoning"] = reasoning
|
||||
if reasoning:
|
||||
entry["reasoning_effort_levels"] = _reasoning_effort_levels(
|
||||
model.get("reasoning_options", [])
|
||||
)
|
||||
|
||||
func = model.get("functionality", {})
|
||||
entry["supports_function_calling"] = func.get("tool_call") is True
|
||||
entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
|
||||
is_struct = func.get("structured_output") is True
|
||||
entry["supports_response_schema"] = is_struct
|
||||
entry["supports_native_structured_output"] = is_struct
|
||||
entry["supports_system_messages"] = func.get("system_messages") is True
|
||||
entry["supports_tool_choice"] = func.get("tool_choice") is True
|
||||
|
||||
entry.update(_modality_flags(model.get("input_modalities", [])))
|
||||
|
||||
entry["mode"] = model.get("mode", "chat")
|
||||
|
||||
desc = model.get("description")
|
||||
if desc:
|
||||
entry["comment"] = desc
|
||||
|
||||
dep = model.get("deprecation_date")
|
||||
if dep:
|
||||
entry["deprecation_date"] = dep.split("T")[0]
|
||||
|
||||
entry["source"] = FRIENDLI_API_URL
|
||||
|
||||
transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
|
||||
return transformed
|
||||
|
||||
# Synchronize local data with remote data
|
||||
def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
|
||||
# Update existing keys in local_data with values from remote_data
|
||||
# (replace_keys entries are swapped wholesale so a field the remote catalog
|
||||
# dropped, e.g. cache pricing, cannot survive as a stale value)
|
||||
for key in (set(local_data) & set(remote_data)):
|
||||
if key in replace_keys:
|
||||
local_data[key] = remote_data[key]
|
||||
else:
|
||||
local_data[key].update(remote_data[key])
|
||||
|
||||
# Add new keys from remote_data to local_data
|
||||
for key in (set(remote_data) - set(local_data)):
|
||||
local_data[key] = remote_data[key]
|
||||
|
||||
# Write data to the json file
|
||||
def write_to_file(file_path, data):
|
||||
try:
|
||||
# Open the file in write mode
|
||||
with open(file_path, "w") as file:
|
||||
# Dump the data as JSON into the file
|
||||
json.dump(data, file, indent=4)
|
||||
print("Values updated successfully.")
|
||||
except Exception as e:
|
||||
# Print an error message if writing to file fails
|
||||
print("Error updating JSON file:", e)
|
||||
|
||||
# Update the existing models and add the missing models for OpenRouter
|
||||
def transform_openrouter_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Add the fields 'max_tokens' and 'input_cost_per_token'
|
||||
obj = {
|
||||
"max_tokens": row["context_length"],
|
||||
"input_cost_per_token": float(row["pricing"]["prompt"]),
|
||||
}
|
||||
|
||||
# Add 'max_output_tokens' as a field if it is not None
|
||||
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
|
||||
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
|
||||
|
||||
# Add the field 'output_cost_per_token'
|
||||
obj.update({
|
||||
"output_cost_per_token": float(row["pricing"]["completion"]),
|
||||
})
|
||||
|
||||
# Add field 'input_cost_per_image' if it exists and is non-zero
|
||||
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
|
||||
obj['input_cost_per_image'] = float(row["pricing"]["image"])
|
||||
|
||||
# Add the fields 'litellm_provider' and 'mode'
|
||||
obj.update({
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat"
|
||||
})
|
||||
|
||||
# Add the 'supports_vision' field if the modality is 'multimodal'
|
||||
if row.get('architecture', {}).get('modality') == 'multimodal':
|
||||
obj['supports_vision'] = True
|
||||
|
||||
# Use a composite key to store the transformed object
|
||||
transformed[f'openrouter/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
# Update the existing models and add the missing models for Vercel AI Gateway
|
||||
def transform_vercel_ai_gateway_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
|
||||
if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
|
||||
row.get("pricing", {}).get(k) is None for k in ("input", "output")
|
||||
):
|
||||
continue
|
||||
obj = {
|
||||
"max_tokens": row["context_window"],
|
||||
"input_cost_per_token": float(row["pricing"]["input"]),
|
||||
"output_cost_per_token": float(row["pricing"]["output"]),
|
||||
'max_output_tokens': row['max_tokens'],
|
||||
'max_input_tokens': row["context_window"],
|
||||
}
|
||||
|
||||
# Handle cache pricing if available
|
||||
if "pricing" in row:
|
||||
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
|
||||
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
|
||||
|
||||
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
|
||||
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
|
||||
|
||||
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
|
||||
|
||||
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
|
||||
|
||||
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
# Load local data from a specified file
|
||||
def load_local_data(file_path):
|
||||
try:
|
||||
# Open the file in read mode
|
||||
with open(file_path, "r") as file:
|
||||
# Load and return the JSON data
|
||||
return json.load(file)
|
||||
except FileNotFoundError:
|
||||
# Print an error message if the file is not found
|
||||
print("File not found:", file_path)
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
# Print an error message if JSON decoding fails
|
||||
print("Error decoding JSON:", e)
|
||||
return None
|
||||
|
||||
def main():
|
||||
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
|
||||
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
|
||||
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
|
||||
|
||||
# Load local data from file
|
||||
local_data = load_local_data(local_file_path)
|
||||
|
||||
# Fetch OpenRouter data
|
||||
openrouter_data = asyncio.run(fetch_data(openrouter_url))
|
||||
# Transform the fetched OpenRouter data
|
||||
openrouter_data = transform_openrouter_data(openrouter_data)
|
||||
|
||||
# Fetch Vercel AI Gateway data
|
||||
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
|
||||
# Transform the fetched Vercel AI Gateway data
|
||||
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
|
||||
|
||||
friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
|
||||
friendli_data = transform_friendli_data(friendli_data, local_data)
|
||||
|
||||
# Combine both datasets
|
||||
all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
|
||||
|
||||
# If both local and openrouter data are available, synchronize and save
|
||||
if local_data and all_remote_data:
|
||||
sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
|
||||
write_to_file(local_file_path, local_data)
|
||||
else:
|
||||
print("Failed to fetch model data from either local file or URL.")
|
||||
|
||||
# Entry point of the script
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
name: Updates model_prices_and_context_window.json and Create Pull Request
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Run every Sundays at midnight
|
||||
#- cron: "0 0 * * *" # Run daily at midnight
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
auto_update_price_and_context_window:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
|
||||
--base main
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
65
.github/workflows/sync-together-ai-models.yml
vendored
65
.github/workflows/sync-together-ai-models.yml
vendored
|
|
@ -1,65 +0,0 @@
|
|||
name: Sync Together AI model registry
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 6 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
sync_together_ai_models:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
BASE_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ env.BASE_BRANCH }}
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Look for an already-open sync PR
|
||||
id: existing
|
||||
run: |
|
||||
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
|
||||
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
|
||||
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "$open_pr" ]; then
|
||||
echo "An open sync PR already exists on branch $open_pr; skipping this run."
|
||||
fi
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
- name: Run the sync
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
|
||||
env:
|
||||
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
||||
- name: Create a pull request when the registry changed
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
if git diff --quiet; then
|
||||
echo "Registry already in sync; no PR needed."
|
||||
exit 0
|
||||
fi
|
||||
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$branch"
|
||||
git add model_prices_and_context_window.json \
|
||||
litellm/model_prices_and_context_window_backup.json
|
||||
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
|
||||
gh auth setup-git
|
||||
git push origin "$branch"
|
||||
gh pr create --title "feat(models): sync together_ai model registry" \
|
||||
--body-file "$RUNNER_TEMP/pr_body.md" \
|
||||
--head "$branch" \
|
||||
--base "$BASE_BRANCH"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
|
|
@ -45,7 +45,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
@ -132,7 +132,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
|
|
|
|||
8
.github/workflows/test-merge-smoke.yml
vendored
8
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -33,7 +33,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
|
|
@ -76,6 +76,12 @@ jobs:
|
|||
- name: Import and CLI checks
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
|
||||
|
||||
- name: Routing and UUID compatibility
|
||||
run: >-
|
||||
$IN_NETNS .venv/bin/python -m pytest
|
||||
tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions
|
||||
tests/unit/test_uuid_helper.py -q
|
||||
|
||||
- name: Proxy startup check
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
|
||||
|
||||
|
|
|
|||
106
.github/workflows/test-rust-db.yml
vendored
Normal file
106
.github/workflows/test-rust-db.yml
vendored
Normal file
|
|
@ -0,0 +1,106 @@
|
|||
name: LiteLLM Rust DB
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- "litellm-rust/crates/db/**"
|
||||
- "litellm-rust/Cargo.toml"
|
||||
- "litellm-rust/Cargo.lock"
|
||||
- "litellm-rust/clippy.toml"
|
||||
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
|
||||
- "**/schema.prisma"
|
||||
- ".cargo/**"
|
||||
- "rust-toolchain.toml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/workflows/test-rust-db.yml"
|
||||
pull_request:
|
||||
branches:
|
||||
- main
|
||||
- "litellm_**"
|
||||
paths:
|
||||
- "litellm-rust/crates/db/**"
|
||||
- "litellm-rust/Cargo.toml"
|
||||
- "litellm-rust/Cargo.lock"
|
||||
- "litellm-rust/clippy.toml"
|
||||
- "litellm-proxy-extras/litellm_proxy_extras/migrations/**"
|
||||
- "**/schema.prisma"
|
||||
- ".cargo/**"
|
||||
- "rust-toolchain.toml"
|
||||
- "pyproject.toml"
|
||||
- "uv.lock"
|
||||
- ".github/actions/setup-uv-with-retries/**"
|
||||
- ".github/workflows/test-rust-db.yml"
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
jobs:
|
||||
db-queries:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16.6@sha256:557fea37a744d5f4c8faab304b0a90858b53ab119735a88c131fd19dab802f36
|
||||
env:
|
||||
POSTGRES_DB: litellm
|
||||
POSTGRES_PASSWORD: postgres
|
||||
POSTGRES_USER: postgres
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U postgres -d litellm"
|
||||
--health-interval 10s
|
||||
--health-timeout 5s
|
||||
--health-retries 5
|
||||
env:
|
||||
DATABASE_URL: postgresql://postgres:postgres@localhost:5432/litellm
|
||||
SQLX_OFFLINE: "false"
|
||||
defaults:
|
||||
run:
|
||||
working-directory: litellm-rust
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- run: rustup toolchain install --no-self-update
|
||||
|
||||
- uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2
|
||||
with:
|
||||
workspaces: litellm-rust
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Install Python dependencies
|
||||
working-directory: .
|
||||
run: uv sync --frozen --extra extra-proxy
|
||||
|
||||
- name: Deploy Prisma migrations
|
||||
working-directory: .
|
||||
run: uv run --no-sync prisma migrate deploy --schema litellm-proxy-extras/litellm_proxy_extras/schema.prisma
|
||||
|
||||
- name: Install SQLx CLI
|
||||
run: cargo install sqlx-cli --version 0.9.0 --locked --no-default-features -F postgres
|
||||
|
||||
- name: Check SQLx query metadata
|
||||
working-directory: litellm-rust/crates/db
|
||||
run: cargo sqlx prepare --check
|
||||
|
||||
- name: Test database queries
|
||||
run: cargo test -p litellm-db --locked -- --include-ignored
|
||||
|
|
@ -0,0 +1,76 @@
|
|||
# Use your ChatGPT subscription through LiteLLM
|
||||
|
||||
Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution.
|
||||
|
||||
An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding.
|
||||
|
||||
## Add a model to the proxy
|
||||
|
||||
Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-6-luna
|
||||
litellm_params:
|
||||
model: chatgpt/gpt-6-luna
|
||||
model_info:
|
||||
mode: responses
|
||||
```
|
||||
|
||||
Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic.
|
||||
|
||||
## Configure Codex
|
||||
|
||||
Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration.
|
||||
|
||||
Add this provider to `~/.codex/config.toml`, preserving your other settings:
|
||||
|
||||
```toml
|
||||
model = "gpt-6-luna"
|
||||
model_provider = "litellm_subscription"
|
||||
|
||||
[model_providers.litellm_subscription]
|
||||
name = "LiteLLM subscription"
|
||||
base_url = "https://litellm.example.com/v1"
|
||||
wire_api = "responses"
|
||||
supports_websockets = false
|
||||
requires_openai_auth = true
|
||||
env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" }
|
||||
```
|
||||
|
||||
Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway.
|
||||
|
||||
For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway).
|
||||
|
||||
## Native compatibility and Pi
|
||||
|
||||
For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name.
|
||||
|
||||
### Configure Pi
|
||||
|
||||
Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi:
|
||||
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
"openai-codex": {
|
||||
"baseUrl": "https://litellm.example.com/backend-api/codex",
|
||||
"headers": {
|
||||
"x-litellm-api-key": "LITELLM_API_KEY"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file.
|
||||
|
||||
## Verify account usage
|
||||
|
||||
Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person.
|
||||
|
||||
The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
|
||||
|
||||
The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate.
|
||||
|
||||
This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes.
|
||||
15
litellm-rust/Cargo.lock
generated
15
litellm-rust/Cargo.lock
generated
|
|
@ -3716,8 +3716,15 @@ dependencies = [
|
|||
name = "litellm-db"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"macro_rules_attribute",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sqlx",
|
||||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -8257,18 +8264,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.10"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
|
||||
checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.10"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9"
|
||||
checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -110,6 +110,7 @@ serde_with = { version = "=3.21.0", default-features = false, features = ["std",
|
|||
sha2 = "0.10"
|
||||
syn = { version = "2", default-features = false }
|
||||
sqlx = { version = "0.9.0", default-features = false, features = ["json", "macros", "postgres", "runtime-tokio", "chrono", "tls-rustls-ring-native-roots"] }
|
||||
chrono = { version = "0.4.45", default-features = false, features = ["std"] }
|
||||
proc-macro2 = "1"
|
||||
quote = "1"
|
||||
subtle = "2"
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ mod rules;
|
|||
mod status;
|
||||
mod vertex_ai;
|
||||
|
||||
pub use original::{ExceptionFamily, OriginalException};
|
||||
pub use original::{ExceptionFamily, OriginalException, TimeoutPhase};
|
||||
pub use public::{MappedFailure, PublicError, UpstreamResponse};
|
||||
|
||||
use rules::{Rule, contains_any, first_match};
|
||||
|
|
@ -67,7 +67,6 @@ const TIMEOUT_MARKERS: &[&str] = &[
|
|||
pub struct ExceptionContext {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub asynchronous: bool,
|
||||
pub vertex_project: Option<String>,
|
||||
pub vertex_location: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
|
|
@ -105,11 +104,12 @@ pub fn exception_type(
|
|||
(None, message.clone(), None)
|
||||
}
|
||||
OriginalException::Timeout {
|
||||
phase,
|
||||
timeout_seconds,
|
||||
elapsed_seconds,
|
||||
} => (
|
||||
None,
|
||||
timeout_message(context.asynchronous, *timeout_seconds, *elapsed_seconds),
|
||||
timeout_message(*phase, *timeout_seconds, *elapsed_seconds),
|
||||
None,
|
||||
),
|
||||
};
|
||||
|
|
@ -161,21 +161,16 @@ fn family_rules(family: ExceptionFamily) -> &'static [Rule] {
|
|||
}
|
||||
}
|
||||
|
||||
/// The text the Python HTTP handler's timeout carries: the sync and async handlers word it
|
||||
/// differently.
|
||||
fn timeout_message(
|
||||
asynchronous: bool,
|
||||
/// The text the Python HTTP handler's timeout carries: the phase that expired and the limit
|
||||
/// httpx enforced for it, both rendered as Python would print them.
|
||||
pub fn timeout_message(
|
||||
phase: TimeoutPhase,
|
||||
timeout_seconds: Option<f64>,
|
||||
elapsed_seconds: Option<f64>,
|
||||
) -> String {
|
||||
let timeout = python_float(timeout_seconds);
|
||||
if asynchronous {
|
||||
let elapsed =
|
||||
python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
|
||||
format!("Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
|
||||
} else {
|
||||
format!("Connection timed out after {timeout} seconds.")
|
||||
}
|
||||
let elapsed = python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
|
||||
format!("{phase} timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
|
||||
}
|
||||
|
||||
fn python_float(value: Option<f64>) -> String {
|
||||
|
|
@ -356,6 +351,7 @@ mod tests {
|
|||
#[test]
|
||||
fn a_handler_timeout_is_a_408_without_a_response() {
|
||||
let original = OriginalException::Timeout {
|
||||
phase: TimeoutPhase::Read,
|
||||
timeout_seconds: Some(0.5),
|
||||
elapsed_seconds: Some(0.5031),
|
||||
};
|
||||
|
|
@ -363,7 +359,7 @@ mod tests {
|
|||
mapped("mistral", &original),
|
||||
MappedFailure {
|
||||
error: PublicError::Timeout { status: 408 },
|
||||
message: "MistralException - Connection timed out after 0.5 seconds.".into(),
|
||||
message: "MistralException - Read timed out. Timeout passed=0.5, time taken=0.503 seconds".into(),
|
||||
upstream: None,
|
||||
debug_info: DEBUG.into(),
|
||||
}
|
||||
|
|
@ -440,39 +436,44 @@ mod tests {
|
|||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::sync(
|
||||
false,
|
||||
#[case::connect(TimeoutPhase::Connect, "Connection timed out.")]
|
||||
#[case::read(TimeoutPhase::Read, "Read timed out.")]
|
||||
#[case::write(TimeoutPhase::Write, "Write timed out.")]
|
||||
#[case::pool(TimeoutPhase::Pool, "Connection pool timed out.")]
|
||||
fn timeout_text_names_the_phase_that_expired(
|
||||
#[case] phase: TimeoutPhase,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
timeout_message(phase, Some(0.5), Some(0.5031)),
|
||||
format!("{expected} Timeout passed=0.5, time taken=0.503 seconds")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::rounds_the_elapsed_time(
|
||||
Some(0.5),
|
||||
Some(0.5031),
|
||||
"Connection timed out after 0.5 seconds."
|
||||
)]
|
||||
#[case::async_rounds_the_elapsed_time(
|
||||
true,
|
||||
Some(0.5),
|
||||
Some(0.5031),
|
||||
"Connection timed out. Timeout passed=0.5, time taken=0.503 seconds"
|
||||
"Timeout passed=0.5, time taken=0.503 seconds"
|
||||
)]
|
||||
#[case::whole_seconds_keep_a_decimal(
|
||||
true,
|
||||
Some(600.0),
|
||||
Some(2.0),
|
||||
"Connection timed out. Timeout passed=600.0, time taken=2.0 seconds"
|
||||
"Timeout passed=600.0, time taken=2.0 seconds"
|
||||
)]
|
||||
#[case::unknown_values_render_as_none(
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
"Connection timed out. Timeout passed=None, time taken=None seconds"
|
||||
"Timeout passed=None, time taken=None seconds"
|
||||
)]
|
||||
fn timeout_text_follows_the_delivery_mode(
|
||||
#[case] asynchronous: bool,
|
||||
fn timeout_text_renders_durations_as_python_floats(
|
||||
#[case] timeout_seconds: Option<f64>,
|
||||
#[case] elapsed_seconds: Option<f64>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
timeout_message(asynchronous, timeout_seconds, elapsed_seconds),
|
||||
expected
|
||||
timeout_message(TimeoutPhase::Read, timeout_seconds, elapsed_seconds),
|
||||
format!("Read timed out. {expected}")
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ pub enum OriginalException {
|
|||
message: String,
|
||||
},
|
||||
Timeout {
|
||||
phase: TimeoutPhase,
|
||||
timeout_seconds: Option<f64>,
|
||||
elapsed_seconds: Option<f64>,
|
||||
},
|
||||
|
|
@ -20,6 +21,19 @@ pub enum OriginalException {
|
|||
},
|
||||
}
|
||||
|
||||
/// The httpx timeout that expired, spelled the way the Python HTTP handler names it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::VariantArray)]
|
||||
pub enum TimeoutPhase {
|
||||
#[strum(serialize = "Connection")]
|
||||
Connect,
|
||||
#[strum(serialize = "Read")]
|
||||
Read,
|
||||
#[strum(serialize = "Write")]
|
||||
Write,
|
||||
#[strum(serialize = "Connection pool")]
|
||||
Pool,
|
||||
}
|
||||
|
||||
/// Which provider-specific text rules apply before the shared status table.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ExceptionFamily {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,23 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS \"locked!\"\n",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "locked!",
|
||||
"type_info": "Bool",
|
||||
"origin": "Expression"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "085edfb9af2a914466c75b447bc7bb2d97532a9cd461cc5d8da1e1090560ca1f"
|
||||
}
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_SpendLogs\" (\n request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, \"startTime\", \"endTime\", \"completionStartTime\",\n request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,\n \"user\", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,\n end_user, requester_ip_address, messages, response, proxy_server_request,\n session_id, status, litellm_call_id\n)\nSELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,\n l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),\n l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,\n l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,\n l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,\n l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,\n '{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id\nFROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],\n $7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],\n $13::text[], $14::text[], $15::text[], $16::text[], $17::text[],\n $18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],\n $23::text[], $24::text[], $25::text[], $26::text[], $27::text[])\n AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,\n completion_tokens, start_time, end_time, completion_start_time,\n request_duration_ms, model, model_id, model_group, custom_llm_provider,\n api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,\n end_user, requester_ip_address, session_id, status, litellm_call_id)\nON CONFLICT DO NOTHING\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Float8Array",
|
||||
"Int4Array",
|
||||
"Int4Array",
|
||||
"Int4Array",
|
||||
"TimestampArray",
|
||||
"TimestampArray",
|
||||
"TimestampArray",
|
||||
"Int4Array",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"JsonbArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "32f3c07ced96324b535bbc703c1acf6d06eb22facdf478f88107c9dccdac326b"
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_EndUserTable\" (user_id, spend, blocked)\nSELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\nGROUP BY id\nORDER BY id\nON CONFLICT (user_id) DO UPDATE\nSET spend = \"LiteLLM_EndUserTable\".spend + EXCLUDED.spend\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"Float8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "345c02595f55a7f0274ef7214b02288e0f2c1f12a2a79ed53f0002826724cd0a"
|
||||
}
|
||||
|
|
@ -0,0 +1,78 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT e.spend AS \"spend?\",\n e.blocked AS \"blocked?\",\n b.max_budget AS \"max_budget?\",\n b.rpm_limit AS \"rpm_limit?\",\n b.tpm_limit AS \"tpm_limit?\"\nFROM (SELECT 1) AS one\nLEFT JOIN \"LiteLLM_EndUserTable\" AS e ON e.user_id = $1\nLEFT JOIN \"LiteLLM_BudgetTable\" AS assigned ON assigned.budget_id = e.budget_id\nLEFT JOIN \"LiteLLM_BudgetTable\" AS key_budget ON key_budget.budget_id = $3\nLEFT JOIN \"LiteLLM_BudgetTable\" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)\n",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "spend?",
|
||||
"type_info": "Float8",
|
||||
"origin": {
|
||||
"Table": {
|
||||
"table": "\"LiteLLM_EndUserTable\"",
|
||||
"name": "spend"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 1,
|
||||
"name": "blocked?",
|
||||
"type_info": "Bool",
|
||||
"origin": {
|
||||
"Table": {
|
||||
"table": "\"LiteLLM_EndUserTable\"",
|
||||
"name": "blocked"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 2,
|
||||
"name": "max_budget?",
|
||||
"type_info": "Float8",
|
||||
"origin": {
|
||||
"Table": {
|
||||
"table": "\"LiteLLM_BudgetTable\"",
|
||||
"name": "max_budget"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 3,
|
||||
"name": "rpm_limit?",
|
||||
"type_info": "Int8",
|
||||
"origin": {
|
||||
"Table": {
|
||||
"table": "\"LiteLLM_BudgetTable\"",
|
||||
"name": "rpm_limit"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"ordinal": 4,
|
||||
"name": "tpm_limit?",
|
||||
"type_info": "Int8",
|
||||
"origin": {
|
||||
"Table": {
|
||||
"table": "\"LiteLLM_BudgetTable\"",
|
||||
"name": "tpm_limit"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Text",
|
||||
"Text",
|
||||
"Text"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true,
|
||||
true
|
||||
]
|
||||
},
|
||||
"hash": "377d063cd0fc3a265d07d01f5aa64311d25428fd393bac4250891f148597e92e"
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT t.token FROM \"LiteLLM_VerificationToken\" t\n WHERE t.token IN (SELECT id FROM added)\n ORDER BY t.token FOR UPDATE\n)\nUPDATE \"LiteLLM_VerificationToken\" t\nSET spend = t.spend + added.cost,\n total_spend = t.total_spend + added.cost,\n last_active = (clock_timestamp() AT TIME ZONE 'UTC'),\n updated_at = (clock_timestamp() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.token = added.id AND t.token IN (SELECT token FROM locked)\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"Float8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "3829a383d0ae50578f0fd75da3910731a0cacc1ae5b92eec32967f539948f563"
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n), locked AS (\n SELECT u.user_id FROM \"LiteLLM_UserTable\" u\n WHERE u.user_id IN (SELECT id FROM added)\n ORDER BY u.user_id FOR UPDATE\n)\nUPDATE \"LiteLLM_UserTable\" u\nSET spend = u.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"Float8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "43f2e12a3ffb1de29b5b031a1173b87a20a2664b0bef3616631d2c24487c1e54"
|
||||
}
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "SELECT set_config('lock_timeout', '5000ms', true)",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "set_config",
|
||||
"type_info": "Text",
|
||||
"origin": "Expression"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": []
|
||||
},
|
||||
"nullable": [
|
||||
null
|
||||
]
|
||||
},
|
||||
"hash": "5c74f47a2b84574133c520d235192bf33e16fea1dbb0c32cf89a376a874927f4"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_DailyTeamSpend\" (\n id, team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n team_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyTeamSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyTeamSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyTeamSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyTeamSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyTeamSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyTeamSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyTeamSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyTeamSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyTeamSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Float8Array",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "9c51401a8907f818e96da28050b74dca981ff954452b35ffbace3e022035b416"
|
||||
}
|
||||
|
|
@ -0,0 +1,14 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_TeamMembership\" (user_id, team_id, spend, total_spend)\nSELECT member.user_id, member.team_id, member.cost, member.cost\nFROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)\nWHERE EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id = member.team_id\n AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))\n)\n OR EXISTS (\n SELECT 1 FROM \"LiteLLM_TeamMembership\" m\n WHERE m.user_id = member.user_id AND m.team_id = member.team_id\n)\nON CONFLICT (user_id, team_id) DO UPDATE\nSET spend = \"LiteLLM_TeamMembership\".spend + EXCLUDED.spend,\n total_spend = \"LiteLLM_TeamMembership\".total_spend + EXCLUDED.total_spend\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Jsonb"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "b0eb48b795bdaf728c79e986d2027df60ff71d8fed05b53be4458b335af88147"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_DailyUserSpend\" (\n id, user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Float8Array",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "bec3b53762af021a118d63b841b8c2ebd7fc9fcbf19c1c25aa131a1053d96cfe"
|
||||
}
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "WITH added AS (\n SELECT id, SUM(cost) AS cost\n FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)\n GROUP BY id\n),\nlocked AS (\n SELECT t.team_id FROM \"LiteLLM_TeamTable\" t\n WHERE t.team_id IN (SELECT id FROM added)\n ORDER BY t.team_id FOR UPDATE\n)\nUPDATE \"LiteLLM_TeamTable\" t\nSET spend = t.spend + added.cost,\n updated_at = (NOW() AT TIME ZONE 'UTC')\nFROM added\nWHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"Float8Array"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "c3e5542906e89995fae2a16eca1edd37a575d641bb68d91e61f5123173c275ce"
|
||||
}
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "INSERT INTO \"LiteLLM_DailyEndUserSpend\" (\n id, end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,\n completion_tokens, api_requests, successful_requests, failed_requests,\n cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,\n timed_requests, spend, updated_at\n)\nSELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,\n r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,\n r.completion_tokens, r.api_requests, r.successful_requests,\n r.failed_requests, r.cache_read, r.cache_creation, r.response_time,\n r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')\nFROM UNNEST(\n $1::text[], $2::text[], $3::text[], $4::text[], $5::text[],\n $6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],\n $11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],\n $16::int8[], $17::float8[], $18::text[]\n) AS r(\n entity_id, date, api_key, model, custom_llm_provider, endpoint,\n model_group, prompt_tokens, completion_tokens, api_requests,\n successful_requests, failed_requests, cache_read, cache_creation,\n response_time, timed_requests, spend, mcp_namespaced_tool_name\n)\nORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,\n r.mcp_namespaced_tool_name, r.endpoint\nON CONFLICT (\n end_user_id, date, api_key, model, custom_llm_provider,\n mcp_namespaced_tool_name, endpoint\n) DO UPDATE SET\n prompt_tokens = \"LiteLLM_DailyEndUserSpend\".prompt_tokens + EXCLUDED.prompt_tokens,\n completion_tokens = \"LiteLLM_DailyEndUserSpend\".completion_tokens + EXCLUDED.completion_tokens,\n api_requests = \"LiteLLM_DailyEndUserSpend\".api_requests + EXCLUDED.api_requests,\n successful_requests = \"LiteLLM_DailyEndUserSpend\".successful_requests + EXCLUDED.successful_requests,\n failed_requests = \"LiteLLM_DailyEndUserSpend\".failed_requests + EXCLUDED.failed_requests,\n cache_read_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,\n cache_creation_input_tokens = \"LiteLLM_DailyEndUserSpend\".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,\n total_response_time_ms = \"LiteLLM_DailyEndUserSpend\".total_response_time_ms + EXCLUDED.total_response_time_ms,\n timed_requests = \"LiteLLM_DailyEndUserSpend\".timed_requests + EXCLUDED.timed_requests,\n spend = \"LiteLLM_DailyEndUserSpend\".spend + EXCLUDED.spend,\n updated_at = (NOW() AT TIME ZONE 'UTC')\n",
|
||||
"describe": {
|
||||
"columns": [],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"TextArray",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Int8Array",
|
||||
"Float8Array",
|
||||
"TextArray"
|
||||
]
|
||||
},
|
||||
"nullable": []
|
||||
},
|
||||
"hash": "f207ff0f5310dea0833361d55993bcab69ff6ad3555a3926537586f5d81d933d"
|
||||
}
|
||||
25
litellm-rust/crates/db/AGENTS.md
Normal file
25
litellm-rust/crates/db/AGENTS.md
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
# Database crate
|
||||
|
||||
Keep each static query in `queries/` and call it through a typed SQLx query function. Rust does not create or migrate the database
|
||||
|
||||
- Own reusable row projections in `src/` and export those consumed by other crates or integration tests
|
||||
- Apply `#[macro_rules_attribute::apply(crate::db_row)]` to row types for shared derives, without adding serialization or schema derives automatically
|
||||
- Apply `#[macro_rules_attribute::apply(crate::db_input)]` to write inputs, adding `Default` only where supported
|
||||
- Keep function tests inline in their owning query module and reuse the strict database fixture in `test_support`
|
||||
- Transaction-local setup in `begin` is the approved exception to static query files
|
||||
- Keep write inputs separate from stored rows and keep test-only assertions out of public types
|
||||
- `FromRow` supports runtime reads, but production static reads still use `query_file_as!` for compile-time checking
|
||||
|
||||
Database integration tests are ignored by default, but still compile in workspace checks. Run them explicitly against a local database already migrated by Prisma. Missing or invalid database configuration must fail the selected tests:
|
||||
|
||||
```sh
|
||||
DATABASE_URL=postgresql://... cargo test -p litellm-db --locked -- --include-ignored
|
||||
```
|
||||
|
||||
After changing a query, refresh the checked query metadata from this directory:
|
||||
|
||||
```sh
|
||||
SQLX_OFFLINE=false cargo sqlx prepare
|
||||
```
|
||||
|
||||
Tests use transactions and leave them uncommitted so fixture rows roll back when each test finishes
|
||||
|
|
@ -6,5 +6,14 @@ license.workspace = true
|
|||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
chrono.workspace = true
|
||||
macro_rules_attribute.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
sqlx.workspace = true
|
||||
strum.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
rstest.workspace = true
|
||||
tokio.workspace = true
|
||||
|
|
|
|||
10
litellm-rust/crates/db/queries/end_user/find_end_user.sql
Normal file
10
litellm-rust/crates/db/queries/end_user/find_end_user.sql
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
SELECT e.spend AS "spend?",
|
||||
e.blocked AS "blocked?",
|
||||
b.max_budget AS "max_budget?",
|
||||
b.rpm_limit AS "rpm_limit?",
|
||||
b.tpm_limit AS "tpm_limit?"
|
||||
FROM (SELECT 1) AS one
|
||||
LEFT JOIN "LiteLLM_EndUserTable" AS e ON e.user_id = $1
|
||||
LEFT JOIN "LiteLLM_BudgetTable" AS assigned ON assigned.budget_id = e.budget_id
|
||||
LEFT JOIN "LiteLLM_BudgetTable" AS key_budget ON key_budget.budget_id = $3
|
||||
LEFT JOIN "LiteLLM_BudgetTable" AS b ON b.budget_id = COALESCE(assigned.budget_id, key_budget.budget_id, $2)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
|
||||
SELECT id, SUM(cost), false FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
|
||||
GROUP BY id
|
||||
ORDER BY id
|
||||
ON CONFLICT (user_id) DO UPDATE
|
||||
SET spend = "LiteLLM_EndUserTable".spend + EXCLUDED.spend
|
||||
16
litellm-rust/crates/db/queries/spend/add_key_spend.sql
Normal file
16
litellm-rust/crates/db/queries/spend/add_key_spend.sql
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
WITH added AS (
|
||||
SELECT id, SUM(cost) AS cost
|
||||
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
|
||||
GROUP BY id
|
||||
), locked AS (
|
||||
SELECT t.token FROM "LiteLLM_VerificationToken" t
|
||||
WHERE t.token IN (SELECT id FROM added)
|
||||
ORDER BY t.token FOR UPDATE
|
||||
)
|
||||
UPDATE "LiteLLM_VerificationToken" t
|
||||
SET spend = t.spend + added.cost,
|
||||
total_spend = t.total_spend + added.cost,
|
||||
last_active = (clock_timestamp() AT TIME ZONE 'UTC'),
|
||||
updated_at = (clock_timestamp() AT TIME ZONE 'UTC')
|
||||
FROM added
|
||||
WHERE t.token = added.id AND t.token IN (SELECT token FROM locked)
|
||||
15
litellm-rust/crates/db/queries/spend/add_team_spend.sql
Normal file
15
litellm-rust/crates/db/queries/spend/add_team_spend.sql
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
WITH added AS (
|
||||
SELECT id, SUM(cost) AS cost
|
||||
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
|
||||
GROUP BY id
|
||||
),
|
||||
locked AS (
|
||||
SELECT t.team_id FROM "LiteLLM_TeamTable" t
|
||||
WHERE t.team_id IN (SELECT id FROM added)
|
||||
ORDER BY t.team_id FOR UPDATE
|
||||
)
|
||||
UPDATE "LiteLLM_TeamTable" t
|
||||
SET spend = t.spend + added.cost,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
FROM added
|
||||
WHERE t.team_id = added.id AND t.team_id IN (SELECT team_id FROM locked)
|
||||
14
litellm-rust/crates/db/queries/spend/add_user_spend.sql
Normal file
14
litellm-rust/crates/db/queries/spend/add_user_spend.sql
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
WITH added AS (
|
||||
SELECT id, SUM(cost) AS cost
|
||||
FROM UNNEST($1::text[], $2::float8[]) AS a(id, cost)
|
||||
GROUP BY id
|
||||
), locked AS (
|
||||
SELECT u.user_id FROM "LiteLLM_UserTable" u
|
||||
WHERE u.user_id IN (SELECT id FROM added)
|
||||
ORDER BY u.user_id FOR UPDATE
|
||||
)
|
||||
UPDATE "LiteLLM_UserTable" u
|
||||
SET spend = u.spend + added.cost,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
FROM added
|
||||
WHERE u.user_id = added.id AND u.user_id IN (SELECT user_id FROM locked)
|
||||
26
litellm-rust/crates/db/queries/spend/insert_spend_logs.sql
Normal file
26
litellm-rust/crates/db/queries/spend/insert_spend_logs.sql
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
INSERT INTO "LiteLLM_SpendLogs" (
|
||||
request_id, call_type, api_key, spend, total_tokens, prompt_tokens,
|
||||
completion_tokens, "startTime", "endTime", "completionStartTime",
|
||||
request_duration_ms, model, model_id, model_group, custom_llm_provider, api_base,
|
||||
"user", metadata, cache_hit, cache_key, request_tags, team_id, organization_id,
|
||||
end_user, requester_ip_address, messages, response, proxy_server_request,
|
||||
session_id, status, litellm_call_id
|
||||
)
|
||||
SELECT l.request_id, l.call_type, l.api_key, l.spend, l.total_tokens, l.prompt_tokens,
|
||||
l.completion_tokens, l.start_time::timestamp(3), l.end_time::timestamp(3),
|
||||
l.completion_start_time::timestamp(3), l.request_duration_ms, l.model,
|
||||
l.model_id, l.model_group, l.custom_llm_provider, l.api_base, l.user_id,
|
||||
l.metadata, l.cache_hit, l.cache_key, '[]'::jsonb, l.team_id,
|
||||
l.organization_id, l.end_user, l.requester_ip_address, '{}'::jsonb,
|
||||
'{}'::jsonb, '{}'::jsonb, l.session_id, l.status, l.litellm_call_id
|
||||
FROM UNNEST($1::text[], $2::text[], $3::text[], $4::float8[], $5::int4[], $6::int4[],
|
||||
$7::int4[], $8::timestamp[], $9::timestamp[], $10::timestamp[], $11::int4[], $12::text[],
|
||||
$13::text[], $14::text[], $15::text[], $16::text[], $17::text[],
|
||||
$18::jsonb[], $19::text[], $20::text[], $21::text[], $22::text[],
|
||||
$23::text[], $24::text[], $25::text[], $26::text[], $27::text[])
|
||||
AS l(request_id, call_type, api_key, spend, total_tokens, prompt_tokens,
|
||||
completion_tokens, start_time, end_time, completion_start_time,
|
||||
request_duration_ms, model, model_id, model_group, custom_llm_provider,
|
||||
api_base, user_id, metadata, cache_hit, cache_key, team_id, organization_id,
|
||||
end_user, requester_ip_address, session_id, status, litellm_call_id)
|
||||
ON CONFLICT DO NOTHING
|
||||
1
litellm-rust/crates/db/queries/spend/lock_team.sql
Normal file
1
litellm-rust/crates/db/queries/spend/lock_team.sql
Normal file
|
|
@ -0,0 +1 @@
|
|||
SELECT pg_advisory_xact_lock(hashtext($1)) IS NULL AS "locked!"
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
INSERT INTO "LiteLLM_DailyEndUserSpend" (
|
||||
id, end_user_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests, failed_requests,
|
||||
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, updated_at
|
||||
)
|
||||
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
|
||||
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
|
||||
r.completion_tokens, r.api_requests, r.successful_requests,
|
||||
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
|
||||
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
|
||||
FROM UNNEST(
|
||||
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
|
||||
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
|
||||
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
|
||||
$16::int8[], $17::float8[], $18::text[]
|
||||
) AS r(
|
||||
entity_id, date, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, prompt_tokens, completion_tokens, api_requests,
|
||||
successful_requests, failed_requests, cache_read, cache_creation,
|
||||
response_time, timed_requests, spend, mcp_namespaced_tool_name
|
||||
)
|
||||
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
|
||||
r.mcp_namespaced_tool_name, r.endpoint
|
||||
ON CONFLICT (
|
||||
end_user_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint
|
||||
) DO UPDATE SET
|
||||
prompt_tokens = "LiteLLM_DailyEndUserSpend".prompt_tokens + EXCLUDED.prompt_tokens,
|
||||
completion_tokens = "LiteLLM_DailyEndUserSpend".completion_tokens + EXCLUDED.completion_tokens,
|
||||
api_requests = "LiteLLM_DailyEndUserSpend".api_requests + EXCLUDED.api_requests,
|
||||
successful_requests = "LiteLLM_DailyEndUserSpend".successful_requests + EXCLUDED.successful_requests,
|
||||
failed_requests = "LiteLLM_DailyEndUserSpend".failed_requests + EXCLUDED.failed_requests,
|
||||
cache_read_input_tokens = "LiteLLM_DailyEndUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
|
||||
cache_creation_input_tokens = "LiteLLM_DailyEndUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
|
||||
total_response_time_ms = "LiteLLM_DailyEndUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
|
||||
timed_requests = "LiteLLM_DailyEndUserSpend".timed_requests + EXCLUDED.timed_requests,
|
||||
spend = "LiteLLM_DailyEndUserSpend".spend + EXCLUDED.spend,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
INSERT INTO "LiteLLM_DailyTeamSpend" (
|
||||
id, team_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests, failed_requests,
|
||||
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, updated_at
|
||||
)
|
||||
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
|
||||
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
|
||||
r.completion_tokens, r.api_requests, r.successful_requests,
|
||||
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
|
||||
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
|
||||
FROM UNNEST(
|
||||
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
|
||||
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
|
||||
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
|
||||
$16::int8[], $17::float8[], $18::text[]
|
||||
) AS r(
|
||||
entity_id, date, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, prompt_tokens, completion_tokens, api_requests,
|
||||
successful_requests, failed_requests, cache_read, cache_creation,
|
||||
response_time, timed_requests, spend, mcp_namespaced_tool_name
|
||||
)
|
||||
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
|
||||
r.mcp_namespaced_tool_name, r.endpoint
|
||||
ON CONFLICT (
|
||||
team_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint
|
||||
) DO UPDATE SET
|
||||
prompt_tokens = "LiteLLM_DailyTeamSpend".prompt_tokens + EXCLUDED.prompt_tokens,
|
||||
completion_tokens = "LiteLLM_DailyTeamSpend".completion_tokens + EXCLUDED.completion_tokens,
|
||||
api_requests = "LiteLLM_DailyTeamSpend".api_requests + EXCLUDED.api_requests,
|
||||
successful_requests = "LiteLLM_DailyTeamSpend".successful_requests + EXCLUDED.successful_requests,
|
||||
failed_requests = "LiteLLM_DailyTeamSpend".failed_requests + EXCLUDED.failed_requests,
|
||||
cache_read_input_tokens = "LiteLLM_DailyTeamSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
|
||||
cache_creation_input_tokens = "LiteLLM_DailyTeamSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
|
||||
total_response_time_ms = "LiteLLM_DailyTeamSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
|
||||
timed_requests = "LiteLLM_DailyTeamSpend".timed_requests + EXCLUDED.timed_requests,
|
||||
spend = "LiteLLM_DailyTeamSpend".spend + EXCLUDED.spend,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
|
|
@ -0,0 +1,40 @@
|
|||
INSERT INTO "LiteLLM_DailyUserSpend" (
|
||||
id, user_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint, model_group, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests, failed_requests,
|
||||
cache_read_input_tokens, cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, updated_at
|
||||
)
|
||||
SELECT gen_random_uuid()::text, r.entity_id, r.date, r.api_key, r.model,
|
||||
r.custom_llm_provider, r.mcp_namespaced_tool_name, r.endpoint, r.model_group, r.prompt_tokens,
|
||||
r.completion_tokens, r.api_requests, r.successful_requests,
|
||||
r.failed_requests, r.cache_read, r.cache_creation, r.response_time,
|
||||
r.timed_requests, r.spend, (NOW() AT TIME ZONE 'UTC')
|
||||
FROM UNNEST(
|
||||
$1::text[], $2::text[], $3::text[], $4::text[], $5::text[],
|
||||
$6::text[], $7::text[], $8::int8[], $9::int8[], $10::int8[],
|
||||
$11::int8[], $12::int8[], $13::int8[], $14::int8[], $15::int8[],
|
||||
$16::int8[], $17::float8[], $18::text[]
|
||||
) AS r(
|
||||
entity_id, date, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, prompt_tokens, completion_tokens, api_requests,
|
||||
successful_requests, failed_requests, cache_read, cache_creation,
|
||||
response_time, timed_requests, spend, mcp_namespaced_tool_name
|
||||
)
|
||||
ORDER BY r.date, r.entity_id, r.api_key, r.model, r.custom_llm_provider,
|
||||
r.mcp_namespaced_tool_name, r.endpoint
|
||||
ON CONFLICT (
|
||||
user_id, date, api_key, model, custom_llm_provider,
|
||||
mcp_namespaced_tool_name, endpoint
|
||||
) DO UPDATE SET
|
||||
prompt_tokens = "LiteLLM_DailyUserSpend".prompt_tokens + EXCLUDED.prompt_tokens,
|
||||
completion_tokens = "LiteLLM_DailyUserSpend".completion_tokens + EXCLUDED.completion_tokens,
|
||||
api_requests = "LiteLLM_DailyUserSpend".api_requests + EXCLUDED.api_requests,
|
||||
successful_requests = "LiteLLM_DailyUserSpend".successful_requests + EXCLUDED.successful_requests,
|
||||
failed_requests = "LiteLLM_DailyUserSpend".failed_requests + EXCLUDED.failed_requests,
|
||||
cache_read_input_tokens = "LiteLLM_DailyUserSpend".cache_read_input_tokens + EXCLUDED.cache_read_input_tokens,
|
||||
cache_creation_input_tokens = "LiteLLM_DailyUserSpend".cache_creation_input_tokens + EXCLUDED.cache_creation_input_tokens,
|
||||
total_response_time_ms = "LiteLLM_DailyUserSpend".total_response_time_ms + EXCLUDED.total_response_time_ms,
|
||||
timed_requests = "LiteLLM_DailyUserSpend".timed_requests + EXCLUDED.timed_requests,
|
||||
spend = "LiteLLM_DailyUserSpend".spend + EXCLUDED.spend,
|
||||
updated_at = (NOW() AT TIME ZONE 'UTC')
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
INSERT INTO "LiteLLM_TeamMembership" (user_id, team_id, spend, total_spend)
|
||||
SELECT member.user_id, member.team_id, member.cost, member.cost
|
||||
FROM jsonb_to_recordset($1::jsonb) AS member(user_id text, team_id text, cost float8)
|
||||
WHERE EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_TeamTable" t
|
||||
WHERE t.team_id = member.team_id
|
||||
AND t.members_with_roles @> jsonb_build_array(jsonb_build_object('user_id', member.user_id))
|
||||
)
|
||||
OR EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_TeamMembership" m
|
||||
WHERE m.user_id = member.user_id AND m.team_id = member.team_id
|
||||
)
|
||||
ON CONFLICT (user_id, team_id) DO UPDATE
|
||||
SET spend = "LiteLLM_TeamMembership".spend + EXCLUDED.spend,
|
||||
total_spend = "LiteLLM_TeamMembership".total_spend + EXCLUDED.total_spend
|
||||
76
litellm-rust/crates/db/src/daily_spend.rs
Normal file
76
litellm-rust/crates/db/src/daily_spend.rs
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
use sqlx::types::chrono::NaiveDateTime;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_row)]
|
||||
pub struct DailyUserRow {
|
||||
pub id: String,
|
||||
pub user_id: Option<String>,
|
||||
pub api_key: String,
|
||||
pub model: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
pub mcp_namespaced_tool_name: Option<String>,
|
||||
pub prompt_tokens: i64,
|
||||
pub completion_tokens: i64,
|
||||
pub api_requests: i64,
|
||||
pub successful_requests: i64,
|
||||
pub failed_requests: i64,
|
||||
pub cache_read_input_tokens: i64,
|
||||
pub cache_creation_input_tokens: i64,
|
||||
pub total_response_time_ms: i64,
|
||||
pub timed_requests: i64,
|
||||
pub spend: f64,
|
||||
pub date: String,
|
||||
pub created_at: NaiveDateTime,
|
||||
pub updated_at: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_row)]
|
||||
pub struct DailyTeamRow {
|
||||
pub id: String,
|
||||
pub team_id: Option<String>,
|
||||
pub api_key: String,
|
||||
pub model: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
pub mcp_namespaced_tool_name: Option<String>,
|
||||
pub prompt_tokens: i64,
|
||||
pub completion_tokens: i64,
|
||||
pub api_requests: i64,
|
||||
pub successful_requests: i64,
|
||||
pub failed_requests: i64,
|
||||
pub cache_read_input_tokens: i64,
|
||||
pub cache_creation_input_tokens: i64,
|
||||
pub total_response_time_ms: i64,
|
||||
pub timed_requests: i64,
|
||||
pub spend: f64,
|
||||
pub date: String,
|
||||
pub created_at: NaiveDateTime,
|
||||
pub updated_at: NaiveDateTime,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_row)]
|
||||
pub struct DailyEndUserRow {
|
||||
pub id: String,
|
||||
pub end_user_id: Option<String>,
|
||||
pub api_key: String,
|
||||
pub model: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub endpoint: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
pub mcp_namespaced_tool_name: Option<String>,
|
||||
pub prompt_tokens: i64,
|
||||
pub completion_tokens: i64,
|
||||
pub api_requests: i64,
|
||||
pub successful_requests: i64,
|
||||
pub failed_requests: i64,
|
||||
pub cache_read_input_tokens: i64,
|
||||
pub cache_creation_input_tokens: i64,
|
||||
pub total_response_time_ms: i64,
|
||||
pub timed_requests: i64,
|
||||
pub spend: f64,
|
||||
pub date: String,
|
||||
pub created_at: NaiveDateTime,
|
||||
pub updated_at: NaiveDateTime,
|
||||
}
|
||||
359
litellm-rust/crates/db/src/end_user.rs
Normal file
359
litellm-rust/crates/db/src/end_user.rs
Normal file
|
|
@ -0,0 +1,359 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct RateLimits {
|
||||
pub rpm: Option<i64>,
|
||||
pub tpm: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq)]
|
||||
pub struct EndUser {
|
||||
pub spend: Option<f64>,
|
||||
pub blocked: bool,
|
||||
pub max_budget: Option<f64>,
|
||||
pub limits: RateLimits,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_row)]
|
||||
struct EndUserRow {
|
||||
spend: Option<f64>,
|
||||
blocked: Option<bool>,
|
||||
max_budget: Option<f64>,
|
||||
rpm_limit: Option<i64>,
|
||||
tpm_limit: Option<i64>,
|
||||
}
|
||||
|
||||
pub async fn find_end_user(
|
||||
executor: impl PgExecutor<'_>,
|
||||
end_user_id: &str,
|
||||
default_budget_id: Option<&str>,
|
||||
key_budget_id: Option<&str>,
|
||||
) -> Result<EndUser, Error> {
|
||||
let row = sqlx::query_file_as!(
|
||||
EndUserRow,
|
||||
"queries/end_user/find_end_user.sql",
|
||||
end_user_id,
|
||||
default_budget_id,
|
||||
key_budget_id,
|
||||
)
|
||||
.fetch_one(executor)
|
||||
.await?;
|
||||
Ok(EndUser {
|
||||
spend: row.spend,
|
||||
blocked: row.blocked.unwrap_or(false),
|
||||
max_budget: row.max_budget,
|
||||
limits: RateLimits {
|
||||
rpm: row.rpm_limit,
|
||||
tpm: row.tpm_limit,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{EndUser, RateLimits, find_end_user};
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::rstest;
|
||||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct BudgetFixture {
|
||||
budget_id: &'static str,
|
||||
max_budget: f64,
|
||||
rpm_limit: i64,
|
||||
tpm_limit: i64,
|
||||
}
|
||||
|
||||
async fn insert_budget(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
budget: BudgetFixture,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture inserts are outside the compile-checked query set"
|
||||
)]
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO "LiteLLM_BudgetTable"
|
||||
(budget_id, max_budget, rpm_limit, tpm_limit, created_by, updated_by)
|
||||
VALUES ($1, $2, $3, $4, 'db-query-test', 'db-query-test')
|
||||
"#,
|
||||
)
|
||||
.bind(budget.budget_id)
|
||||
.bind(budget.max_budget)
|
||||
.bind(budget.rpm_limit)
|
||||
.bind(budget.tpm_limit)
|
||||
.execute(&mut **tx)
|
||||
.await;
|
||||
result.map(|_| ())
|
||||
}
|
||||
|
||||
async fn insert_end_user(
|
||||
tx: &mut Transaction<'_, Postgres>,
|
||||
user_id: &str,
|
||||
spend: f64,
|
||||
blocked: bool,
|
||||
budget_id: Option<&str>,
|
||||
) -> Result<(), sqlx::Error> {
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture inserts are outside the compile-checked query set"
|
||||
)]
|
||||
let result = sqlx::query(
|
||||
r#"
|
||||
INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind(spend)
|
||||
.bind(blocked)
|
||||
.bind(budget_id)
|
||||
.execute(&mut **tx)
|
||||
.await;
|
||||
result.map(|_| ())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::without_default_budget(None)]
|
||||
#[case::with_default_budget(Some(BudgetFixture {
|
||||
budget_id: "missing-user-default-budget",
|
||||
max_budget: 11.5,
|
||||
rpm_limit: 12,
|
||||
tpm_limit: 1300,
|
||||
}))]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn missing_end_user_maps_to_defaults(
|
||||
#[case] default_budget: Option<BudgetFixture>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
if let Some(budget) = default_budget {
|
||||
insert_budget(&mut tx, budget).await?;
|
||||
}
|
||||
|
||||
let actual = find_end_user(
|
||||
&mut *tx,
|
||||
"missing-end-user",
|
||||
default_budget.map(|budget| budget.budget_id),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
actual,
|
||||
EndUser {
|
||||
spend: None,
|
||||
blocked: false,
|
||||
max_budget: default_budget.map(|budget| budget.max_budget),
|
||||
limits: RateLimits {
|
||||
rpm: default_budget.map(|budget| budget.rpm_limit),
|
||||
tpm: default_budget.map(|budget| budget.tpm_limit),
|
||||
},
|
||||
}
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::row_budget_precedes_default(
|
||||
"row-budget-priority-user",
|
||||
Some(BudgetFixture {
|
||||
budget_id: "row-budget-priority-row",
|
||||
max_budget: 10.0,
|
||||
rpm_limit: 101,
|
||||
tpm_limit: 1001,
|
||||
}),
|
||||
BudgetFixture {
|
||||
budget_id: "row-budget-priority-default",
|
||||
max_budget: 20.0,
|
||||
rpm_limit: 201,
|
||||
tpm_limit: 2001,
|
||||
},
|
||||
None,
|
||||
Some(BudgetFixture {
|
||||
budget_id: "row-budget-priority-row",
|
||||
max_budget: 10.0,
|
||||
rpm_limit: 101,
|
||||
tpm_limit: 1001,
|
||||
}),
|
||||
false
|
||||
)]
|
||||
#[case::default_applies_without_row_budget(
|
||||
"default-budget-user",
|
||||
None,
|
||||
BudgetFixture {
|
||||
budget_id: "default-budget-fallback",
|
||||
max_budget: 30.0,
|
||||
rpm_limit: 301,
|
||||
tpm_limit: 3001,
|
||||
},
|
||||
None,
|
||||
Some(BudgetFixture {
|
||||
budget_id: "default-budget-fallback",
|
||||
max_budget: 30.0,
|
||||
rpm_limit: 301,
|
||||
tpm_limit: 3001,
|
||||
}),
|
||||
false
|
||||
)]
|
||||
#[case::assigned_budget_precedes_key_when_all_ids_differ(
|
||||
"override-priority-user",
|
||||
Some(BudgetFixture {
|
||||
budget_id: "override-priority-row",
|
||||
max_budget: 40.0,
|
||||
rpm_limit: 401,
|
||||
tpm_limit: 4001,
|
||||
}),
|
||||
BudgetFixture {
|
||||
budget_id: "override-priority-default",
|
||||
max_budget: 50.0,
|
||||
rpm_limit: 501,
|
||||
tpm_limit: 5001,
|
||||
},
|
||||
Some(BudgetFixture {
|
||||
budget_id: "override-priority-override",
|
||||
max_budget: 60.0,
|
||||
rpm_limit: 601,
|
||||
tpm_limit: 6001,
|
||||
}),
|
||||
Some(BudgetFixture {
|
||||
budget_id: "override-priority-row",
|
||||
max_budget: 40.0,
|
||||
rpm_limit: 401,
|
||||
tpm_limit: 4001,
|
||||
}),
|
||||
true
|
||||
)]
|
||||
#[case::missing_key_budget_keeps_assigned_budget(
|
||||
"dangling-override-user",
|
||||
Some(BudgetFixture {
|
||||
budget_id: "dangling-override-row",
|
||||
max_budget: 70.0,
|
||||
rpm_limit: 701,
|
||||
tpm_limit: 7001,
|
||||
}),
|
||||
BudgetFixture {
|
||||
budget_id: "dangling-override-default",
|
||||
max_budget: 80.0,
|
||||
rpm_limit: 801,
|
||||
tpm_limit: 8001,
|
||||
},
|
||||
Some(BudgetFixture {
|
||||
budget_id: "dangling-override-missing",
|
||||
max_budget: 90.0,
|
||||
rpm_limit: 901,
|
||||
tpm_limit: 9001,
|
||||
}),
|
||||
Some(BudgetFixture {
|
||||
budget_id: "dangling-override-row",
|
||||
max_budget: 70.0,
|
||||
rpm_limit: 701,
|
||||
tpm_limit: 7001,
|
||||
}),
|
||||
false
|
||||
)]
|
||||
#[case::key_budget_precedes_proxy_default(
|
||||
"key-default-user",
|
||||
None,
|
||||
BudgetFixture {
|
||||
budget_id: "key-default-proxy",
|
||||
max_budget: 80.0,
|
||||
rpm_limit: 801,
|
||||
tpm_limit: 8001,
|
||||
},
|
||||
Some(BudgetFixture {
|
||||
budget_id: "key-default-key",
|
||||
max_budget: 90.0,
|
||||
rpm_limit: 901,
|
||||
tpm_limit: 9001,
|
||||
}),
|
||||
Some(BudgetFixture {
|
||||
budget_id: "key-default-key",
|
||||
max_budget: 90.0,
|
||||
rpm_limit: 901,
|
||||
tpm_limit: 9001,
|
||||
}),
|
||||
true
|
||||
)]
|
||||
#[case::missing_key_budget_falls_back_to_proxy_default(
|
||||
"missing-key-default-user",
|
||||
None,
|
||||
BudgetFixture {
|
||||
budget_id: "missing-key-default-proxy",
|
||||
max_budget: 80.0,
|
||||
rpm_limit: 801,
|
||||
tpm_limit: 8001,
|
||||
},
|
||||
Some(BudgetFixture {
|
||||
budget_id: "missing-key-default-key",
|
||||
max_budget: 90.0,
|
||||
rpm_limit: 901,
|
||||
tpm_limit: 9001,
|
||||
}),
|
||||
Some(BudgetFixture {
|
||||
budget_id: "missing-key-default-proxy",
|
||||
max_budget: 80.0,
|
||||
rpm_limit: 801,
|
||||
tpm_limit: 8001,
|
||||
}),
|
||||
false
|
||||
)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn budget_resolution_matches_python_precedence(
|
||||
#[case] user_id: &'static str,
|
||||
#[case] user_budget: Option<BudgetFixture>,
|
||||
#[case] default_budget: BudgetFixture,
|
||||
#[case] override_budget: Option<BudgetFixture>,
|
||||
#[case] expected_budget: Option<BudgetFixture>,
|
||||
#[case] insert_override_budget: bool,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
insert_budget(&mut tx, default_budget).await?;
|
||||
if let Some(budget) = user_budget {
|
||||
insert_budget(&mut tx, budget).await?;
|
||||
}
|
||||
if let (Some(budget), true) = (override_budget, insert_override_budget) {
|
||||
insert_budget(&mut tx, budget).await?;
|
||||
}
|
||||
insert_end_user(
|
||||
&mut tx,
|
||||
user_id,
|
||||
7.5,
|
||||
true,
|
||||
user_budget.map(|budget| budget.budget_id),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let actual = find_end_user(
|
||||
&mut *tx,
|
||||
user_id,
|
||||
Some(default_budget.budget_id),
|
||||
override_budget.map(|budget| budget.budget_id),
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_eq!(
|
||||
actual,
|
||||
EndUser {
|
||||
spend: Some(7.5),
|
||||
blocked: true,
|
||||
max_budget: expected_budget.map(|budget| budget.max_budget),
|
||||
limits: RateLimits {
|
||||
rpm: expected_budget.map(|budget| budget.rpm_limit),
|
||||
tpm: expected_budget.map(|budget| budget.tpm_limit),
|
||||
},
|
||||
}
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
79
litellm-rust/crates/db/src/error.rs
Normal file
79
litellm-rust/crates/db/src/error.rs
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum Error {
|
||||
#[error("database query failed")]
|
||||
Query(#[from] sqlx::Error),
|
||||
#[error("spend batch has {ids} ids and {costs} costs")]
|
||||
InvalidSpendBatch { ids: usize, costs: usize },
|
||||
#[error("invalid spend timestamp")]
|
||||
InvalidTimestamp(#[from] chrono::ParseError),
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn is_retryable(&self) -> bool {
|
||||
match self {
|
||||
Self::Query(sqlx::Error::Database(error)) => {
|
||||
matches!(
|
||||
error.code().as_deref(),
|
||||
Some("40P01" | "55P03" | "40001" | "53300")
|
||||
)
|
||||
}
|
||||
Self::Query(sqlx::Error::Io(_) | sqlx::Error::PoolTimedOut) => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
use super::Error;
|
||||
use crate::test_support::database_pool;
|
||||
|
||||
#[rstest]
|
||||
#[case::closed_pool(sqlx::Error::PoolClosed, false)]
|
||||
#[case::pool_timeout(sqlx::Error::PoolTimedOut, true)]
|
||||
#[case::io(sqlx::Error::Io(std::io::ErrorKind::ConnectionReset.into()), true)]
|
||||
#[case::missing_row(sqlx::Error::RowNotFound, false)]
|
||||
fn classifies_client_errors(#[case] error: sqlx::Error, #[case] retryable: bool) {
|
||||
assert_eq!(Error::from(error).is_retryable(), retryable);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::connection_capacity("53300", true)]
|
||||
#[case::deadlock("40P01", true)]
|
||||
#[case::lock_unavailable("55P03", true)]
|
||||
#[case::serialization_failure("40001", true)]
|
||||
#[case::unique_violation("23505", false)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn classifies_server_errors(
|
||||
#[case] code: &str,
|
||||
#[case] retryable: bool,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test query must raise the requested SQLSTATE"
|
||||
)]
|
||||
let error = sqlx::query(sqlx::AssertSqlSafe(format!(
|
||||
"DO $$ BEGIN RAISE EXCEPTION USING ERRCODE = '{code}'; END $$"
|
||||
)))
|
||||
.execute(&mut *tx)
|
||||
.await
|
||||
.expect_err("PostgreSQL must raise the requested error");
|
||||
|
||||
assert_eq!(
|
||||
error.as_database_error().and_then(|error| error.code()),
|
||||
Some(code.into())
|
||||
);
|
||||
assert_eq!(Error::from(error).is_retryable(), retryable);
|
||||
tx.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -1,3 +1,26 @@
|
|||
mod failure;
|
||||
macro_rules_attribute::attribute_alias! {
|
||||
#[apply(db_input)] =
|
||||
#[derive(Clone, Debug, PartialEq)];
|
||||
|
||||
#[apply(db_row)] =
|
||||
#[derive(Clone, Debug, PartialEq, sqlx::FromRow)];
|
||||
}
|
||||
|
||||
mod daily_spend;
|
||||
mod end_user;
|
||||
mod error;
|
||||
mod failure;
|
||||
mod spend;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
|
||||
pub use daily_spend::{DailyEndUserRow, DailyTeamRow, DailyUserRow};
|
||||
pub use end_user::{EndUser, RateLimits, find_end_user};
|
||||
pub use error::Error;
|
||||
pub use failure::DbFailure;
|
||||
pub use spend::{
|
||||
DailySpend, DailyTable, EntitySpend, SpendLog, SpendLogRow, TeamMemberSpend,
|
||||
add_end_user_spend, add_key_spend, add_team_member_spend, add_team_spend, add_user_spend,
|
||||
begin, insert_spend_logs, lock_team, upsert_daily_end_user_spend, upsert_daily_spend,
|
||||
upsert_daily_team_spend, upsert_daily_user_spend,
|
||||
};
|
||||
|
|
|
|||
584
litellm-rust/crates/db/src/spend/daily_end_user_spend.rs
Normal file
584
litellm-rust/crates/db/src/spend/daily_end_user_spend.rs
Normal file
|
|
@ -0,0 +1,584 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use super::{DailySpend, upsert_daily_team_spend, upsert_daily_user_spend};
|
||||
use crate::Error;
|
||||
|
||||
pub async fn upsert_daily_end_user_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
rows: &[DailySpend],
|
||||
) -> Result<(), Error> {
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
macro_rules! column {
|
||||
($field:ident) => {
|
||||
rows.iter()
|
||||
.map(|row| row.$field.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
}
|
||||
sqlx::query_file!(
|
||||
"queries/spend/upsert_daily_end_user_spend.sql",
|
||||
&column!(entity_id),
|
||||
&column!(date),
|
||||
&column!(api_key),
|
||||
&column!(model),
|
||||
&column!(custom_llm_provider),
|
||||
&column!(endpoint),
|
||||
&column!(model_group) as &[Option<String>],
|
||||
&column!(prompt_tokens),
|
||||
&column!(completion_tokens),
|
||||
&column!(api_requests),
|
||||
&column!(successful_requests),
|
||||
&column!(failed_requests),
|
||||
&column!(cache_read_input_tokens),
|
||||
&column!(cache_creation_input_tokens),
|
||||
&column!(total_response_time_ms),
|
||||
&column!(timed_requests),
|
||||
&column!(spend),
|
||||
&column!(mcp_namespaced_tool_name),
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq, strum::VariantArray)]
|
||||
pub enum DailyTable {
|
||||
User,
|
||||
Team,
|
||||
EndUser,
|
||||
}
|
||||
|
||||
pub async fn upsert_daily_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
table: DailyTable,
|
||||
rows: &[DailySpend],
|
||||
) -> Result<(), Error> {
|
||||
match table {
|
||||
DailyTable::User => upsert_daily_user_spend(executor, rows).await,
|
||||
DailyTable::Team => upsert_daily_team_spend(executor, rows).await,
|
||||
DailyTable::EndUser => upsert_daily_end_user_spend(executor, rows).await,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{DailySpend, DailyTable, upsert_daily_end_user_spend, upsert_daily_spend};
|
||||
use crate::{DailyEndUserRow, test_support::database_pool};
|
||||
use rstest::rstest;
|
||||
use sqlx::{PgPool, types::chrono::NaiveDateTime};
|
||||
|
||||
#[rstest]
|
||||
#[case::full_keys(
|
||||
"daily-end-user-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
Some("first-group")
|
||||
)]
|
||||
#[case::initial_model_group_none(
|
||||
"daily-end-user-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
None
|
||||
)]
|
||||
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_end_user_spend_upsert_preserves_keys_and_accumulates(
|
||||
#[case] end_user_id: &'static str,
|
||||
#[case] api_key: &'static str,
|
||||
#[case] model: &'static str,
|
||||
#[case] custom_llm_provider: &'static str,
|
||||
#[case] endpoint: &'static str,
|
||||
#[case] first_model_group: Option<&'static str>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
|
||||
)]
|
||||
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let first_batch = [
|
||||
DailySpend {
|
||||
entity_id: end_user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: first_model_group.map(str::to_owned),
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 2,
|
||||
api_requests: 3,
|
||||
successful_requests: 4,
|
||||
failed_requests: 5,
|
||||
cache_read_input_tokens: 6,
|
||||
cache_creation_input_tokens: 7,
|
||||
total_response_time_ms: 8,
|
||||
timed_requests: 9,
|
||||
spend: 0.125,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: end_user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: "endpoint-sentinel".to_owned(),
|
||||
model_group: Some("sentinel-endpoint-group".to_owned()),
|
||||
prompt_tokens: 101,
|
||||
completion_tokens: 102,
|
||||
api_requests: 103,
|
||||
successful_requests: 104,
|
||||
failed_requests: 105,
|
||||
cache_read_input_tokens: 106,
|
||||
cache_creation_input_tokens: 107,
|
||||
total_response_time_ms: 108,
|
||||
timed_requests: 109,
|
||||
spend: 1.25,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: end_user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: "api-key-sentinel".to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-key-group".to_owned()),
|
||||
prompt_tokens: 201,
|
||||
completion_tokens: 202,
|
||||
api_requests: 203,
|
||||
successful_requests: 204,
|
||||
failed_requests: 205,
|
||||
cache_read_input_tokens: 206,
|
||||
cache_creation_input_tokens: 207,
|
||||
total_response_time_ms: 208,
|
||||
timed_requests: 209,
|
||||
spend: 2.5,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: "entity-sentinel".to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-entity-group".to_owned()),
|
||||
prompt_tokens: 301,
|
||||
completion_tokens: 302,
|
||||
api_requests: 303,
|
||||
successful_requests: 304,
|
||||
failed_requests: 305,
|
||||
cache_read_input_tokens: 306,
|
||||
cache_creation_input_tokens: 307,
|
||||
total_response_time_ms: 308,
|
||||
timed_requests: 309,
|
||||
spend: 3.75,
|
||||
},
|
||||
];
|
||||
upsert_daily_end_user_spend(&mut *tx, &first_batch).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the inserted row identity before conflict handling"
|
||||
)]
|
||||
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
|
||||
r#"SELECT id, created_at
|
||||
FROM "LiteLLM_DailyEndUserSpend"
|
||||
WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4
|
||||
AND custom_llm_provider = $5 AND endpoint = $6"#,
|
||||
)
|
||||
.bind(end_user_id)
|
||||
.bind("2026-01-15")
|
||||
.bind(api_key)
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(endpoint)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert!(!initial_id.is_empty());
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test seeds updated_at before exercising the conflict update"
|
||||
)]
|
||||
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
|
||||
r#"UPDATE "LiteLLM_DailyEndUserSpend"
|
||||
SET updated_at = TIMESTAMP '2000-01-01'
|
||||
WHERE id = $1
|
||||
RETURNING updated_at"#,
|
||||
)
|
||||
.bind(&initial_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let increment = [DailySpend {
|
||||
entity_id: end_user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("second-group".to_owned()),
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
api_requests: 30,
|
||||
successful_requests: 40,
|
||||
failed_requests: 50,
|
||||
cache_read_input_tokens: 60,
|
||||
cache_creation_input_tokens: 70,
|
||||
total_response_time_ms: 80,
|
||||
timed_requests: 90,
|
||||
spend: 0.625,
|
||||
}];
|
||||
upsert_daily_end_user_spend(&mut *tx, &increment).await?;
|
||||
upsert_daily_end_user_spend(&mut *tx, &[]).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the transaction timestamp for assertions"
|
||||
)]
|
||||
let expected_updated_at: NaiveDateTime =
|
||||
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads all rows to verify attribution and accumulation"
|
||||
)]
|
||||
let rows: Vec<DailyEndUserRow> = sqlx::query_as(
|
||||
r#"SELECT id, end_user_id, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, mcp_namespaced_tool_name, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests,
|
||||
failed_requests, cache_read_input_tokens,
|
||||
cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, date, created_at, updated_at
|
||||
FROM "LiteLLM_DailyEndUserSpend"
|
||||
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
|
||||
AND mcp_namespaced_tool_name = '' AND end_user_id = ANY($4)
|
||||
ORDER BY CASE
|
||||
WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
|
||||
WHEN end_user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
|
||||
WHEN end_user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
|
||||
WHEN end_user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
|
||||
ELSE 4
|
||||
END"#,
|
||||
)
|
||||
.bind("2026-01-15")
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(vec![end_user_id, "entity-sentinel"])
|
||||
.bind(end_user_id)
|
||||
.bind(api_key)
|
||||
.bind(endpoint)
|
||||
.bind("api-key-sentinel")
|
||||
.bind("endpoint-sentinel")
|
||||
.bind("entity-sentinel")
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 4);
|
||||
assert_eq!(rows[0].id, initial_id);
|
||||
assert_eq!(rows[0].end_user_id.as_deref(), Some(end_user_id));
|
||||
assert_eq!(rows[0].api_key, api_key);
|
||||
assert_eq!(rows[0].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[0].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
|
||||
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[0].prompt_tokens,
|
||||
rows[0].completion_tokens,
|
||||
rows[0].api_requests,
|
||||
rows[0].successful_requests,
|
||||
rows[0].failed_requests,
|
||||
rows[0].cache_read_input_tokens,
|
||||
rows[0].cache_creation_input_tokens,
|
||||
rows[0].total_response_time_ms,
|
||||
rows[0].timed_requests,
|
||||
rows[0].spend
|
||||
),
|
||||
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
|
||||
);
|
||||
assert_eq!(rows[0].date, "2026-01-15");
|
||||
assert_eq!(rows[0].created_at, initial_created_at);
|
||||
assert!(rows[0].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[0].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[1].id.is_empty());
|
||||
assert_eq!(rows[1].end_user_id.as_deref(), Some(end_user_id));
|
||||
assert_eq!(rows[1].api_key, api_key);
|
||||
assert_eq!(rows[1].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[1].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
|
||||
assert_eq!(
|
||||
rows[1].model_group.as_deref(),
|
||||
Some("sentinel-endpoint-group")
|
||||
);
|
||||
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[1].prompt_tokens,
|
||||
rows[1].completion_tokens,
|
||||
rows[1].api_requests,
|
||||
rows[1].successful_requests,
|
||||
rows[1].failed_requests,
|
||||
rows[1].cache_read_input_tokens,
|
||||
rows[1].cache_creation_input_tokens,
|
||||
rows[1].total_response_time_ms,
|
||||
rows[1].timed_requests,
|
||||
rows[1].spend
|
||||
),
|
||||
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
|
||||
);
|
||||
assert_eq!(rows[1].date, "2026-01-15");
|
||||
assert!(rows[1].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[1].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[2].id.is_empty());
|
||||
assert_eq!(rows[2].end_user_id.as_deref(), Some(end_user_id));
|
||||
assert_eq!(rows[2].api_key, "api-key-sentinel");
|
||||
assert_eq!(rows[2].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[2].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
|
||||
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[2].prompt_tokens,
|
||||
rows[2].completion_tokens,
|
||||
rows[2].api_requests,
|
||||
rows[2].successful_requests,
|
||||
rows[2].failed_requests,
|
||||
rows[2].cache_read_input_tokens,
|
||||
rows[2].cache_creation_input_tokens,
|
||||
rows[2].total_response_time_ms,
|
||||
rows[2].timed_requests,
|
||||
rows[2].spend
|
||||
),
|
||||
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
|
||||
);
|
||||
assert_eq!(rows[2].date, "2026-01-15");
|
||||
assert!(rows[2].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[2].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[3].id.is_empty());
|
||||
assert_eq!(rows[3].end_user_id.as_deref(), Some("entity-sentinel"));
|
||||
assert_eq!(rows[3].api_key, api_key);
|
||||
assert_eq!(rows[3].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[3].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(
|
||||
rows[3].model_group.as_deref(),
|
||||
Some("sentinel-entity-group")
|
||||
);
|
||||
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[3].prompt_tokens,
|
||||
rows[3].completion_tokens,
|
||||
rows[3].api_requests,
|
||||
rows[3].successful_requests,
|
||||
rows[3].failed_requests,
|
||||
rows[3].cache_read_input_tokens,
|
||||
rows[3].cache_creation_input_tokens,
|
||||
rows[3].total_response_time_ms,
|
||||
rows[3].timed_requests,
|
||||
rows[3].spend
|
||||
),
|
||||
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
|
||||
);
|
||||
assert_eq!(rows[3].date, "2026-01-15");
|
||||
assert!(rows[3].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[3].updated_at, expected_updated_at);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::distinct_tools("server/tool-a", "server/tool-b")]
|
||||
#[case::empty_and_named_tool("", "server/tool-b")]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_end_user_spend_preserves_tool_attribution(
|
||||
#[case] first_tool: &str,
|
||||
#[case] second_tool: &str,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
let first = DailySpend {
|
||||
entity_id: "daily-end-user-tools".to_owned(),
|
||||
date: "2026-02-21".to_owned(),
|
||||
api_key: "tool-key".to_owned(),
|
||||
model: "tool-model".to_owned(),
|
||||
custom_llm_provider: "tool-provider".to_owned(),
|
||||
endpoint: "/mcp".to_owned(),
|
||||
mcp_namespaced_tool_name: first_tool.to_owned(),
|
||||
prompt_tokens: 2,
|
||||
spend: 0.25,
|
||||
..Default::default()
|
||||
};
|
||||
let second = DailySpend {
|
||||
mcp_namespaced_tool_name: second_tool.to_owned(),
|
||||
prompt_tokens: 5,
|
||||
spend: 0.5,
|
||||
..first.clone()
|
||||
};
|
||||
upsert_daily_end_user_spend(&mut *tx, &[second, first.clone()]).await?;
|
||||
upsert_daily_end_user_spend(&mut *tx, &[first]).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads stored tool attribution and counters"
|
||||
)]
|
||||
let rows: Vec<DailyEndUserRow> = sqlx::query_as(
|
||||
r#"SELECT * FROM "LiteLLM_DailyEndUserSpend"
|
||||
WHERE end_user_id = $1 AND date = $2 AND api_key = $3
|
||||
ORDER BY mcp_namespaced_tool_name"#,
|
||||
)
|
||||
.bind("daily-end-user-tools")
|
||||
.bind("2026-02-21")
|
||||
.bind("tool-key")
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(
|
||||
rows[0].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(first_tool)
|
||||
);
|
||||
assert_eq!(
|
||||
rows[1].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(second_tool)
|
||||
);
|
||||
assert_eq!((rows[0].prompt_tokens, rows[0].spend), (4, 0.5));
|
||||
assert_eq!((rows[1].prompt_tokens, rows[1].spend), (5, 0.5));
|
||||
assert_ne!(rows[0].id, rows[1].id);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::user(DailyTable::User, "daily-dispatch-user")]
|
||||
#[case::team(DailyTable::Team, "daily-dispatch-team")]
|
||||
#[case::end_user(DailyTable::EndUser, "daily-dispatch-end-user")]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_spend_dispatches_to_selected_table(
|
||||
#[case] table: DailyTable,
|
||||
#[case] entity_id: &'static str,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
let api_key = format!("{entity_id}-key");
|
||||
let rows = [DailySpend {
|
||||
entity_id: entity_id.to_owned(),
|
||||
date: "2026-02-20".to_owned(),
|
||||
api_key: api_key.clone(),
|
||||
model: "dispatcher-model".to_owned(),
|
||||
custom_llm_provider: "dispatcher-provider".to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: "/dispatcher".to_owned(),
|
||||
model_group: Some("dispatcher-group".to_owned()),
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 2,
|
||||
api_requests: 3,
|
||||
successful_requests: 4,
|
||||
failed_requests: 5,
|
||||
cache_read_input_tokens: 6,
|
||||
cache_creation_input_tokens: 7,
|
||||
total_response_time_ms: 8,
|
||||
timed_requests: 9,
|
||||
spend: 0.125,
|
||||
}];
|
||||
upsert_daily_spend(&mut *tx, table, &rows).await?;
|
||||
|
||||
let count: i64 = match table {
|
||||
DailyTable::User =>
|
||||
{
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test checks the user-table dispatcher route"
|
||||
)]
|
||||
sqlx::query_scalar(
|
||||
r#"SELECT COUNT(*) FROM "LiteLLM_DailyUserSpend"
|
||||
WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
|
||||
)
|
||||
.bind(entity_id)
|
||||
.bind("2026-02-20")
|
||||
.bind(&api_key)
|
||||
.bind("dispatcher-model")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?
|
||||
}
|
||||
DailyTable::Team =>
|
||||
{
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test checks the team-table dispatcher route"
|
||||
)]
|
||||
sqlx::query_scalar(
|
||||
r#"SELECT COUNT(*) FROM "LiteLLM_DailyTeamSpend"
|
||||
WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
|
||||
)
|
||||
.bind(entity_id)
|
||||
.bind("2026-02-20")
|
||||
.bind(&api_key)
|
||||
.bind("dispatcher-model")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?
|
||||
}
|
||||
DailyTable::EndUser =>
|
||||
{
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test checks the end-user-table dispatcher route"
|
||||
)]
|
||||
sqlx::query_scalar(
|
||||
r#"SELECT COUNT(*) FROM "LiteLLM_DailyEndUserSpend"
|
||||
WHERE end_user_id = $1 AND date = $2 AND api_key = $3 AND model = $4"#,
|
||||
)
|
||||
.bind(entity_id)
|
||||
.bind("2026-02-20")
|
||||
.bind(&api_key)
|
||||
.bind("dispatcher-model")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?
|
||||
}
|
||||
};
|
||||
assert_eq!(count, 1);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
485
litellm-rust/crates/db/src/spend/daily_team_spend.rs
Normal file
485
litellm-rust/crates/db/src/spend/daily_team_spend.rs
Normal file
|
|
@ -0,0 +1,485 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use super::DailySpend;
|
||||
use crate::Error;
|
||||
|
||||
pub async fn upsert_daily_team_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
rows: &[DailySpend],
|
||||
) -> Result<(), Error> {
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
macro_rules! column {
|
||||
($field:ident) => {
|
||||
rows.iter()
|
||||
.map(|row| row.$field.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
}
|
||||
sqlx::query_file!(
|
||||
"queries/spend/upsert_daily_team_spend.sql",
|
||||
&column!(entity_id),
|
||||
&column!(date),
|
||||
&column!(api_key),
|
||||
&column!(model),
|
||||
&column!(custom_llm_provider),
|
||||
&column!(endpoint),
|
||||
&column!(model_group) as &[Option<String>],
|
||||
&column!(prompt_tokens),
|
||||
&column!(completion_tokens),
|
||||
&column!(api_requests),
|
||||
&column!(successful_requests),
|
||||
&column!(failed_requests),
|
||||
&column!(cache_read_input_tokens),
|
||||
&column!(cache_creation_input_tokens),
|
||||
&column!(total_response_time_ms),
|
||||
&column!(timed_requests),
|
||||
&column!(spend),
|
||||
&column!(mcp_namespaced_tool_name),
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{DailySpend, upsert_daily_team_spend};
|
||||
use crate::{DailyTeamRow, test_support::database_pool};
|
||||
use rstest::rstest;
|
||||
use sqlx::{PgPool, types::chrono::NaiveDateTime};
|
||||
|
||||
#[rstest]
|
||||
#[case::distinct_tools("server.fetch", "server.search")]
|
||||
#[case::default_and_named_tool("", "server.search")]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_team_spend_separates_tools_and_accumulates(
|
||||
#[case] first_tool: &str,
|
||||
#[case] second_tool: &str,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
let first = DailySpend {
|
||||
entity_id: "daily-team-tools".to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: "daily-key-tools".to_owned(),
|
||||
model: "daily-model".to_owned(),
|
||||
custom_llm_provider: "daily-provider".to_owned(),
|
||||
endpoint: "/mcp".to_owned(),
|
||||
mcp_namespaced_tool_name: first_tool.to_owned(),
|
||||
api_requests: 1,
|
||||
spend: 0.125,
|
||||
..Default::default()
|
||||
};
|
||||
let second = DailySpend {
|
||||
mcp_namespaced_tool_name: second_tool.to_owned(),
|
||||
api_requests: 2,
|
||||
spend: 0.25,
|
||||
..first.clone()
|
||||
};
|
||||
upsert_daily_team_spend(&mut *tx, &[second.clone(), first.clone()]).await?;
|
||||
upsert_daily_team_spend(
|
||||
&mut *tx,
|
||||
&[
|
||||
DailySpend {
|
||||
api_requests: 3,
|
||||
spend: 0.5,
|
||||
..first
|
||||
},
|
||||
DailySpend {
|
||||
api_requests: 5,
|
||||
spend: 1.0,
|
||||
..second
|
||||
},
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads stored tool attribution and counters"
|
||||
)]
|
||||
let rows = sqlx::query_as::<_, DailyTeamRow>(
|
||||
r#"SELECT * FROM "LiteLLM_DailyTeamSpend"
|
||||
WHERE team_id = $1 AND date = $2
|
||||
ORDER BY mcp_namespaced_tool_name"#,
|
||||
)
|
||||
.bind("daily-team-tools")
|
||||
.bind("2026-01-15")
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(
|
||||
rows[0].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(first_tool)
|
||||
);
|
||||
assert_eq!(
|
||||
rows[1].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(second_tool)
|
||||
);
|
||||
assert_eq!((rows[0].api_requests, rows[0].spend), (4, 0.625));
|
||||
assert_eq!((rows[1].api_requests, rows[1].spend), (7, 1.25));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::full_keys(
|
||||
"daily-team-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
Some("first-group")
|
||||
)]
|
||||
#[case::initial_model_group_none(
|
||||
"daily-team-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
None
|
||||
)]
|
||||
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_team_spend_upsert_preserves_keys_and_accumulates(
|
||||
#[case] team_id: &'static str,
|
||||
#[case] api_key: &'static str,
|
||||
#[case] model: &'static str,
|
||||
#[case] custom_llm_provider: &'static str,
|
||||
#[case] endpoint: &'static str,
|
||||
#[case] first_model_group: Option<&'static str>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
|
||||
)]
|
||||
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let first_batch = [
|
||||
DailySpend {
|
||||
entity_id: team_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: first_model_group.map(str::to_owned),
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 2,
|
||||
api_requests: 3,
|
||||
successful_requests: 4,
|
||||
failed_requests: 5,
|
||||
cache_read_input_tokens: 6,
|
||||
cache_creation_input_tokens: 7,
|
||||
total_response_time_ms: 8,
|
||||
timed_requests: 9,
|
||||
spend: 0.125,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: team_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: "endpoint-sentinel".to_owned(),
|
||||
model_group: Some("sentinel-endpoint-group".to_owned()),
|
||||
prompt_tokens: 101,
|
||||
completion_tokens: 102,
|
||||
api_requests: 103,
|
||||
successful_requests: 104,
|
||||
failed_requests: 105,
|
||||
cache_read_input_tokens: 106,
|
||||
cache_creation_input_tokens: 107,
|
||||
total_response_time_ms: 108,
|
||||
timed_requests: 109,
|
||||
spend: 1.25,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: team_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: "api-key-sentinel".to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-key-group".to_owned()),
|
||||
prompt_tokens: 201,
|
||||
completion_tokens: 202,
|
||||
api_requests: 203,
|
||||
successful_requests: 204,
|
||||
failed_requests: 205,
|
||||
cache_read_input_tokens: 206,
|
||||
cache_creation_input_tokens: 207,
|
||||
total_response_time_ms: 208,
|
||||
timed_requests: 209,
|
||||
spend: 2.5,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: "entity-sentinel".to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-entity-group".to_owned()),
|
||||
prompt_tokens: 301,
|
||||
completion_tokens: 302,
|
||||
api_requests: 303,
|
||||
successful_requests: 304,
|
||||
failed_requests: 305,
|
||||
cache_read_input_tokens: 306,
|
||||
cache_creation_input_tokens: 307,
|
||||
total_response_time_ms: 308,
|
||||
timed_requests: 309,
|
||||
spend: 3.75,
|
||||
},
|
||||
];
|
||||
upsert_daily_team_spend(&mut *tx, &first_batch).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the inserted row identity before conflict handling"
|
||||
)]
|
||||
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
|
||||
r#"SELECT id, created_at
|
||||
FROM "LiteLLM_DailyTeamSpend"
|
||||
WHERE team_id = $1 AND date = $2 AND api_key = $3 AND model = $4
|
||||
AND custom_llm_provider = $5 AND endpoint = $6"#,
|
||||
)
|
||||
.bind(team_id)
|
||||
.bind("2026-01-15")
|
||||
.bind(api_key)
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(endpoint)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert!(!initial_id.is_empty());
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test seeds updated_at before exercising the conflict update"
|
||||
)]
|
||||
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
|
||||
r#"UPDATE "LiteLLM_DailyTeamSpend"
|
||||
SET updated_at = TIMESTAMP '2000-01-01'
|
||||
WHERE id = $1
|
||||
RETURNING updated_at"#,
|
||||
)
|
||||
.bind(&initial_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let increment = [DailySpend {
|
||||
entity_id: team_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("second-group".to_owned()),
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
api_requests: 30,
|
||||
successful_requests: 40,
|
||||
failed_requests: 50,
|
||||
cache_read_input_tokens: 60,
|
||||
cache_creation_input_tokens: 70,
|
||||
total_response_time_ms: 80,
|
||||
timed_requests: 90,
|
||||
spend: 0.625,
|
||||
}];
|
||||
upsert_daily_team_spend(&mut *tx, &increment).await?;
|
||||
upsert_daily_team_spend(&mut *tx, &[]).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the transaction timestamp for assertions"
|
||||
)]
|
||||
let expected_updated_at: NaiveDateTime =
|
||||
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads all rows to verify attribution and accumulation"
|
||||
)]
|
||||
let rows: Vec<DailyTeamRow> = sqlx::query_as(
|
||||
r#"SELECT id, team_id, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, mcp_namespaced_tool_name, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests,
|
||||
failed_requests, cache_read_input_tokens,
|
||||
cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, date, created_at, updated_at
|
||||
FROM "LiteLLM_DailyTeamSpend"
|
||||
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
|
||||
AND mcp_namespaced_tool_name = '' AND team_id = ANY($4)
|
||||
ORDER BY CASE
|
||||
WHEN team_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
|
||||
WHEN team_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
|
||||
WHEN team_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
|
||||
WHEN team_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
|
||||
ELSE 4
|
||||
END"#,
|
||||
)
|
||||
.bind("2026-01-15")
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(vec![team_id, "entity-sentinel"])
|
||||
.bind(team_id)
|
||||
.bind(api_key)
|
||||
.bind(endpoint)
|
||||
.bind("api-key-sentinel")
|
||||
.bind("endpoint-sentinel")
|
||||
.bind("entity-sentinel")
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 4);
|
||||
assert_eq!(rows[0].id, initial_id);
|
||||
assert_eq!(rows[0].team_id.as_deref(), Some(team_id));
|
||||
assert_eq!(rows[0].api_key, api_key);
|
||||
assert_eq!(rows[0].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[0].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
|
||||
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[0].prompt_tokens,
|
||||
rows[0].completion_tokens,
|
||||
rows[0].api_requests,
|
||||
rows[0].successful_requests,
|
||||
rows[0].failed_requests,
|
||||
rows[0].cache_read_input_tokens,
|
||||
rows[0].cache_creation_input_tokens,
|
||||
rows[0].total_response_time_ms,
|
||||
rows[0].timed_requests,
|
||||
rows[0].spend
|
||||
),
|
||||
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
|
||||
);
|
||||
assert_eq!(rows[0].date, "2026-01-15");
|
||||
assert_eq!(rows[0].created_at, initial_created_at);
|
||||
assert!(rows[0].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[0].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[1].id.is_empty());
|
||||
assert_eq!(rows[1].team_id.as_deref(), Some(team_id));
|
||||
assert_eq!(rows[1].api_key, api_key);
|
||||
assert_eq!(rows[1].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[1].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
|
||||
assert_eq!(
|
||||
rows[1].model_group.as_deref(),
|
||||
Some("sentinel-endpoint-group")
|
||||
);
|
||||
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[1].prompt_tokens,
|
||||
rows[1].completion_tokens,
|
||||
rows[1].api_requests,
|
||||
rows[1].successful_requests,
|
||||
rows[1].failed_requests,
|
||||
rows[1].cache_read_input_tokens,
|
||||
rows[1].cache_creation_input_tokens,
|
||||
rows[1].total_response_time_ms,
|
||||
rows[1].timed_requests,
|
||||
rows[1].spend
|
||||
),
|
||||
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
|
||||
);
|
||||
assert_eq!(rows[1].date, "2026-01-15");
|
||||
assert!(rows[1].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[1].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[2].id.is_empty());
|
||||
assert_eq!(rows[2].team_id.as_deref(), Some(team_id));
|
||||
assert_eq!(rows[2].api_key, "api-key-sentinel");
|
||||
assert_eq!(rows[2].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[2].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
|
||||
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[2].prompt_tokens,
|
||||
rows[2].completion_tokens,
|
||||
rows[2].api_requests,
|
||||
rows[2].successful_requests,
|
||||
rows[2].failed_requests,
|
||||
rows[2].cache_read_input_tokens,
|
||||
rows[2].cache_creation_input_tokens,
|
||||
rows[2].total_response_time_ms,
|
||||
rows[2].timed_requests,
|
||||
rows[2].spend
|
||||
),
|
||||
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
|
||||
);
|
||||
assert_eq!(rows[2].date, "2026-01-15");
|
||||
assert!(rows[2].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[2].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[3].id.is_empty());
|
||||
assert_eq!(rows[3].team_id.as_deref(), Some("entity-sentinel"));
|
||||
assert_eq!(rows[3].api_key, api_key);
|
||||
assert_eq!(rows[3].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[3].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(
|
||||
rows[3].model_group.as_deref(),
|
||||
Some("sentinel-entity-group")
|
||||
);
|
||||
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[3].prompt_tokens,
|
||||
rows[3].completion_tokens,
|
||||
rows[3].api_requests,
|
||||
rows[3].successful_requests,
|
||||
rows[3].failed_requests,
|
||||
rows[3].cache_read_input_tokens,
|
||||
rows[3].cache_creation_input_tokens,
|
||||
rows[3].total_response_time_ms,
|
||||
rows[3].timed_requests,
|
||||
rows[3].spend
|
||||
),
|
||||
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
|
||||
);
|
||||
assert_eq!(rows[3].date, "2026-01-15");
|
||||
assert!(rows[3].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[3].updated_at, expected_updated_at);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
500
litellm-rust/crates/db/src/spend/daily_user_spend.rs
Normal file
500
litellm-rust/crates/db/src/spend/daily_user_spend.rs
Normal file
|
|
@ -0,0 +1,500 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_input)]
|
||||
#[derive(Default)]
|
||||
pub struct DailySpend {
|
||||
pub entity_id: String,
|
||||
pub date: String,
|
||||
pub api_key: String,
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub mcp_namespaced_tool_name: String,
|
||||
pub endpoint: String,
|
||||
pub model_group: Option<String>,
|
||||
pub prompt_tokens: i64,
|
||||
pub completion_tokens: i64,
|
||||
pub api_requests: i64,
|
||||
pub successful_requests: i64,
|
||||
pub failed_requests: i64,
|
||||
pub cache_read_input_tokens: i64,
|
||||
pub cache_creation_input_tokens: i64,
|
||||
pub total_response_time_ms: i64,
|
||||
pub timed_requests: i64,
|
||||
pub spend: f64,
|
||||
}
|
||||
|
||||
pub async fn upsert_daily_user_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
rows: &[DailySpend],
|
||||
) -> Result<(), Error> {
|
||||
if rows.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
macro_rules! column {
|
||||
($field:ident) => {
|
||||
rows.iter()
|
||||
.map(|row| row.$field.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
}
|
||||
sqlx::query_file!(
|
||||
"queries/spend/upsert_daily_user_spend.sql",
|
||||
&column!(entity_id),
|
||||
&column!(date),
|
||||
&column!(api_key),
|
||||
&column!(model),
|
||||
&column!(custom_llm_provider),
|
||||
&column!(endpoint),
|
||||
&column!(model_group) as &[Option<String>],
|
||||
&column!(prompt_tokens),
|
||||
&column!(completion_tokens),
|
||||
&column!(api_requests),
|
||||
&column!(successful_requests),
|
||||
&column!(failed_requests),
|
||||
&column!(cache_read_input_tokens),
|
||||
&column!(cache_creation_input_tokens),
|
||||
&column!(total_response_time_ms),
|
||||
&column!(timed_requests),
|
||||
&column!(spend),
|
||||
&column!(mcp_namespaced_tool_name),
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{DailySpend, upsert_daily_user_spend};
|
||||
use crate::{DailyUserRow, test_support::database_pool};
|
||||
use rstest::rstest;
|
||||
use sqlx::{PgPool, types::chrono::NaiveDateTime};
|
||||
|
||||
#[rstest]
|
||||
#[case::distinct_tools("server/search", "server/write")]
|
||||
#[case::default_empty_tool("", "server/search")]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_user_spend_separates_tools_and_accumulates(
|
||||
#[case] first_tool: &str,
|
||||
#[case] second_tool: &str,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
let first = DailySpend {
|
||||
entity_id: format!("daily-user-tools-{first_tool}"),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: "tool-key".to_owned(),
|
||||
model: "tool-model".to_owned(),
|
||||
custom_llm_provider: "tool-provider".to_owned(),
|
||||
endpoint: "/mcp".to_owned(),
|
||||
model_group: Some("tool-group".to_owned()),
|
||||
mcp_namespaced_tool_name: first_tool.to_owned(),
|
||||
prompt_tokens: 3,
|
||||
api_requests: 1,
|
||||
spend: 0.25,
|
||||
..Default::default()
|
||||
};
|
||||
let second = DailySpend {
|
||||
mcp_namespaced_tool_name: second_tool.to_owned(),
|
||||
prompt_tokens: 7,
|
||||
api_requests: 2,
|
||||
spend: 0.5,
|
||||
..first.clone()
|
||||
};
|
||||
upsert_daily_user_spend(&mut *tx, &[second.clone(), first.clone()]).await?;
|
||||
upsert_daily_user_spend(&mut *tx, &[first.clone(), second]).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads stored rows to verify tool attribution"
|
||||
)]
|
||||
let rows: Vec<DailyUserRow> = sqlx::query_as(
|
||||
r#"SELECT * FROM "LiteLLM_DailyUserSpend"
|
||||
WHERE user_id = $1
|
||||
ORDER BY mcp_namespaced_tool_name, endpoint"#,
|
||||
)
|
||||
.bind(&first.entity_id)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 2);
|
||||
assert_eq!(
|
||||
rows[0].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(first_tool)
|
||||
);
|
||||
assert_eq!(
|
||||
rows[1].mcp_namespaced_tool_name.as_deref(),
|
||||
Some(second_tool)
|
||||
);
|
||||
assert_eq!(
|
||||
(rows[0].prompt_tokens, rows[0].api_requests, rows[0].spend),
|
||||
(6, 2, 0.5)
|
||||
);
|
||||
assert_eq!(
|
||||
(rows[1].prompt_tokens, rows[1].api_requests, rows[1].spend),
|
||||
(14, 4, 1.0)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::full_keys(
|
||||
"daily-user-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
Some("first-group")
|
||||
)]
|
||||
#[case::initial_model_group_none(
|
||||
"daily-user-base",
|
||||
"daily-key-base",
|
||||
"daily-model",
|
||||
"daily-provider",
|
||||
"/chat/completions",
|
||||
None
|
||||
)]
|
||||
#[case::empty_optional_keys("", "", "", "", "", Some(""))]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn daily_user_spend_upsert_preserves_keys_and_accumulates(
|
||||
#[case] user_id: &'static str,
|
||||
#[case] api_key: &'static str,
|
||||
#[case] model: &'static str,
|
||||
#[case] custom_llm_provider: &'static str,
|
||||
#[case] endpoint: &'static str,
|
||||
#[case] first_model_group: Option<&'static str>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test exercises UTC timestamp behavior under a non-UTC session"
|
||||
)]
|
||||
sqlx::query("SET LOCAL TIME ZONE 'America/Los_Angeles'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let first_batch = [
|
||||
DailySpend {
|
||||
entity_id: user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: first_model_group.map(str::to_owned),
|
||||
prompt_tokens: 1,
|
||||
completion_tokens: 2,
|
||||
api_requests: 3,
|
||||
successful_requests: 4,
|
||||
failed_requests: 5,
|
||||
cache_read_input_tokens: 6,
|
||||
cache_creation_input_tokens: 7,
|
||||
total_response_time_ms: 8,
|
||||
timed_requests: 9,
|
||||
spend: 0.125,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: "endpoint-sentinel".to_owned(),
|
||||
model_group: Some("sentinel-endpoint-group".to_owned()),
|
||||
prompt_tokens: 101,
|
||||
completion_tokens: 102,
|
||||
api_requests: 103,
|
||||
successful_requests: 104,
|
||||
failed_requests: 105,
|
||||
cache_read_input_tokens: 106,
|
||||
cache_creation_input_tokens: 107,
|
||||
total_response_time_ms: 108,
|
||||
timed_requests: 109,
|
||||
spend: 1.25,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: "api-key-sentinel".to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-key-group".to_owned()),
|
||||
prompt_tokens: 201,
|
||||
completion_tokens: 202,
|
||||
api_requests: 203,
|
||||
successful_requests: 204,
|
||||
failed_requests: 205,
|
||||
cache_read_input_tokens: 206,
|
||||
cache_creation_input_tokens: 207,
|
||||
total_response_time_ms: 208,
|
||||
timed_requests: 209,
|
||||
spend: 2.5,
|
||||
},
|
||||
DailySpend {
|
||||
entity_id: "entity-sentinel".to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("sentinel-entity-group".to_owned()),
|
||||
prompt_tokens: 301,
|
||||
completion_tokens: 302,
|
||||
api_requests: 303,
|
||||
successful_requests: 304,
|
||||
failed_requests: 305,
|
||||
cache_read_input_tokens: 306,
|
||||
cache_creation_input_tokens: 307,
|
||||
total_response_time_ms: 308,
|
||||
timed_requests: 309,
|
||||
spend: 3.75,
|
||||
},
|
||||
];
|
||||
upsert_daily_user_spend(&mut *tx, &first_batch).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the inserted row identity before conflict handling"
|
||||
)]
|
||||
let (initial_id, initial_created_at): (String, NaiveDateTime) = sqlx::query_as(
|
||||
r#"SELECT id, created_at
|
||||
FROM "LiteLLM_DailyUserSpend"
|
||||
WHERE user_id = $1 AND date = $2 AND api_key = $3 AND model = $4
|
||||
AND custom_llm_provider = $5 AND endpoint = $6"#,
|
||||
)
|
||||
.bind(user_id)
|
||||
.bind("2026-01-15")
|
||||
.bind(api_key)
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(endpoint)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert!(!initial_id.is_empty());
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test seeds updated_at before exercising the conflict update"
|
||||
)]
|
||||
let seeded_updated_at: NaiveDateTime = sqlx::query_scalar(
|
||||
r#"UPDATE "LiteLLM_DailyUserSpend"
|
||||
SET updated_at = TIMESTAMP '2000-01-01'
|
||||
WHERE id = $1
|
||||
RETURNING updated_at"#,
|
||||
)
|
||||
.bind(&initial_id)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
let increment = [DailySpend {
|
||||
entity_id: user_id.to_owned(),
|
||||
date: "2026-01-15".to_owned(),
|
||||
api_key: api_key.to_owned(),
|
||||
model: model.to_owned(),
|
||||
custom_llm_provider: custom_llm_provider.to_owned(),
|
||||
mcp_namespaced_tool_name: String::new(),
|
||||
endpoint: endpoint.to_owned(),
|
||||
model_group: Some("second-group".to_owned()),
|
||||
prompt_tokens: 10,
|
||||
completion_tokens: 20,
|
||||
api_requests: 30,
|
||||
successful_requests: 40,
|
||||
failed_requests: 50,
|
||||
cache_read_input_tokens: 60,
|
||||
cache_creation_input_tokens: 70,
|
||||
total_response_time_ms: 80,
|
||||
timed_requests: 90,
|
||||
spend: 0.625,
|
||||
}];
|
||||
upsert_daily_user_spend(&mut *tx, &increment).await?;
|
||||
upsert_daily_user_spend(&mut *tx, &[]).await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test captures the transaction timestamp for assertions"
|
||||
)]
|
||||
let expected_updated_at: NaiveDateTime =
|
||||
sqlx::query_scalar("SELECT (NOW() AT TIME ZONE 'UTC')::timestamp(3)")
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The integration test reads all rows to verify attribution and accumulation"
|
||||
)]
|
||||
let rows: Vec<DailyUserRow> = sqlx::query_as(
|
||||
r#"SELECT id, user_id, api_key, model, custom_llm_provider, endpoint,
|
||||
model_group, mcp_namespaced_tool_name, prompt_tokens,
|
||||
completion_tokens, api_requests, successful_requests,
|
||||
failed_requests, cache_read_input_tokens,
|
||||
cache_creation_input_tokens, total_response_time_ms,
|
||||
timed_requests, spend, date, created_at, updated_at
|
||||
FROM "LiteLLM_DailyUserSpend"
|
||||
WHERE date = $1 AND model = $2 AND custom_llm_provider = $3
|
||||
AND mcp_namespaced_tool_name = '' AND user_id = ANY($4)
|
||||
ORDER BY CASE
|
||||
WHEN user_id = $5 AND api_key = $6 AND endpoint = $7 THEN 0
|
||||
WHEN user_id = $5 AND api_key = $6 AND endpoint = $9 THEN 1
|
||||
WHEN user_id = $5 AND api_key = $8 AND endpoint = $7 THEN 2
|
||||
WHEN user_id = $10 AND api_key = $6 AND endpoint = $7 THEN 3
|
||||
ELSE 4
|
||||
END"#,
|
||||
)
|
||||
.bind("2026-01-15")
|
||||
.bind(model)
|
||||
.bind(custom_llm_provider)
|
||||
.bind(vec![user_id, "entity-sentinel"])
|
||||
.bind(user_id)
|
||||
.bind(api_key)
|
||||
.bind(endpoint)
|
||||
.bind("api-key-sentinel")
|
||||
.bind("endpoint-sentinel")
|
||||
.bind("entity-sentinel")
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
|
||||
assert_eq!(rows.len(), 4);
|
||||
assert_eq!(rows[0].id, initial_id);
|
||||
assert_eq!(rows[0].user_id.as_deref(), Some(user_id));
|
||||
assert_eq!(rows[0].api_key, api_key);
|
||||
assert_eq!(rows[0].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[0].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[0].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[0].model_group.as_deref(), first_model_group);
|
||||
assert_eq!(rows[0].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[0].prompt_tokens,
|
||||
rows[0].completion_tokens,
|
||||
rows[0].api_requests,
|
||||
rows[0].successful_requests,
|
||||
rows[0].failed_requests,
|
||||
rows[0].cache_read_input_tokens,
|
||||
rows[0].cache_creation_input_tokens,
|
||||
rows[0].total_response_time_ms,
|
||||
rows[0].timed_requests,
|
||||
rows[0].spend
|
||||
),
|
||||
(11, 22, 33, 44, 55, 66, 77, 88, 99, 0.75)
|
||||
);
|
||||
assert_eq!(rows[0].date, "2026-01-15");
|
||||
assert_eq!(rows[0].created_at, initial_created_at);
|
||||
assert!(rows[0].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[0].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[1].id.is_empty());
|
||||
assert_eq!(rows[1].user_id.as_deref(), Some(user_id));
|
||||
assert_eq!(rows[1].api_key, api_key);
|
||||
assert_eq!(rows[1].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[1].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[1].endpoint.as_deref(), Some("endpoint-sentinel"));
|
||||
assert_eq!(
|
||||
rows[1].model_group.as_deref(),
|
||||
Some("sentinel-endpoint-group")
|
||||
);
|
||||
assert_eq!(rows[1].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[1].prompt_tokens,
|
||||
rows[1].completion_tokens,
|
||||
rows[1].api_requests,
|
||||
rows[1].successful_requests,
|
||||
rows[1].failed_requests,
|
||||
rows[1].cache_read_input_tokens,
|
||||
rows[1].cache_creation_input_tokens,
|
||||
rows[1].total_response_time_ms,
|
||||
rows[1].timed_requests,
|
||||
rows[1].spend
|
||||
),
|
||||
(101, 102, 103, 104, 105, 106, 107, 108, 109, 1.25)
|
||||
);
|
||||
assert_eq!(rows[1].date, "2026-01-15");
|
||||
assert!(rows[1].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[1].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[2].id.is_empty());
|
||||
assert_eq!(rows[2].user_id.as_deref(), Some(user_id));
|
||||
assert_eq!(rows[2].api_key, "api-key-sentinel");
|
||||
assert_eq!(rows[2].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[2].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[2].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(rows[2].model_group.as_deref(), Some("sentinel-key-group"));
|
||||
assert_eq!(rows[2].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[2].prompt_tokens,
|
||||
rows[2].completion_tokens,
|
||||
rows[2].api_requests,
|
||||
rows[2].successful_requests,
|
||||
rows[2].failed_requests,
|
||||
rows[2].cache_read_input_tokens,
|
||||
rows[2].cache_creation_input_tokens,
|
||||
rows[2].total_response_time_ms,
|
||||
rows[2].timed_requests,
|
||||
rows[2].spend
|
||||
),
|
||||
(201, 202, 203, 204, 205, 206, 207, 208, 209, 2.5)
|
||||
);
|
||||
assert_eq!(rows[2].date, "2026-01-15");
|
||||
assert!(rows[2].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[2].updated_at, expected_updated_at);
|
||||
|
||||
assert!(!rows[3].id.is_empty());
|
||||
assert_eq!(rows[3].user_id.as_deref(), Some("entity-sentinel"));
|
||||
assert_eq!(rows[3].api_key, api_key);
|
||||
assert_eq!(rows[3].model.as_deref(), Some(model));
|
||||
assert_eq!(
|
||||
rows[3].custom_llm_provider.as_deref(),
|
||||
Some(custom_llm_provider)
|
||||
);
|
||||
assert_eq!(rows[3].endpoint.as_deref(), Some(endpoint));
|
||||
assert_eq!(
|
||||
rows[3].model_group.as_deref(),
|
||||
Some("sentinel-entity-group")
|
||||
);
|
||||
assert_eq!(rows[3].mcp_namespaced_tool_name.as_deref(), Some(""));
|
||||
assert_eq!(
|
||||
(
|
||||
rows[3].prompt_tokens,
|
||||
rows[3].completion_tokens,
|
||||
rows[3].api_requests,
|
||||
rows[3].successful_requests,
|
||||
rows[3].failed_requests,
|
||||
rows[3].cache_read_input_tokens,
|
||||
rows[3].cache_creation_input_tokens,
|
||||
rows[3].total_response_time_ms,
|
||||
rows[3].timed_requests,
|
||||
rows[3].spend
|
||||
),
|
||||
(301, 302, 303, 304, 305, 306, 307, 308, 309, 3.75)
|
||||
);
|
||||
assert_eq!(rows[3].date, "2026-01-15");
|
||||
assert!(rows[3].updated_at > seeded_updated_at);
|
||||
assert_eq!(rows[3].updated_at, expected_updated_at);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
197
litellm-rust/crates/db/src/spend/end_user_spend.rs
Normal file
197
litellm-rust/crates/db/src/spend/end_user_spend.rs
Normal file
|
|
@ -0,0 +1,197 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use super::EntitySpend;
|
||||
use crate::Error;
|
||||
|
||||
pub async fn add_end_user_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
spend: &EntitySpend,
|
||||
) -> Result<(), Error> {
|
||||
spend.validate()?;
|
||||
if spend.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
sqlx::query_file!(
|
||||
"queries/spend/add_end_user_spend.sql",
|
||||
&spend.ids,
|
||||
&spend.costs
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{EntitySpend, add_end_user_spend};
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[rstest]
|
||||
#[case::fractional_costs(0.25, 0.75, false, 2.5, 1.5)]
|
||||
#[case::zero_costs(0.0, 0.0, false, 2.0, 0.0)]
|
||||
#[case::duplicate_ids(0.25, 0.75, true, 3.0, 3.0)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn end_user_spend_matches_python_upsert(
|
||||
#[case] existing_cost: f64,
|
||||
#[case] new_cost: f64,
|
||||
#[case] duplicate_ids: bool,
|
||||
#[case] expected_existing_spend: f64,
|
||||
#[case] expected_new_spend: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _budget = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_BudgetTable" (budget_id, created_by, updated_by)
|
||||
VALUES ('end-user-spend-budget', 'db-query-test', 'db-query-test')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _users = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked, budget_id)
|
||||
VALUES ('end-user-spend-z', 2, true, 'end-user-spend-budget'),
|
||||
('end-user-spend-untouched', 100, true, NULL)"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = if duplicate_ids {
|
||||
EntitySpend {
|
||||
ids: vec![
|
||||
"end-user-spend-z".into(),
|
||||
"end-user-spend-a".into(),
|
||||
"end-user-spend-z".into(),
|
||||
"end-user-spend-a".into(),
|
||||
],
|
||||
costs: vec![existing_cost, new_cost, existing_cost, new_cost],
|
||||
}
|
||||
} else {
|
||||
EntitySpend {
|
||||
ids: vec!["end-user-spend-z".into(), "end-user-spend-a".into()],
|
||||
costs: vec![existing_cost, new_cost],
|
||||
}
|
||||
};
|
||||
add_end_user_spend(&mut *tx, &spend).await?;
|
||||
add_end_user_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64, bool, Option<String>)> = sqlx::query_as(
|
||||
r#"SELECT user_id, spend, blocked, budget_id FROM "LiteLLM_EndUserTable"
|
||||
WHERE user_id = ANY($1) ORDER BY user_id"#,
|
||||
)
|
||||
.bind(vec![
|
||||
"end-user-spend-a",
|
||||
"end-user-spend-z",
|
||||
"end-user-spend-untouched",
|
||||
])
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("end-user-spend-a".into(), expected_new_spend, false, None),
|
||||
("end-user-spend-untouched".into(), 100.0, true, None),
|
||||
(
|
||||
"end-user-spend-z".into(),
|
||||
expected_existing_spend,
|
||||
true,
|
||||
Some("end-user-spend-budget".into()),
|
||||
),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_cost(vec!["end-user-invalid-existing".into(), "end-user-invalid-new".into()], vec![1.0])]
|
||||
#[case::extra_cost(vec!["end-user-invalid-existing".into()], vec![1.0, 2.0])]
|
||||
#[case::empty_ids(vec![], vec![1.0])]
|
||||
#[case::empty_costs(vec!["end-user-invalid-existing".into()], vec![])]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn malformed_end_user_spend_is_rejected_before_writes(
|
||||
#[case] ids: Vec<String>,
|
||||
#[case] costs: Vec<f64>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _user = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
|
||||
VALUES ('end-user-invalid-existing', 8.5, true)"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend { ids, costs };
|
||||
let result = add_end_user_spend(&mut *tx, &spend).await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(crate::Error::InvalidSpendBatch { ids, costs })
|
||||
if ids == spend.ids.len() && costs == spend.costs.len()
|
||||
));
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
|
||||
r#"SELECT user_id, spend, blocked FROM "LiteLLM_EndUserTable"
|
||||
WHERE user_id IN ('end-user-invalid-existing', 'end-user-invalid-new')"#,
|
||||
)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(rows, vec![("end-user-invalid-existing".into(), 8.5, true)]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty_batch()]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn empty_end_user_spend_is_a_noop(
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _user = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_EndUserTable" (user_id, spend, blocked)
|
||||
VALUES ('end-user-empty-spend', 8.5, true)"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
add_end_user_spend(&mut *tx, &EntitySpend::default()).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (f64, bool, Option<String>) = sqlx::query_as(
|
||||
r#"SELECT spend, blocked, budget_id FROM "LiteLLM_EndUserTable"
|
||||
WHERE user_id = 'end-user-empty-spend'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(row, (8.5, true, None));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
309
litellm-rust/crates/db/src/spend/key_spend.rs
Normal file
309
litellm-rust/crates/db/src/spend/key_spend.rs
Normal file
|
|
@ -0,0 +1,309 @@
|
|||
use sqlx::PgExecutor;
|
||||
use std::collections::BTreeMap;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_input)]
|
||||
#[derive(Default)]
|
||||
pub struct EntitySpend {
|
||||
pub ids: Vec<String>,
|
||||
pub costs: Vec<f64>,
|
||||
}
|
||||
|
||||
impl EntitySpend {
|
||||
pub(crate) fn validate(&self) -> Result<(), Error> {
|
||||
if self.ids.len() != self.costs.len() {
|
||||
return Err(Error::InvalidSpendBatch {
|
||||
ids: self.ids.len(),
|
||||
costs: self.costs.len(),
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.ids.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
impl FromIterator<(String, f64)> for EntitySpend {
|
||||
fn from_iter<I: IntoIterator<Item = (String, f64)>>(entries: I) -> Self {
|
||||
let entries = entries
|
||||
.into_iter()
|
||||
.fold(BTreeMap::new(), |mut sums, (id, cost)| {
|
||||
*sums.entry(id).or_insert(0.0) += cost;
|
||||
sums
|
||||
});
|
||||
let (ids, costs) = entries.into_iter().unzip();
|
||||
Self { ids, costs }
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn add_key_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
spend: &EntitySpend,
|
||||
) -> Result<(), Error> {
|
||||
spend.validate()?;
|
||||
if spend.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
sqlx::query_file!("queries/spend/add_key_spend.sql", &spend.ids, &spend.costs)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{EntitySpend, add_key_spend};
|
||||
use crate::{Error, test_support::database_pool};
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[rstest]
|
||||
#[case::direct(false)]
|
||||
#[case::collected(true)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn duplicate_key_spend_accumulates(
|
||||
#[case] collected: bool,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at)
|
||||
VALUES ('key-spend-duplicate', 2, 7, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = if collected {
|
||||
[
|
||||
("key-spend-duplicate".into(), 0.25),
|
||||
("key-spend-duplicate".into(), 0.75),
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
} else {
|
||||
EntitySpend {
|
||||
ids: vec!["key-spend-duplicate".into(), "key-spend-duplicate".into()],
|
||||
costs: vec![0.25, 0.75],
|
||||
}
|
||||
};
|
||||
add_key_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (f64, f64) = sqlx::query_as(
|
||||
r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken"
|
||||
WHERE token = 'key-spend-duplicate'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(row, (3.0, 8.0));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::extra_id(vec!["key-spend-invalid".into(), "key-spend-invalid".into()], vec![0.25])]
|
||||
#[case::extra_cost(vec!["key-spend-invalid".into()], vec![0.25, 0.75])]
|
||||
#[case::empty_ids(vec![], vec![0.25])]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn malformed_key_spend_leaves_transaction_usable(
|
||||
#[case] ids: Vec<String>,
|
||||
#[case] costs: Vec<f64>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_VerificationToken"
|
||||
(token, spend, total_spend, last_active, updated_at)
|
||||
VALUES ('key-spend-invalid', 2, 7, NULL, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend { ids, costs };
|
||||
let result = add_key_spend(&mut *tx, &spend).await;
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(Error::InvalidSpendBatch { ids, costs })
|
||||
if ids == spend.ids.len() && costs == spend.costs.len()
|
||||
));
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let unchanged: (f64, f64, bool, bool) = sqlx::query_as(
|
||||
r#"SELECT spend, total_spend, last_active IS NULL,
|
||||
updated_at = TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_VerificationToken" WHERE token = 'key-spend-invalid'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(unchanged, (2.0, 7.0, true, true));
|
||||
let valid = EntitySpend {
|
||||
ids: vec!["key-spend-invalid".into()],
|
||||
costs: vec![0.5],
|
||||
};
|
||||
add_key_spend(&mut *tx, &valid).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let updated: (f64, f64) = sqlx::query_as(
|
||||
r#"SELECT spend, total_spend FROM "LiteLLM_VerificationToken"
|
||||
WHERE token = 'key-spend-invalid'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(updated, (2.5, 7.5));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn key_activity_uses_update_time_in_old_transaction(
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_VerificationToken" (token, spend, total_spend, updated_at)
|
||||
VALUES ('key-spend-old-transaction', 2, 7, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Transaction timing setup is outside the compile-checked query set"
|
||||
)]
|
||||
sqlx::query("SELECT pg_sleep(0.02)")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let boundary: chrono::NaiveDateTime = sqlx::query_scalar(
|
||||
"SELECT date_trunc('milliseconds', clock_timestamp() AT TIME ZONE 'UTC')",
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec!["key-spend-old-transaction".into()],
|
||||
costs: vec![0.25],
|
||||
};
|
||||
add_key_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (chrono::NaiveDateTime, chrono::NaiveDateTime) = sqlx::query_as(
|
||||
r#"SELECT last_active, updated_at FROM "LiteLLM_VerificationToken"
|
||||
WHERE token = 'key-spend-old-transaction'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert!(row.0 >= boundary, "last_active predates update boundary");
|
||||
assert!(row.1 >= boundary, "updated_at predates update boundary");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::fractional_costs(0.25, 2.5, 7.5, 4.0, 12.0)]
|
||||
#[case::zero_costs(0.0, 2.0, 7.0, 3.0, 11.0)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn key_spend_matches_python_update_many(
|
||||
#[case] cost: f64,
|
||||
#[case] expected_a_spend: f64,
|
||||
#[case] expected_a_total: f64,
|
||||
#[case] expected_z_spend: f64,
|
||||
#[case] expected_z_total: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_VerificationToken"
|
||||
(token, spend, total_spend, last_active, updated_at)
|
||||
VALUES ('key-spend-a', 2, 7, NULL, '2000-01-01'),
|
||||
('key-spend-z', 3, 11, NULL, '2000-01-01'),
|
||||
('key-spend-untouched', 100, 200, NULL, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec![
|
||||
"key-spend-z".into(),
|
||||
"key-spend-a".into(),
|
||||
"key-spend-missing".into(),
|
||||
],
|
||||
costs: vec![2.0 * cost, cost, 50.0],
|
||||
};
|
||||
add_key_spend(&mut *tx, &EntitySpend::default()).await?;
|
||||
add_key_spend(&mut *tx, &spend).await?;
|
||||
add_key_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64, f64, bool, bool)> = sqlx::query_as(
|
||||
r#"SELECT token, spend, total_spend, last_active IS NOT NULL,
|
||||
updated_at > TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_VerificationToken"
|
||||
WHERE token = ANY($1) ORDER BY token"#,
|
||||
)
|
||||
.bind(vec![
|
||||
"key-spend-a",
|
||||
"key-spend-z",
|
||||
"key-spend-missing",
|
||||
"key-spend-untouched",
|
||||
])
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
(
|
||||
"key-spend-a".into(),
|
||||
expected_a_spend,
|
||||
expected_a_total,
|
||||
true,
|
||||
true
|
||||
),
|
||||
("key-spend-untouched".into(), 100.0, 200.0, false, false),
|
||||
(
|
||||
"key-spend-z".into(),
|
||||
expected_z_spend,
|
||||
expected_z_total,
|
||||
true,
|
||||
true
|
||||
),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
21
litellm-rust/crates/db/src/spend/mod.rs
Normal file
21
litellm-rust/crates/db/src/spend/mod.rs
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
mod daily_end_user_spend;
|
||||
mod daily_team_spend;
|
||||
mod daily_user_spend;
|
||||
mod end_user_spend;
|
||||
mod key_spend;
|
||||
mod spend_logs;
|
||||
mod team_lock;
|
||||
mod team_member_spend;
|
||||
mod team_spend;
|
||||
mod user_spend;
|
||||
|
||||
pub use daily_end_user_spend::{DailyTable, upsert_daily_end_user_spend, upsert_daily_spend};
|
||||
pub use daily_team_spend::upsert_daily_team_spend;
|
||||
pub use daily_user_spend::{DailySpend, upsert_daily_user_spend};
|
||||
pub use end_user_spend::add_end_user_spend;
|
||||
pub use key_spend::{EntitySpend, add_key_spend};
|
||||
pub use spend_logs::{SpendLog, SpendLogRow, insert_spend_logs};
|
||||
pub use team_lock::{begin, lock_team};
|
||||
pub use team_member_spend::{TeamMemberSpend, add_team_member_spend};
|
||||
pub use team_spend::add_team_spend;
|
||||
pub use user_spend::add_user_spend;
|
||||
517
litellm-rust/crates/db/src/spend/spend_logs.rs
Normal file
517
litellm-rust/crates/db/src/spend/spend_logs.rs
Normal file
|
|
@ -0,0 +1,517 @@
|
|||
use chrono::{DateTime, NaiveDateTime};
|
||||
use sqlx::PgExecutor;
|
||||
|
||||
use crate::Error;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_input)]
|
||||
#[derive(Default)]
|
||||
pub struct SpendLog {
|
||||
pub request_id: String,
|
||||
pub call_type: String,
|
||||
pub api_key: String,
|
||||
pub spend: f64,
|
||||
pub total_tokens: i32,
|
||||
pub prompt_tokens: i32,
|
||||
pub completion_tokens: i32,
|
||||
pub start_time: String,
|
||||
pub end_time: String,
|
||||
pub completion_start_time: Option<String>,
|
||||
pub request_duration_ms: Option<i32>,
|
||||
pub model: String,
|
||||
pub model_id: String,
|
||||
pub model_group: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub api_base: String,
|
||||
pub user: String,
|
||||
pub metadata: serde_json::Value,
|
||||
pub cache_hit: String,
|
||||
pub cache_key: String,
|
||||
pub team_id: String,
|
||||
pub organization_id: String,
|
||||
pub end_user: String,
|
||||
pub requester_ip_address: Option<String>,
|
||||
pub session_id: Option<String>,
|
||||
pub status: String,
|
||||
pub litellm_call_id: Option<String>,
|
||||
}
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_row)]
|
||||
pub struct SpendLogRow {
|
||||
pub request_id: String,
|
||||
pub call_type: String,
|
||||
pub api_key: String,
|
||||
pub spend: f64,
|
||||
pub total_tokens: i32,
|
||||
pub prompt_tokens: i32,
|
||||
pub completion_tokens: i32,
|
||||
#[sqlx(rename = "startTime")]
|
||||
pub start_time: NaiveDateTime,
|
||||
#[sqlx(rename = "endTime")]
|
||||
pub end_time: NaiveDateTime,
|
||||
#[sqlx(rename = "completionStartTime")]
|
||||
pub completion_start_time: Option<NaiveDateTime>,
|
||||
pub request_duration_ms: Option<i32>,
|
||||
pub model: String,
|
||||
pub model_id: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub api_base: Option<String>,
|
||||
pub user: Option<String>,
|
||||
pub metadata: Option<serde_json::Value>,
|
||||
pub cache_hit: Option<String>,
|
||||
pub cache_key: Option<String>,
|
||||
pub request_tags: Option<serde_json::Value>,
|
||||
pub team_id: Option<String>,
|
||||
pub organization_id: Option<String>,
|
||||
pub end_user: Option<String>,
|
||||
pub requester_ip_address: Option<String>,
|
||||
pub messages: Option<serde_json::Value>,
|
||||
pub response: Option<serde_json::Value>,
|
||||
pub proxy_server_request: Option<serde_json::Value>,
|
||||
pub session_id: Option<String>,
|
||||
pub status: Option<String>,
|
||||
pub litellm_call_id: Option<String>,
|
||||
}
|
||||
|
||||
fn parse_timestamp(value: &str) -> Result<NaiveDateTime, chrono::ParseError> {
|
||||
DateTime::parse_from_rfc3339(value)
|
||||
.map(|timestamp| timestamp.naive_utc())
|
||||
.or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%dT%H:%M:%S%.f"))
|
||||
.or_else(|_| NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S%.f"))
|
||||
}
|
||||
|
||||
pub async fn insert_spend_logs(
|
||||
executor: impl PgExecutor<'_>,
|
||||
logs: &[SpendLog],
|
||||
) -> Result<(), Error> {
|
||||
if logs.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let start_times = logs
|
||||
.iter()
|
||||
.map(|log| parse_timestamp(&log.start_time))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let end_times = logs
|
||||
.iter()
|
||||
.map(|log| parse_timestamp(&log.end_time))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let completion_start_times = logs
|
||||
.iter()
|
||||
.map(|log| {
|
||||
log.completion_start_time
|
||||
.as_deref()
|
||||
.map(parse_timestamp)
|
||||
.transpose()
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
macro_rules! column {
|
||||
($field:ident) => {
|
||||
logs.iter()
|
||||
.map(|log| log.$field.clone())
|
||||
.collect::<Vec<_>>()
|
||||
};
|
||||
}
|
||||
sqlx::query_file!(
|
||||
"queries/spend/insert_spend_logs.sql",
|
||||
&column!(request_id),
|
||||
&column!(call_type),
|
||||
&column!(api_key),
|
||||
&column!(spend),
|
||||
&column!(total_tokens),
|
||||
&column!(prompt_tokens),
|
||||
&column!(completion_tokens),
|
||||
&start_times,
|
||||
&end_times,
|
||||
&completion_start_times as &[Option<NaiveDateTime>],
|
||||
&column!(request_duration_ms) as &[Option<i32>],
|
||||
&column!(model),
|
||||
&column!(model_id),
|
||||
&column!(model_group),
|
||||
&column!(custom_llm_provider),
|
||||
&column!(api_base),
|
||||
&column!(user),
|
||||
&column!(metadata),
|
||||
&column!(cache_hit),
|
||||
&column!(cache_key),
|
||||
&column!(team_id),
|
||||
&column!(organization_id),
|
||||
&column!(end_user),
|
||||
&column!(requester_ip_address) as &[Option<String>],
|
||||
&column!(session_id) as &[Option<String>],
|
||||
&column!(status),
|
||||
&column!(litellm_call_id) as &[Option<String>],
|
||||
)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{SpendLog, SpendLogRow, insert_spend_logs};
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::json;
|
||||
use sqlx::{Acquire, Connection, PgExecutor, PgPool, types::chrono::NaiveDateTime};
|
||||
|
||||
#[fixture]
|
||||
fn logs() -> [SpendLog; 2] {
|
||||
[
|
||||
SpendLog {
|
||||
request_id: "spend-logs-a".into(),
|
||||
call_type: "acompletion".into(),
|
||||
api_key: "key-a".into(),
|
||||
spend: 0.125,
|
||||
total_tokens: 17,
|
||||
prompt_tokens: 11,
|
||||
completion_tokens: 6,
|
||||
start_time: "2026-01-02 03:04:05.123".into(),
|
||||
end_time: "2026-01-02 03:04:06.789".into(),
|
||||
completion_start_time: Some("2026-01-02 03:04:05.456".into()),
|
||||
request_duration_ms: Some(1666),
|
||||
model: "model-a".into(),
|
||||
model_id: "deployment-a".into(),
|
||||
model_group: "group-a".into(),
|
||||
custom_llm_provider: "provider-a".into(),
|
||||
api_base: "https://a.example.test".into(),
|
||||
user: "user-a".into(),
|
||||
metadata: json!({"tenant": "a", "nested": {"tokens": 11}, "flag": false}),
|
||||
cache_hit: "False".into(),
|
||||
cache_key: "cache-a".into(),
|
||||
team_id: "team-a".into(),
|
||||
organization_id: "org-a".into(),
|
||||
end_user: "end-user-a".into(),
|
||||
requester_ip_address: Some("192.0.2.1".into()),
|
||||
session_id: Some("session-a".into()),
|
||||
status: "success".into(),
|
||||
litellm_call_id: Some("call-a".into()),
|
||||
},
|
||||
SpendLog {
|
||||
request_id: "spend-logs-b".into(),
|
||||
call_type: "aembedding".into(),
|
||||
api_key: "key-b".into(),
|
||||
spend: 2.75,
|
||||
total_tokens: 43,
|
||||
prompt_tokens: 29,
|
||||
completion_tokens: 14,
|
||||
start_time: "2026-02-03 14:15:16.234".into(),
|
||||
end_time: "2026-02-03 14:15:18.567".into(),
|
||||
completion_start_time: None,
|
||||
request_duration_ms: None,
|
||||
model: "model-b".into(),
|
||||
model_id: "deployment-b".into(),
|
||||
model_group: "group-b".into(),
|
||||
custom_llm_provider: "provider-b".into(),
|
||||
api_base: "https://b.example.test".into(),
|
||||
user: "user-b".into(),
|
||||
metadata: json!({"tenant": "b", "tags": ["distinct", 7], "nullable": null}),
|
||||
cache_hit: "True".into(),
|
||||
cache_key: "cache-b".into(),
|
||||
team_id: "team-b".into(),
|
||||
organization_id: "org-b".into(),
|
||||
end_user: "end-user-b".into(),
|
||||
requester_ip_address: None,
|
||||
session_id: None,
|
||||
status: "failure".into(),
|
||||
litellm_call_id: None,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
async fn read_log(
|
||||
executor: impl PgExecutor<'_>,
|
||||
request_id: &str,
|
||||
) -> Result<SpendLogRow, sqlx::Error> {
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
sqlx::query_as::<_, SpendLogRow>(
|
||||
r#"SELECT * FROM "LiteLLM_SpendLogs" WHERE request_id = $1"#,
|
||||
)
|
||||
.bind(request_id)
|
||||
.fetch_one(executor)
|
||||
.await
|
||||
}
|
||||
|
||||
fn expected_log(log: &SpendLog) -> Result<SpendLogRow, chrono::ParseError> {
|
||||
Ok(SpendLogRow {
|
||||
request_id: log.request_id.clone(),
|
||||
call_type: log.call_type.clone(),
|
||||
api_key: log.api_key.clone(),
|
||||
spend: log.spend,
|
||||
total_tokens: log.total_tokens,
|
||||
prompt_tokens: log.prompt_tokens,
|
||||
completion_tokens: log.completion_tokens,
|
||||
start_time: log.start_time.replace(' ', "T").parse()?,
|
||||
end_time: log.end_time.replace(' ', "T").parse()?,
|
||||
completion_start_time: log
|
||||
.completion_start_time
|
||||
.as_ref()
|
||||
.map(|value| value.replace(' ', "T").parse())
|
||||
.transpose()?,
|
||||
request_duration_ms: log.request_duration_ms,
|
||||
model: log.model.clone(),
|
||||
model_id: Some(log.model_id.clone()),
|
||||
model_group: Some(log.model_group.clone()),
|
||||
custom_llm_provider: Some(log.custom_llm_provider.clone()),
|
||||
api_base: Some(log.api_base.clone()),
|
||||
user: Some(log.user.clone()),
|
||||
metadata: Some(log.metadata.clone()),
|
||||
cache_hit: Some(log.cache_hit.clone()),
|
||||
cache_key: Some(log.cache_key.clone()),
|
||||
request_tags: Some(json!([])),
|
||||
team_id: Some(log.team_id.clone()),
|
||||
organization_id: Some(log.organization_id.clone()),
|
||||
end_user: Some(log.end_user.clone()),
|
||||
requester_ip_address: log.requester_ip_address.clone(),
|
||||
messages: Some(json!({})),
|
||||
response: Some(json!({})),
|
||||
proxy_server_request: Some(json!({})),
|
||||
session_id: log.session_id.clone(),
|
||||
status: Some(log.status.clone()),
|
||||
litellm_call_id: log.litellm_call_id.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::all_columns_and_duplicate()]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn inserts_distinct_rows_without_replacing_duplicates(
|
||||
logs: [SpendLog; 2],
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
insert_spend_logs(&mut *tx, &logs).await?;
|
||||
let replacement = SpendLog {
|
||||
request_id: logs[0].request_id.clone(),
|
||||
..logs[1].clone()
|
||||
};
|
||||
insert_spend_logs(&mut *tx, &[replacement]).await?;
|
||||
assert_eq!(
|
||||
read_log(&mut *tx, &logs[0].request_id).await?,
|
||||
expected_log(&logs[0])?
|
||||
);
|
||||
assert_eq!(
|
||||
read_log(&mut *tx, &logs[1].request_id).await?,
|
||||
expected_log(&logs[1])?
|
||||
);
|
||||
tx.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::partitioned_duplicate()]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn inserts_into_partitioned_logs_without_replacing_duplicates(
|
||||
logs: [SpendLog; 2],
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The test creates a temporary partitioned table at runtime"
|
||||
)]
|
||||
sqlx::raw_sql(
|
||||
r#"CREATE TEMP TABLE "LiteLLM_SpendLogs" (
|
||||
LIKE public."LiteLLM_SpendLogs" INCLUDING DEFAULTS INCLUDING GENERATED,
|
||||
PRIMARY KEY (request_id, "startTime")
|
||||
) PARTITION BY RANGE ("startTime");
|
||||
CREATE TEMP TABLE spend_logs_default
|
||||
PARTITION OF pg_temp."LiteLLM_SpendLogs" DEFAULT;
|
||||
SET LOCAL search_path TO pg_temp, public;"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
tx.clear_cached_statements().await?;
|
||||
insert_spend_logs(&mut *tx, &logs).await?;
|
||||
let replacement = SpendLog {
|
||||
spend: 99.0,
|
||||
..logs[0].clone()
|
||||
};
|
||||
insert_spend_logs(&mut *tx, &[replacement]).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The partitioned test fixture is created at runtime"
|
||||
)]
|
||||
let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(count.0, 2);
|
||||
assert_eq!(
|
||||
read_log(&mut *tx, &logs[0].request_id).await?,
|
||||
expected_log(&logs[0])?
|
||||
);
|
||||
assert_eq!(
|
||||
read_log(&mut *tx, &logs[1].request_id).await?,
|
||||
expected_log(&logs[1])?
|
||||
);
|
||||
let distinct_partition_key = SpendLog {
|
||||
request_id: logs[0].request_id.clone(),
|
||||
..logs[1].clone()
|
||||
};
|
||||
insert_spend_logs(&mut *tx, &[distinct_partition_key]).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The partitioned test fixture is created at runtime"
|
||||
)]
|
||||
let count: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM pg_temp."LiteLLM_SpendLogs""#)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(count.0, 3);
|
||||
tx.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::positive_offset_across_midnight(
|
||||
["2026-01-02T00:04:05.1236+02:00", "2026-01-02T00:04:06.7894+02:00", "2026-01-02T00:04:05.4567+02:00"],
|
||||
["2026-01-01T22:04:05.124", "2026-01-01T22:04:06.789", "2026-01-01T22:04:05.457"]
|
||||
)]
|
||||
#[case::negative_offset_across_midnight(
|
||||
["2026-01-01T23:04:05-02:00", "2026-01-01T23:04:06-02:00", "2026-01-01T23:04:05.456-02:00"],
|
||||
["2026-01-02T01:04:05", "2026-01-02T01:04:06", "2026-01-02T01:04:05.456"]
|
||||
)]
|
||||
#[case::utc(
|
||||
["2026-01-02T03:04:05Z", "2026-01-02T03:04:06Z", "2026-01-02T03:04:05.456Z"],
|
||||
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
|
||||
)]
|
||||
#[case::naive_iso(
|
||||
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"],
|
||||
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
|
||||
)]
|
||||
#[case::naive_space(
|
||||
["2026-01-02 03:04:05", "2026-01-02 03:04:06", "2026-01-02 03:04:05.456"],
|
||||
["2026-01-02T03:04:05", "2026-01-02T03:04:06", "2026-01-02T03:04:05.456"]
|
||||
)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn stores_utc_timestamps(
|
||||
#[case] input: [&str; 3],
|
||||
#[case] expected: [&str; 3],
|
||||
logs: [SpendLog; 2],
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "The test sets a transaction-local timezone to verify timestamp handling"
|
||||
)]
|
||||
sqlx::raw_sql("SET LOCAL TIME ZONE 'America/Los_Angeles'")
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let log = SpendLog {
|
||||
start_time: input[0].into(),
|
||||
end_time: input[1].into(),
|
||||
completion_start_time: Some(input[2].into()),
|
||||
..logs[0].clone()
|
||||
};
|
||||
insert_spend_logs(&mut *tx, std::slice::from_ref(&log)).await?;
|
||||
let row = read_log(&mut *tx, &log.request_id).await?;
|
||||
assert_eq!(row.start_time, expected[0].parse::<NaiveDateTime>()?);
|
||||
assert_eq!(row.end_time, expected[1].parse::<NaiveDateTime>()?);
|
||||
assert_eq!(
|
||||
row.completion_start_time,
|
||||
Some(expected[2].parse::<NaiveDateTime>()?)
|
||||
);
|
||||
tx.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::empty_batch()]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn empty_input_leaves_rows_unchanged(
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let before: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
insert_spend_logs(&mut *tx, &[]).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let after: (i64,) = sqlx::query_as(r#"SELECT count(*) FROM "LiteLLM_SpendLogs""#)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(before, after);
|
||||
tx.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::invalid_timestamp("not-a-timestamp", "2026-01-02T03:04:06", None)]
|
||||
#[case::invalid_end("2026-01-02T03:04:05", "not-a-timestamp", None)]
|
||||
#[case::invalid_completion(
|
||||
"2026-01-02T03:04:05",
|
||||
"2026-01-02T03:04:06",
|
||||
Some("not-a-timestamp")
|
||||
)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn invalid_timestamp_allows_caller_rollback(
|
||||
#[case] start_time: &str,
|
||||
#[case] end_time: &str,
|
||||
#[case] completion_start_time: Option<&str>,
|
||||
logs: [SpendLog; 2],
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut outer = pool.begin().await?;
|
||||
let mut tx = outer.begin().await?;
|
||||
let valid = SpendLog {
|
||||
request_id: "spend-logs-rollback-valid".into(),
|
||||
..logs[0].clone()
|
||||
};
|
||||
let invalid = SpendLog {
|
||||
request_id: "spend-logs-rollback-invalid".into(),
|
||||
start_time: start_time.into(),
|
||||
end_time: end_time.into(),
|
||||
completion_start_time: completion_start_time.map(str::to_owned),
|
||||
..logs[1].clone()
|
||||
};
|
||||
insert_spend_logs(&mut *tx, std::slice::from_ref(&valid)).await?;
|
||||
let result = insert_spend_logs(&mut *tx, &[logs[0].clone(), invalid]).await;
|
||||
assert!(matches!(result, Err(crate::Error::InvalidTimestamp(_))));
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let partial_batch: (i64,) = sqlx::query_as(
|
||||
r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#,
|
||||
)
|
||||
.bind([logs[0].request_id.as_str(), "spend-logs-rollback-invalid"].as_slice())
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(partial_batch.0, 0);
|
||||
tx.rollback().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let remaining: (i64,) = sqlx::query_as(
|
||||
r#"SELECT count(*) FROM "LiteLLM_SpendLogs" WHERE request_id = ANY($1)"#,
|
||||
)
|
||||
.bind([valid.request_id.as_str(), "spend-logs-rollback-invalid"].as_slice())
|
||||
.fetch_one(&mut *outer)
|
||||
.await?;
|
||||
assert_eq!(remaining.0, 0);
|
||||
outer.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
98
litellm-rust/crates/db/src/spend/team_lock.rs
Normal file
98
litellm-rust/crates/db/src/spend/team_lock.rs
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
use sqlx::{PgPool, Postgres, Transaction};
|
||||
|
||||
use crate::Error;
|
||||
|
||||
pub async fn begin(pool: &PgPool) -> Result<Transaction<'static, Postgres>, Error> {
|
||||
let mut transaction = pool.begin().await?;
|
||||
sqlx::query!("SELECT set_config('lock_timeout', '5000ms', true)")
|
||||
.fetch_one(&mut *transaction)
|
||||
.await?;
|
||||
Ok(transaction)
|
||||
}
|
||||
|
||||
pub async fn lock_team(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
team_id: &str,
|
||||
) -> Result<(), Error> {
|
||||
sqlx::query_file!("queries/spend/lock_team.sql", team_id)
|
||||
.fetch_one(&mut **transaction)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{begin, lock_team};
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::rstest;
|
||||
use sqlx::{PgPool, postgres::PgPoolOptions};
|
||||
|
||||
#[rstest]
|
||||
#[case::transaction_scoped("team-lock-scope")]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn team_lock_is_held_until_rollback(
|
||||
#[case] team_id: &str,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let database_pool = database_pool?;
|
||||
let pool = PgPoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect_with(database_pool.connect_options().as_ref().clone())
|
||||
.await?;
|
||||
let mut first = begin(&pool).await?;
|
||||
let mut second = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let original_timeout: String = sqlx::query_scalar("SELECT current_setting('lock_timeout')")
|
||||
.fetch_one(&mut *second)
|
||||
.await?;
|
||||
lock_team(&mut first, team_id).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let (backend, timeout): (i32, String) =
|
||||
sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')")
|
||||
.fetch_one(&mut *first)
|
||||
.await?;
|
||||
assert_eq!(timeout, "5s");
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let held: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))")
|
||||
.bind(team_id)
|
||||
.fetch_one(&mut *second)
|
||||
.await?;
|
||||
assert!(!held);
|
||||
first.rollback().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let released: bool = sqlx::query_scalar("SELECT pg_try_advisory_xact_lock(hashtext($1))")
|
||||
.bind(team_id)
|
||||
.fetch_one(&mut *second)
|
||||
.await?;
|
||||
assert!(released);
|
||||
let mut next = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let (next_backend, next_timeout): (i32, String) =
|
||||
sqlx::query_as("SELECT pg_backend_pid(), current_setting('lock_timeout')")
|
||||
.fetch_one(&mut *next)
|
||||
.await?;
|
||||
assert_eq!(next_backend, backend);
|
||||
assert_eq!(next_timeout, original_timeout);
|
||||
next.rollback().await?;
|
||||
second.rollback().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
162
litellm-rust/crates/db/src/spend/team_member_spend.rs
Normal file
162
litellm-rust/crates/db/src/spend/team_member_spend.rs
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
use sqlx::{Postgres, Transaction};
|
||||
|
||||
use super::lock_team;
|
||||
use crate::Error;
|
||||
|
||||
#[macro_rules_attribute::apply(crate::db_input)]
|
||||
pub struct TeamMemberSpend {
|
||||
pub team_id: String,
|
||||
pub user_id: String,
|
||||
pub cost: f64,
|
||||
}
|
||||
|
||||
pub async fn add_team_member_spend(
|
||||
transaction: &mut Transaction<'_, Postgres>,
|
||||
members: &[TeamMemberSpend],
|
||||
) -> Result<(), Error> {
|
||||
if members.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
let mut members = members.to_vec();
|
||||
members.sort_by(|left, right| {
|
||||
(&left.team_id, &left.user_id).cmp(&(&right.team_id, &right.user_id))
|
||||
});
|
||||
let mut teams: Vec<&str> = members
|
||||
.iter()
|
||||
.map(|member| member.team_id.as_str())
|
||||
.collect();
|
||||
teams.dedup();
|
||||
for team in teams {
|
||||
lock_team(transaction, team).await?;
|
||||
}
|
||||
let rows = serde_json::Value::Array(
|
||||
members
|
||||
.iter()
|
||||
.map(|member| {
|
||||
serde_json::json!({
|
||||
"user_id": member.user_id,
|
||||
"team_id": member.team_id,
|
||||
"cost": member.cost,
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
sqlx::query_file!("queries/spend/upsert_team_member_spend.sql", rows)
|
||||
.execute(&mut **transaction)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::super::begin;
|
||||
use super::{TeamMemberSpend, add_team_member_spend};
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
fn members(cost: f64) -> [TeamMemberSpend; 3] {
|
||||
[
|
||||
TeamMemberSpend {
|
||||
team_id: "member-spend-z".into(),
|
||||
user_id: "off-roster-existing".into(),
|
||||
cost: 2.0 * cost,
|
||||
},
|
||||
TeamMemberSpend {
|
||||
team_id: "member-spend-a".into(),
|
||||
user_id: "on-roster-missing".into(),
|
||||
cost,
|
||||
},
|
||||
TeamMemberSpend {
|
||||
team_id: "member-spend-a".into(),
|
||||
user_id: "off-roster-missing".into(),
|
||||
cost: 50.0 * cost,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::zero_then_quarter(0.25, 0.5, 3.0, 8.0)]
|
||||
#[case::zero_then_eighth(0.125, 0.25, 2.5, 7.5)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn membership_spend_preserves_roster_guard_and_fractional_costs(
|
||||
#[case] cost: f64,
|
||||
#[case] expected_created: f64,
|
||||
#[case] expected_spend: f64,
|
||||
#[case] expected_total: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = begin(&pool).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _teams = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, members_with_roles)
|
||||
VALUES ('member-spend-a', '[{"user_id":"on-roster-missing","role":"user"}]'),
|
||||
('member-spend-z', '[]')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _members = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_TeamMembership" (team_id, user_id, spend, total_spend)
|
||||
VALUES ('member-spend-z', 'off-roster-existing', 2, 7),
|
||||
('member-spend-a', 'untouched', 100, 200)"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
add_team_member_spend(&mut tx, &[]).await?;
|
||||
add_team_member_spend(&mut tx, &members(0.0)).await?;
|
||||
add_team_member_spend(&mut tx, &members(cost)).await?;
|
||||
add_team_member_spend(&mut tx, &members(cost)).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, String, f64, f64)> = sqlx::query_as(
|
||||
r#"SELECT team_id, user_id, spend, total_spend FROM "LiteLLM_TeamMembership"
|
||||
WHERE team_id IN ('member-spend-a', 'member-spend-z') ORDER BY team_id, user_id"#,
|
||||
)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
(
|
||||
"member-spend-a".into(),
|
||||
"on-roster-missing".into(),
|
||||
expected_created,
|
||||
expected_created
|
||||
),
|
||||
("member-spend-a".into(), "untouched".into(), 100.0, 200.0),
|
||||
(
|
||||
"member-spend-z".into(),
|
||||
"off-roster-existing".into(),
|
||||
expected_spend,
|
||||
expected_total
|
||||
),
|
||||
]
|
||||
);
|
||||
tx.rollback().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let remaining: i64 = sqlx::query_scalar(
|
||||
r#"SELECT COUNT(*) FROM "LiteLLM_TeamMembership"
|
||||
WHERE team_id IN ('member-spend-a', 'member-spend-z')"#,
|
||||
)
|
||||
.fetch_one(&pool)
|
||||
.await?;
|
||||
assert_eq!(remaining, 0);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
182
litellm-rust/crates/db/src/spend/team_spend.rs
Normal file
182
litellm-rust/crates/db/src/spend/team_spend.rs
Normal file
|
|
@ -0,0 +1,182 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use super::EntitySpend;
|
||||
use crate::Error;
|
||||
|
||||
pub async fn add_team_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
spend: &EntitySpend,
|
||||
) -> Result<(), Error> {
|
||||
spend.validate()?;
|
||||
if spend.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
sqlx::query_file!("queries/spend/add_team_spend.sql", &spend.ids, &spend.costs)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{EntitySpend, add_team_spend};
|
||||
use crate::Error;
|
||||
use crate::test_support::database_pool;
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn duplicate_team_ids_add_every_cost(
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
|
||||
VALUES ('team-spend-duplicate-a', 2, '2000-01-01', '2000-01-01'),
|
||||
('team-spend-duplicate-z', 3, '2000-01-01', '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec![
|
||||
"team-spend-duplicate-z".into(),
|
||||
"team-spend-duplicate-a".into(),
|
||||
"team-spend-duplicate-z".into(),
|
||||
"team-spend-duplicate-a".into(),
|
||||
],
|
||||
costs: vec![0.5, 0.25, 1.0, 0.5],
|
||||
};
|
||||
add_team_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64)> = sqlx::query_as(
|
||||
r#"SELECT team_id, spend FROM "LiteLLM_TeamTable"
|
||||
WHERE team_id = ANY($1) ORDER BY team_id"#,
|
||||
)
|
||||
.bind(&spend.ids)
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("team-spend-duplicate-a".into(), 2.75),
|
||||
("team-spend-duplicate-z".into(), 4.5),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_cost(vec!["team-spend-malformed".into()], vec![])]
|
||||
#[case::extra_cost(vec!["team-spend-malformed".into()], vec![0.25, 0.5])]
|
||||
#[case::empty_ids(vec![], vec![0.25])]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn malformed_team_batch_does_not_write(
|
||||
#[case] ids: Vec<String>,
|
||||
#[case] costs: Vec<f64>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
|
||||
VALUES ('team-spend-malformed', 2, '2000-01-01', '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let result = add_team_spend(&mut *tx, &EntitySpend { ids, costs }).await;
|
||||
assert!(matches!(result, Err(Error::InvalidSpendBatch { .. })));
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (f64, bool) = sqlx::query_as(
|
||||
r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_TeamTable" WHERE team_id = 'team-spend-malformed'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(row, (2.0, true));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::fractional_costs(0.25, 2.5, 4.0)]
|
||||
#[case::zero_costs(0.0, 2.0, 3.0)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn team_spend_matches_python_update_many(
|
||||
#[case] cost: f64,
|
||||
#[case] expected_a_spend: f64,
|
||||
#[case] expected_z_spend: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_TeamTable" (team_id, spend, created_at, updated_at)
|
||||
VALUES ('team-spend-a', 2, '2000-01-01', '2000-01-01'),
|
||||
('team-spend-z', 3, '2000-01-01', '2000-01-01'),
|
||||
('team-spend-untouched', 100, '2000-01-01', '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec![
|
||||
"team-spend-z".into(),
|
||||
"team-spend-a".into(),
|
||||
"team-spend-missing".into(),
|
||||
],
|
||||
costs: vec![2.0 * cost, cost, 50.0],
|
||||
};
|
||||
add_team_spend(&mut *tx, &EntitySpend::default()).await?;
|
||||
add_team_spend(&mut *tx, &spend).await?;
|
||||
add_team_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
|
||||
r#"SELECT team_id, spend, updated_at > TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_TeamTable"
|
||||
WHERE team_id = ANY($1) ORDER BY team_id"#,
|
||||
)
|
||||
.bind(vec![
|
||||
"team-spend-a",
|
||||
"team-spend-z",
|
||||
"team-spend-missing",
|
||||
"team-spend-untouched",
|
||||
])
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("team-spend-a".into(), expected_a_spend, true),
|
||||
("team-spend-untouched".into(), 100.0, false),
|
||||
("team-spend-z".into(), expected_z_spend, true),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
173
litellm-rust/crates/db/src/spend/user_spend.rs
Normal file
173
litellm-rust/crates/db/src/spend/user_spend.rs
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
use sqlx::PgExecutor;
|
||||
|
||||
use super::EntitySpend;
|
||||
use crate::Error;
|
||||
|
||||
pub async fn add_user_spend(
|
||||
executor: impl PgExecutor<'_>,
|
||||
spend: &EntitySpend,
|
||||
) -> Result<(), Error> {
|
||||
spend.validate()?;
|
||||
if spend.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
sqlx::query_file!("queries/spend/add_user_spend.sql", &spend.ids, &spend.costs)
|
||||
.execute(executor)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::error::Error as StdError;
|
||||
|
||||
use super::{EntitySpend, add_user_spend};
|
||||
use crate::{Error, test_support::database_pool};
|
||||
use rstest::rstest;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[rstest]
|
||||
#[case::fractional_costs(0.25, 2.5, 4.0)]
|
||||
#[case::zero_costs(0.0, 2.0, 3.0)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn user_spend_matches_python_update_many(
|
||||
#[case] cost: f64,
|
||||
#[case] expected_a_spend: f64,
|
||||
#[case] expected_z_spend: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
|
||||
VALUES ('user-spend-a', 2, '2000-01-01'),
|
||||
('user-spend-z', 3, '2000-01-01'),
|
||||
('user-spend-untouched', 100, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec![
|
||||
"user-spend-z".into(),
|
||||
"user-spend-a".into(),
|
||||
"user-spend-missing".into(),
|
||||
],
|
||||
costs: vec![2.0 * cost, cost, 50.0],
|
||||
};
|
||||
add_user_spend(&mut *tx, &EntitySpend::default()).await?;
|
||||
add_user_spend(&mut *tx, &spend).await?;
|
||||
add_user_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let rows: Vec<(String, f64, bool)> = sqlx::query_as(
|
||||
r#"SELECT user_id, spend, updated_at > TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_UserTable"
|
||||
WHERE user_id = ANY($1) ORDER BY user_id"#,
|
||||
)
|
||||
.bind(vec![
|
||||
"user-spend-a",
|
||||
"user-spend-z",
|
||||
"user-spend-missing",
|
||||
"user-spend-untouched",
|
||||
])
|
||||
.fetch_all(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rows,
|
||||
vec![
|
||||
("user-spend-a".into(), expected_a_spend, true),
|
||||
("user-spend-untouched".into(), 100.0, false),
|
||||
("user-spend-z".into(), expected_z_spend, true),
|
||||
]
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::direct_duplicate_batch(vec![0.25, 0.75], 3.0)]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn user_spend_sums_duplicate_ids(
|
||||
#[case] costs: Vec<f64>,
|
||||
#[case] expected_spend: f64,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
|
||||
VALUES ('user-spend-duplicate', 2, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec!["user-spend-duplicate".into(), "user-spend-duplicate".into()],
|
||||
costs,
|
||||
};
|
||||
add_user_spend(&mut *tx, &spend).await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (f64, bool) = sqlx::query_as(
|
||||
r#"SELECT spend, updated_at > TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-duplicate'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(row, (expected_spend, true));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing_cost(vec![])]
|
||||
#[case::extra_cost(vec![0.25, 0.75])]
|
||||
#[tokio::test]
|
||||
#[ignore = "requires DATABASE_URL pointing to a Prisma-migrated PostgreSQL database"]
|
||||
async fn user_spend_rejects_malformed_batch_without_writes(
|
||||
#[case] costs: Vec<f64>,
|
||||
#[future(awt)] database_pool: Result<PgPool, Box<dyn StdError>>,
|
||||
) -> Result<(), Box<dyn StdError>> {
|
||||
let pool = database_pool?;
|
||||
let mut tx = pool.begin().await?;
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Fixture writes are outside the compile-checked query set"
|
||||
)]
|
||||
let _inserted = sqlx::query(
|
||||
r#"INSERT INTO "LiteLLM_UserTable" (user_id, spend, updated_at)
|
||||
VALUES ('user-spend-malformed', 2, '2000-01-01')"#,
|
||||
)
|
||||
.execute(&mut *tx)
|
||||
.await?;
|
||||
let spend = EntitySpend {
|
||||
ids: vec!["user-spend-malformed".into()],
|
||||
costs,
|
||||
};
|
||||
let result = add_user_spend(&mut *tx, &spend).await;
|
||||
assert!(matches!(result, Err(Error::InvalidSpendBatch { .. })));
|
||||
#[expect(
|
||||
clippy::disallowed_methods,
|
||||
reason = "Test assertions are outside the compile-checked query set"
|
||||
)]
|
||||
let row: (f64, bool) = sqlx::query_as(
|
||||
r#"SELECT spend, updated_at = TIMESTAMP '2000-01-01'
|
||||
FROM "LiteLLM_UserTable" WHERE user_id = 'user-spend-malformed'"#,
|
||||
)
|
||||
.fetch_one(&mut *tx)
|
||||
.await?;
|
||||
assert_eq!(row, (2.0, true));
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
10
litellm-rust/crates/db/src/test_support.rs
Normal file
10
litellm-rust/crates/db/src/test_support.rs
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
use std::error::Error as StdError;
|
||||
|
||||
use rstest::fixture;
|
||||
use sqlx::PgPool;
|
||||
|
||||
#[fixture]
|
||||
pub async fn database_pool() -> Result<PgPool, Box<dyn StdError>> {
|
||||
let database_url = std::env::var("DATABASE_URL")?;
|
||||
Ok(PgPool::connect(&database_url).await?)
|
||||
}
|
||||
|
|
@ -88,20 +88,23 @@ fn aws_context(
|
|||
}
|
||||
|
||||
fn read_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<Duration>> {
|
||||
let Some(value) = value.filter(|value| !value.is_none()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let seconds = match value.extract::<f64>() {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => value.getattr("read")?.extract::<Option<f64>>()?,
|
||||
};
|
||||
seconds
|
||||
read_timeout_seconds(value)?
|
||||
.map(|value| {
|
||||
Duration::try_from_secs_f64(value).map_err(|_| PyValueError::new_err("invalid timeout"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(super) fn read_timeout_seconds(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<f64>> {
|
||||
let Some(value) = value.filter(|value| !value.is_none()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
match value.extract::<f64>() {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(_) => value.getattr("read")?.extract::<Option<f64>>(),
|
||||
}
|
||||
}
|
||||
|
||||
fn vault_context(params: Option<&Bound<'_, PyAny>>) -> PyResult<HashicorpOperationContext> {
|
||||
let params = params.and_then(|value| value.cast::<PyDict>().ok());
|
||||
let nested = params
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ mod operation;
|
|||
|
||||
pub(super) use operation::{Failure, FailureKind, FailureStage, delete, rotate, write};
|
||||
|
||||
use litellm_core_utils::exception_mapping_utils::{TimeoutPhase, timeout_message};
|
||||
use litellm_secrets::hashicorp::{Error, RawOperationError};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use super::mutation::{error_value, http_message, json_value};
|
||||
use super::provider::read_timeout_seconds;
|
||||
|
||||
const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport";
|
||||
|
||||
|
|
@ -69,21 +71,17 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P
|
|||
}
|
||||
FailureKind::Native(RawOperationError::Timeout { method, elapsed }) => {
|
||||
if method == "POST" {
|
||||
let elapsed = py
|
||||
.import("builtins")?
|
||||
.call_method1("round", (elapsed.as_secs_f64(), 3))?;
|
||||
let kwargs = pyo3::types::PyDict::new(py);
|
||||
kwargs.set_item(
|
||||
"message",
|
||||
format!(
|
||||
"Connection timed out. Timeout passed={}, time taken={} seconds",
|
||||
context.timeout.as_deref().unwrap_or("None"),
|
||||
elapsed.str()?
|
||||
timeout_message(
|
||||
TimeoutPhase::Read,
|
||||
context.timeout,
|
||||
Some(elapsed.as_secs_f64()),
|
||||
),
|
||||
)?;
|
||||
kwargs.set_item("model", "default-model-name")?;
|
||||
kwargs.set_item("llm_provider", "litellm-httpx-handler")?;
|
||||
kwargs.set_item("headers", pyo3::types::PyDict::new(py))?;
|
||||
py.import("litellm.exceptions")?
|
||||
.getattr("Timeout")?
|
||||
.call((), Some(&kwargs))?
|
||||
|
|
@ -159,7 +157,7 @@ fn response_text(py: Python<'_>, body: &[u8]) -> PyResult<String> {
|
|||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ErrorContext {
|
||||
timeout: Option<String>,
|
||||
timeout: Option<f64>,
|
||||
aiohttp: bool,
|
||||
}
|
||||
|
||||
|
|
@ -173,7 +171,7 @@ impl ErrorContext {
|
|||
return Ok(Self::default());
|
||||
}
|
||||
Ok(Self {
|
||||
timeout: timeout.map(|value| value.str()?.extract()).transpose()?,
|
||||
timeout: read_timeout_seconds(timeout)?,
|
||||
aiohttp: py
|
||||
.import(HTTP_TRANSPORT_MODULE)?
|
||||
.getattr("uses_aiohttp_transport")?
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
"""
|
||||
Internal unified UUID helper.
|
||||
|
||||
Always uses fastuuid for performance.
|
||||
Uses fastuuid where supported and the standard library on Python 3.15+.
|
||||
"""
|
||||
|
||||
import fastuuid as _uuid
|
||||
import sys
|
||||
import uuid as _uuid
|
||||
|
||||
if sys.version_info < (3, 15):
|
||||
import fastuuid as _uuid
|
||||
|
||||
# Expose a module-like alias so callers can use: uuid.uuid4()
|
||||
uuid = _uuid
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ class OtelDestination(LiteLLMBaseModel):
|
|||
success_sampling_rate: float | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; "
|
||||
"Share of the request trees forwarded, 0.0..1.0, picked by a hash of the trace id; "
|
||||
"``None`` forwards every one."
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,13 +1,14 @@
|
|||
"""Provider / exporter factory + the Baggage span processor."""
|
||||
|
||||
import queue
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
import weakref
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from hashlib import sha256
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
from typing import TYPE_CHECKING, Final, Literal, TypeAlias
|
||||
|
||||
from opentelemetry import _logs, baggage, metrics, trace
|
||||
from opentelemetry._logs import Logger, LoggerProvider, NoOpLoggerProvider
|
||||
|
|
@ -486,15 +487,15 @@ def is_llm_call_span(span: ReadableSpan) -> bool:
|
|||
return GenAI.OPERATION_NAME in attributes and MCP.METHOD_NAME not in attributes
|
||||
|
||||
|
||||
#: Request trees held back for a sampled destination until the last span of the trace
|
||||
#: Request trees held back for a sampled destination until the last span of the request
|
||||
#: still open ends, the verdicts kept for the spans that start after that (a post-call
|
||||
#: database write), and the spans in flight per trace. All bounded, so a span that never
|
||||
#: ends, or a flood of requests, cannot hold spans for ever: the oldest tree is decided
|
||||
#: on what it has.
|
||||
#: database write), the spans in flight per request, and the request each open span
|
||||
#: belongs to. All bounded, so a span that never ends, or a flood of requests, cannot
|
||||
#: hold spans for ever: the oldest tree is decided on what it has.
|
||||
_MAX_PENDING_TREES: Final = 1024
|
||||
_MAX_PENDING_SPANS_PER_TREE: Final = 512
|
||||
_MAX_REMEMBERED_VERDICTS: Final = 4096
|
||||
_MAX_OPEN_TRACES: Final = 16384
|
||||
_MAX_OPEN_REQUESTS: Final = 16384
|
||||
|
||||
|
||||
class _PendingTree:
|
||||
|
|
@ -512,25 +513,35 @@ def _is_sampled(destination: "OtelDestination") -> bool:
|
|||
return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None
|
||||
|
||||
|
||||
#: A verdict is one request tree's draw for one destination at its rates: the exporter
|
||||
#: A request is its trace id and the id of its root, the span of it whose parent is
|
||||
#: remote or missing. Calls sharing a caller's trace are separate requests, each decided
|
||||
#: on its own outcome.
|
||||
_RequestKey: TypeAlias = tuple[int, int]
|
||||
|
||||
#: A verdict is one request's keep or drop for one destination at its rates: the exporter
|
||||
#: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with
|
||||
#: different rates would otherwise share one draw.
|
||||
_VerdictKey = tuple[int, object, float | None, float | None]
|
||||
#: different rates would otherwise share one verdict.
|
||||
_VerdictKey: TypeAlias = tuple[int, int, object, float | None, float | None]
|
||||
|
||||
|
||||
def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey:
|
||||
return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
|
||||
def _verdict_key(request: _RequestKey, destination: "OtelDestination") -> _VerdictKey:
|
||||
return (*request, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
|
||||
|
||||
|
||||
def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool:
|
||||
"""Whether one draw against the destination's rate keeps a request tree, read the
|
||||
way the legacy Arize callback reads ``arize_success_sampling_rate`` /
|
||||
``arize_error_sampling_rate``: a tree with a failed span answers to the error
|
||||
rate, an unset rate keeps everything and ``0.0`` keeps nothing."""
|
||||
def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool:
|
||||
"""Whether the destination's rate keeps a request tree of ``trace_id``: a tree with
|
||||
a failed span answers to ``arize_error_sampling_rate``, the rest to
|
||||
``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps
|
||||
nothing.
|
||||
|
||||
The first eight bytes of a SHA-256 of the trace id are read against the rate, so
|
||||
every process deciding a request of the trace with the same outcome reaches the same
|
||||
verdict."""
|
||||
rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate
|
||||
if rate is None:
|
||||
return True
|
||||
return rate > 0.0 and draw() <= rate
|
||||
digest: Final = sha256(trace_id.to_bytes(16, "big")).digest()
|
||||
return int.from_bytes(digest[:8], "big") < round(rate * 2**64)
|
||||
|
||||
|
||||
def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool:
|
||||
|
|
@ -667,10 +678,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
|
||||
pending_drains: int = _MAX_PENDING_DRAINS,
|
||||
drain_pool: _DrainPool | None = None,
|
||||
sampling_draw: Callable[[], float] | None = None,
|
||||
) -> None:
|
||||
self._operator_sinks: Final = operator_sinks
|
||||
self._draw: Final = sampling_draw if sampling_draw is not None else random.random
|
||||
self._excluded_db_systems: Final = excluded_db_systems
|
||||
self._default_capture: Final = default_capture
|
||||
self._default_capture_by_backend: Final = default_capture_by_backend
|
||||
|
|
@ -682,24 +691,32 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
self._processors: OrderedDict[object, SpanProcessor] = OrderedDict() # mutable-ok: bounded LRU
|
||||
self._retired: OrderedDict[int, SpanProcessor] = OrderedDict() # mutable-ok: drains as exports finish
|
||||
self._exporting: dict[int, int] = {} # mutable-ok: per-processor in-flight export count
|
||||
self._pending: OrderedDict[int, _PendingTree] = OrderedDict() # mutable-ok: bounded, by trace id
|
||||
self._open: OrderedDict[int, int] = OrderedDict() # mutable-ok: bounded, spans in flight by trace id
|
||||
self._pending: OrderedDict[_RequestKey, _PendingTree] = OrderedDict() # mutable-ok: bounded, by request
|
||||
self._open: OrderedDict[_RequestKey, set[int]] = OrderedDict() # mutable-ok: bounded, open span ids by request
|
||||
self._roots: dict[int, _RequestKey] = {} # mutable-ok: holds only the spans ``_open`` holds, request by span id
|
||||
self._span_roots: weakref.WeakKeyDictionary[SDKSpan, int] = weakref.WeakKeyDictionary()
|
||||
self._verdicts: OrderedDict[_VerdictKey, bool] = OrderedDict() # mutable-ok: bounded LRU
|
||||
self._drain: Final = drain_pool if drain_pool is not None else _DrainPool(capacity=pending_drains)
|
||||
|
||||
def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None:
|
||||
context: Final = span.context
|
||||
trace_id: Final = context.trace_id if context is not None else 0
|
||||
with self._lock:
|
||||
self._open[trace_id] = self._open.get(trace_id, 0) + 1
|
||||
while len(self._open) > _MAX_OPEN_TRACES:
|
||||
self._open.popitem(last=False)
|
||||
request: Final = self._request_of(span, trace.get_current_span(parent_context))
|
||||
if context is not None:
|
||||
self._span_roots[span] = request[1]
|
||||
self._roots[context.span_id] = request
|
||||
self._open.setdefault(request, set()).add(context.span_id)
|
||||
while len(self._open) > _MAX_OPEN_REQUESTS:
|
||||
for forgotten in self._open.popitem(last=False)[1]:
|
||||
self._roots.pop(forgotten, None)
|
||||
|
||||
def on_end(self, span: ReadableSpan) -> None:
|
||||
suppressed: Final = suppressed_backends()
|
||||
attributes: Final = span.attributes or _NO_ATTRIBUTES
|
||||
context: Final = span.context
|
||||
trace_id: Final = context.trace_id if context is not None else 0
|
||||
with self._lock:
|
||||
request: Final = self._request_of(span)
|
||||
if span.context is not None:
|
||||
self._roots.pop(span.context.span_id, None)
|
||||
failed: Final = span.status.status_code is StatusCode.ERROR
|
||||
for destination in request_destinations():
|
||||
if (
|
||||
|
|
@ -716,11 +733,32 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
if not _is_sampled(destination):
|
||||
self._forward(span, destination)
|
||||
continue
|
||||
for held_span, held_destination in self._route(trace_id, span, destination, failed):
|
||||
for held_span, held_destination in self._route(request, span, destination, failed):
|
||||
self._forward(held_span, held_destination)
|
||||
for held_span, held_destination in self._settle(trace_id, failed):
|
||||
for held_span, held_destination in self._settle(request, span, failed):
|
||||
self._forward(held_span, held_destination)
|
||||
|
||||
def _request_of(self, span: ReadableSpan, parent_span: Span | None = None) -> _RequestKey:
|
||||
"""The request ``span`` belongs to, read under ``_lock``: the request recorded while
|
||||
it is open, else the span itself when its parent is remote or missing, else its
|
||||
parent's request, recorded while the parent is open or on ``parent_span`` for a
|
||||
child starting under a parent that has ended, else the parent itself."""
|
||||
context: Final = span.context
|
||||
if context is None:
|
||||
return (0, 0)
|
||||
recorded: Final = self._roots.get(context.span_id)
|
||||
if recorded is not None:
|
||||
return recorded
|
||||
parent: Final = span.parent
|
||||
if parent is None or parent.is_remote:
|
||||
return (context.trace_id, context.span_id)
|
||||
parent_request: Final = self._roots.get(parent.span_id)
|
||||
if parent_request is not None:
|
||||
return parent_request
|
||||
if isinstance(parent_span, SDKSpan):
|
||||
return (context.trace_id, self._span_roots.get(parent_span, parent.span_id))
|
||||
return (context.trace_id, parent.span_id)
|
||||
|
||||
def _default_capture_for(self, destination: "OtelDestination") -> str:
|
||||
backend: Final = destination.callback_name
|
||||
return (
|
||||
|
|
@ -743,14 +781,14 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
self._release(processor)
|
||||
|
||||
def _route(
|
||||
self, trace_id: int, span: ReadableSpan, destination: "OtelDestination", failed: bool
|
||||
self, request: _RequestKey, span: ReadableSpan, destination: "OtelDestination", failed: bool
|
||||
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""The spans to forward now for ``span`` on a sampled ``destination``: the span
|
||||
itself when its tree's verdict is already known and keeps it, nothing while the
|
||||
tree is held for the rest of its trace, or what the bounds made the fan-out decide
|
||||
tree is held for the rest of its request, or what the bounds made the fan-out decide
|
||||
early (a tree that outgrew its cap is decided on what it has, and so is the oldest
|
||||
tree once too many are waiting, so a span that never ends holds nothing back for
|
||||
ever). A failed span marks its tree before any such early draw. Once shutdown
|
||||
ever). A failed span marks its tree before any such early decision. Once shutdown
|
||||
has started flushing, nothing is held: a span ending then is decided at once,
|
||||
since no later flush would reach it.
|
||||
|
||||
|
|
@ -758,58 +796,62 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
another thread between the two would leave this span in a fresh tree.
|
||||
"""
|
||||
with self._lock:
|
||||
verdict: Final = self._verdicts.get(_verdict_key(trace_id, destination))
|
||||
verdict: Final = self._verdicts.get(_verdict_key(request, destination))
|
||||
if verdict is not None:
|
||||
return ((span, destination),) if verdict else ()
|
||||
tree: Final = self._pending.setdefault(trace_id, _PendingTree())
|
||||
tree: Final = self._pending.setdefault(request, _PendingTree())
|
||||
tree.held.append((span, destination))
|
||||
if failed:
|
||||
tree.failed = True
|
||||
if self._draining or len(tree.held) >= _MAX_PENDING_SPANS_PER_TREE:
|
||||
return self._decide_locked(trace_id)
|
||||
return self._decide_locked(request)
|
||||
if len(self._pending) > _MAX_PENDING_TREES:
|
||||
oldest: Final = next(iter(self._pending))
|
||||
return self._decide_locked(oldest)
|
||||
return ()
|
||||
|
||||
def _decide(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
def _decide(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
with self._lock:
|
||||
return self._decide_locked(trace_id)
|
||||
return self._decide_locked(request)
|
||||
|
||||
def _settle(self, trace_id: int, failed: bool) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""What the trace's held tree exports once the span that just ended was the last
|
||||
of its trace still open. The model-call span ends in the post-call callback,
|
||||
after the server span, so a tree is decided when its trace goes quiet rather than
|
||||
def _settle(
|
||||
self, request: _RequestKey, span: ReadableSpan, failed: bool
|
||||
) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""What the request's held tree exports once the span that just ended was the last
|
||||
of its request still open. The model-call span ends in the post-call callback,
|
||||
after the server span, so a tree is decided when its request goes quiet rather than
|
||||
when its root ends, and a model call that fails late still answers to the error
|
||||
rate. A trace whose starts this fan-out never saw (attached mid-flight, or forgotten
|
||||
by the bound) is decided as each of its spans ends.
|
||||
rate. A request whose starts this fan-out never saw (attached mid-flight, or
|
||||
forgotten by the bound) is decided as each of its spans ends.
|
||||
"""
|
||||
with self._lock:
|
||||
tree: Final = self._pending.get(trace_id)
|
||||
tree: Final = self._pending.get(request)
|
||||
if tree is not None and failed:
|
||||
tree.failed = True
|
||||
still_open: Final = self._open.pop(trace_id, 1) - 1
|
||||
if still_open > 0:
|
||||
self._open[trace_id] = still_open
|
||||
still_open: Final = self._open.pop(request, set())
|
||||
if span.context is not None:
|
||||
still_open.discard(span.context.span_id)
|
||||
if still_open:
|
||||
self._open[request] = still_open
|
||||
return ()
|
||||
return self._decide_locked(trace_id)
|
||||
return self._decide_locked(request)
|
||||
|
||||
def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""Draw once per destination for the held tree; the spans the draws keep, and a
|
||||
verdict remembered for the spans of this trace that start later."""
|
||||
tree: Final = self._pending.pop(trace_id, None)
|
||||
def _decide_locked(self, request: _RequestKey) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""Decide once per destination for the held tree; the spans the verdicts keep, and
|
||||
a verdict remembered for the spans of this request that start later."""
|
||||
tree: Final = self._pending.pop(request, None)
|
||||
if tree is None:
|
||||
return ()
|
||||
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree
|
||||
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree
|
||||
for _, destination in tree.held:
|
||||
key = _verdict_key(trace_id, destination)
|
||||
key = _verdict_key(request, destination)
|
||||
if key not in verdicts:
|
||||
verdicts[key] = _keeps_tree(destination, tree.failed, self._draw)
|
||||
verdicts[key] = _keeps_tree(destination, tree.failed, request[0])
|
||||
self._verdicts[key] = verdicts[key]
|
||||
while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS:
|
||||
self._verdicts.popitem(last=False)
|
||||
return tuple(
|
||||
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(trace_id, destination)]
|
||||
(span, destination) for span, destination in tree.held if verdicts[_verdict_key(request, destination)]
|
||||
)
|
||||
|
||||
def _operator_already_writes(
|
||||
|
|
@ -850,8 +892,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
with self._lock:
|
||||
self._draining = True
|
||||
undecided: Final = tuple(self._pending)
|
||||
for trace_id in undecided:
|
||||
for held_span, held_destination in self._decide(trace_id):
|
||||
for request in undecided:
|
||||
for held_span, held_destination in self._decide(request):
|
||||
self._forward(held_span, held_destination)
|
||||
deadline: Final = time.monotonic() + self._drain_seconds
|
||||
with self._lock:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ _BUDGET_DURATION_WORD_ALIASES: Final[dict[str, str]] = {
|
|||
}
|
||||
|
||||
|
||||
def _normalize_duration(duration: str) -> str:
|
||||
def normalize_duration(duration: str) -> str:
|
||||
return _BUDGET_DURATION_WORD_ALIASES.get(duration.strip().lower(), duration)
|
||||
|
||||
|
||||
|
|
@ -61,7 +61,7 @@ def duration_in_seconds(duration: str) -> int:
|
|||
|
||||
Returns time in seconds till when budget needs to be reset
|
||||
"""
|
||||
value, unit = _extract_from_regex(duration=_normalize_duration(duration))
|
||||
value, unit = _extract_from_regex(duration=normalize_duration(duration))
|
||||
|
||||
if unit == "s":
|
||||
return value
|
||||
|
|
@ -89,7 +89,7 @@ def duration_in_seconds_from(start: datetime, duration: str) -> int:
|
|||
month of a different length needs its own start to know when it ends. Every other
|
||||
unit is a fixed length and ignores `start`.
|
||||
"""
|
||||
value, unit = _extract_from_regex(duration=_normalize_duration(duration))
|
||||
value, unit = _extract_from_regex(duration=normalize_duration(duration))
|
||||
if unit == "mo":
|
||||
return _months_in_seconds(start=start, months=value)
|
||||
return duration_in_seconds(duration)
|
||||
|
|
@ -129,7 +129,7 @@ def get_next_standardized_reset_time(
|
|||
current_time, _ = _setup_timezone(current_time, timezone_str)
|
||||
|
||||
# Parse duration
|
||||
value, unit = _parse_duration(_normalize_duration(duration))
|
||||
value, unit = _parse_duration(normalize_duration(duration))
|
||||
if value is None:
|
||||
verbose_logger.warning(
|
||||
"Unrecognized budget_duration %r; falling back to a next-midnight reset. "
|
||||
|
|
@ -171,7 +171,7 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
|
|||
"""Start of the budget period that ends at `reset_at`, under the same rules
|
||||
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
|
||||
months and a duration it does not recognize resets at the next midnight."""
|
||||
value, unit = _parse_duration(_normalize_duration(duration))
|
||||
value, unit = _parse_duration(normalize_duration(duration))
|
||||
if value is None:
|
||||
return reset_at - timedelta(days=1)
|
||||
match unit:
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ OPTIONAL_KWARGS_KEYS: Final = (
|
|||
"itpm",
|
||||
"otpm",
|
||||
"use_xai_oauth",
|
||||
"chatgpt_auth_file",
|
||||
"github_copilot_auth_type",
|
||||
"github_copilot_host",
|
||||
"github_copilot_user_session",
|
||||
|
|
|
|||
|
|
@ -158,6 +158,14 @@ def declared_authenticating_provider(model: str | None, custom_llm_provider: str
|
|||
return declared if declared in PROVIDERS_THAT_AUTHENTICATE_ON_PROVIDER_INFO else None
|
||||
|
||||
|
||||
def resolve_model_and_provider_for_metadata(model: str, custom_llm_provider: str | None = None) -> tuple[str, str]:
|
||||
declared: Final = declared_authenticating_provider(model, custom_llm_provider)
|
||||
if declared is not None:
|
||||
return model.removeprefix(f"{declared}/"), declared
|
||||
resolved_model, resolved_provider, _, _ = get_llm_provider(model=model, custom_llm_provider=custom_llm_provider)
|
||||
return resolved_model, resolved_provider
|
||||
|
||||
|
||||
def inferred_provider(model: str | None) -> str | None:
|
||||
if not model:
|
||||
return None
|
||||
|
|
@ -842,7 +850,9 @@ def _get_openai_compatible_provider_info(
|
|||
api_base,
|
||||
dynamic_api_key,
|
||||
custom_llm_provider,
|
||||
) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
|
||||
) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(
|
||||
model, api_base, api_key, custom_llm_provider, litellm_params
|
||||
)
|
||||
elif custom_llm_provider == "novita":
|
||||
api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai"
|
||||
dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY")
|
||||
|
|
|
|||
|
|
@ -1,10 +1,23 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.types.utils import ProviderSpecificHeader
|
||||
|
||||
_PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter(
|
||||
ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None
|
||||
)
|
||||
|
||||
|
||||
class ProviderSpecificHeaderUtils:
|
||||
@staticmethod
|
||||
def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None:
|
||||
headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
_PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider
|
||||
)
|
||||
return next((value for key, value in headers.items() if key.lower() == "authorization"), None)
|
||||
|
||||
@staticmethod
|
||||
def get_provider_specific_headers(
|
||||
provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None,
|
||||
|
|
|
|||
|
|
@ -1490,6 +1490,7 @@ def convert_to_gemini_tool_call_result(
|
|||
message: ChatCompletionToolMessage | ChatCompletionFunctionMessage,
|
||||
last_message_with_tool_calls: dict | None,
|
||||
forward_function_call_id: bool = False,
|
||||
supports_multimodal_function_responses: bool = True,
|
||||
) -> VertexPartType | list[VertexPartType]:
|
||||
"""
|
||||
OpenAI message with a tool result looks like:
|
||||
|
|
@ -1652,9 +1653,9 @@ def convert_to_gemini_tool_call_result(
|
|||
|
||||
_part: Final[VertexPartType] = {"function_response": _function_response}
|
||||
|
||||
# For multimodal function responses, Gemini expects media parts nested
|
||||
# inside functionResponse.parts instead of sibling content parts.
|
||||
if inline_data_list:
|
||||
if not supports_multimodal_function_responses:
|
||||
return [_part, *(VertexPartType(inline_data=inline_data) for inline_data in inline_data_list)]
|
||||
_function_response["parts"] = [{"inline_data": inline_data} for inline_data in inline_data_list]
|
||||
return [_part]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
import base64
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from typing import Final, TypeAlias
|
||||
from functools import lru_cache
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter, ValidationError
|
||||
|
|
@ -24,19 +24,21 @@ from .common_utils import (
|
|||
CHATGPT_OAUTH_TOKEN_URL,
|
||||
GetAccessTokenError,
|
||||
GetDeviceCodeError,
|
||||
JsonObject,
|
||||
RefreshAccessTokenError,
|
||||
decode_jwt_claims,
|
||||
extract_chatgpt_account_id,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
TOKEN_EXPIRY_SKEW_SECONDS: Final = 60
|
||||
TOKEN_REFRESH_TIMEOUT_SECONDS: Final = 30
|
||||
DEVICE_CODE_TIMEOUT_SECONDS: Final = 15 * 60
|
||||
DEVICE_CODE_COOLDOWN_SECONDS: Final = 5 * 60
|
||||
DEVICE_CODE_POLL_SLEEP_SECONDS: Final = 5
|
||||
|
||||
OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth"
|
||||
|
||||
JsonObject: TypeAlias = Mapping[str, JsonValue]
|
||||
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject)
|
||||
|
||||
|
||||
|
|
@ -44,6 +46,26 @@ def _optional_str(value: JsonValue | None) -> str | None:
|
|||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
def get_chatgpt_auth_file(
|
||||
litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None",
|
||||
) -> str | None:
|
||||
if litellm_params is None:
|
||||
return None
|
||||
value: Final = (
|
||||
litellm_params.get("chatgpt_auth_file")
|
||||
if isinstance(litellm_params, Mapping)
|
||||
else litellm_params.chatgpt_auth_file
|
||||
)
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
return os.path.abspath(os.path.expanduser(value))
|
||||
|
||||
|
||||
@lru_cache(maxsize=128)
|
||||
def get_cached_authenticator(auth_file: str) -> "Authenticator":
|
||||
return Authenticator(auth_file=auth_file)
|
||||
|
||||
|
||||
def _token_refresh_timeout() -> httpx.Timeout:
|
||||
configured: Final = get_configured_request_timeout()
|
||||
seconds: Final = (
|
||||
|
|
@ -53,13 +75,13 @@ def _token_refresh_timeout() -> httpx.Timeout:
|
|||
|
||||
|
||||
class Authenticator:
|
||||
def __init__(self) -> None:
|
||||
self.token_dir = os.getenv(
|
||||
"CHATGPT_TOKEN_DIR",
|
||||
os.path.expanduser("~/.config/litellm/chatgpt"),
|
||||
def __init__(self, auth_file: str | None = None) -> None:
|
||||
default_auth_file: Final = os.path.join(
|
||||
os.getenv("CHATGPT_TOKEN_DIR", os.path.expanduser("~/.config/litellm/chatgpt")),
|
||||
os.getenv("CHATGPT_AUTH_FILE", "auth.json"),
|
||||
)
|
||||
self.auth_file = os.path.join(self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json"))
|
||||
self._ensure_token_dir()
|
||||
self.auth_file = os.path.abspath(os.path.expanduser(auth_file or default_auth_file))
|
||||
self.token_dir = os.path.dirname(self.auth_file)
|
||||
|
||||
def get_api_base(self) -> str:
|
||||
return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE
|
||||
|
|
@ -99,16 +121,18 @@ class Authenticator:
|
|||
tokens: Final = self._login_device_code()
|
||||
return tokens["access_token"]
|
||||
|
||||
def get_account_id(self) -> str | None:
|
||||
def get_account_id(self, access_token: str | None = None) -> str | None:
|
||||
auth_data: Final = self._read_auth_file()
|
||||
if access_token and (not auth_data or auth_data.get("access_token") != access_token):
|
||||
return extract_chatgpt_account_id(access_token)
|
||||
if not auth_data:
|
||||
return None
|
||||
account_id: Final = _optional_str(auth_data.get("account_id"))
|
||||
if account_id:
|
||||
return account_id
|
||||
id_token: Final = auth_data.get("id_token")
|
||||
access_token: Final = auth_data.get("access_token")
|
||||
derived: Final = self._extract_account_id(_optional_str(id_token or access_token))
|
||||
saved_access_token: Final = auth_data.get("access_token")
|
||||
derived: Final = extract_chatgpt_account_id(_optional_str(id_token or saved_access_token))
|
||||
if derived:
|
||||
self._write_auth_file({**auth_data, "account_id": derived})
|
||||
return derived
|
||||
|
|
@ -129,6 +153,7 @@ class Authenticator:
|
|||
|
||||
def _write_auth_file(self, data: JsonObject) -> None:
|
||||
try:
|
||||
self._ensure_token_dir()
|
||||
with open(self.auth_file, "w") as f:
|
||||
json.dump(data, f)
|
||||
except OSError as exc:
|
||||
|
|
@ -146,34 +171,11 @@ class Authenticator:
|
|||
return time.time() >= float(derived_expires_at) - TOKEN_EXPIRY_SKEW_SECONDS
|
||||
|
||||
def _get_expires_at(self, token: str) -> int | None:
|
||||
claims: Final = self._decode_jwt_claims(token)
|
||||
exp: Final = claims.get("exp")
|
||||
exp: Final = decode_jwt_claims(token).get("exp")
|
||||
if isinstance(exp, (int, float)):
|
||||
return int(exp)
|
||||
return None
|
||||
|
||||
def _decode_jwt_claims(self, token: str) -> JsonObject:
|
||||
try:
|
||||
parts: Final = token.split(".")
|
||||
if len(parts) < 2:
|
||||
return {}
|
||||
payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4)
|
||||
payload_bytes: Final = base64.urlsafe_b64decode(payload_b64)
|
||||
return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8")))
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
def _extract_account_id(self, token: str | None) -> str | None:
|
||||
if not token:
|
||||
return None
|
||||
claims: Final = self._decode_jwt_claims(token)
|
||||
auth_claims: Final = claims.get(OPENAI_AUTH_CLAIM_KEY)
|
||||
if isinstance(auth_claims, dict):
|
||||
account_id: Final = auth_claims.get("chatgpt_account_id")
|
||||
if isinstance(account_id, str) and account_id:
|
||||
return account_id
|
||||
return None
|
||||
|
||||
def _login_device_code(self) -> dict[str, str]:
|
||||
cooldown_remaining: Final = self._get_device_code_cooldown_remaining(self._read_auth_file())
|
||||
if cooldown_remaining > 0:
|
||||
|
|
@ -361,15 +363,16 @@ class Authenticator:
|
|||
"refresh_token": _optional_str(data.get("refresh_token")) or refresh_token,
|
||||
"id_token": id_token,
|
||||
}
|
||||
account_id: Final = self.get_account_id()
|
||||
auth_data: Final = self._build_auth_record(refreshed)
|
||||
self._write_auth_file(auth_data)
|
||||
self._write_auth_file({**auth_data, "account_id": account_id} if account_id else auth_data)
|
||||
return refreshed
|
||||
|
||||
def _build_auth_record(self, tokens: dict[str, str]) -> JsonObject:
|
||||
access_token: Final = tokens.get("access_token")
|
||||
id_token: Final = tokens.get("id_token")
|
||||
expires_at: Final = self._get_expires_at(access_token) if access_token else None
|
||||
account_id: Final = self._extract_account_id(id_token or access_token)
|
||||
account_id: Final = extract_chatgpt_account_id(id_token or access_token)
|
||||
return {
|
||||
"access_token": access_token,
|
||||
"refresh_token": tokens.get("refresh_token"),
|
||||
|
|
|
|||
|
|
@ -1,14 +1,20 @@
|
|||
from collections.abc import Mapping
|
||||
from typing import Any, Final
|
||||
|
||||
from litellm.exceptions import AuthenticationError
|
||||
from litellm.llms.openai.openai import OpenAIConfig
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from ..authenticator import Authenticator
|
||||
from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
|
||||
from ..common_utils import (
|
||||
GetAccessTokenError,
|
||||
chatgpt_client_api_base,
|
||||
ensure_chatgpt_session_id,
|
||||
get_chatgpt_client_credential,
|
||||
get_chatgpt_default_headers,
|
||||
get_chatgpt_request_credential,
|
||||
strip_chatgpt_client_credential_headers,
|
||||
)
|
||||
from .streaming_utils import ChatGPTToolCallNormalizer
|
||||
|
||||
|
|
@ -19,9 +25,27 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
custom_llm_provider: str = "openai",
|
||||
authenticator: Authenticator | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.authenticator = Authenticator()
|
||||
self.authenticator = authenticator if authenticator is not None else Authenticator()
|
||||
|
||||
def _resolve_authenticator(
|
||||
self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None
|
||||
) -> Authenticator:
|
||||
auth_file: Final = get_chatgpt_auth_file(litellm_params)
|
||||
return get_cached_authenticator(auth_file) if auth_file else self.authenticator
|
||||
|
||||
@staticmethod
|
||||
def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str:
|
||||
try:
|
||||
return authenticator.get_access_token()
|
||||
except GetAccessTokenError as e:
|
||||
raise AuthenticationError(
|
||||
model=model,
|
||||
llm_provider=llm_provider,
|
||||
message=str(e),
|
||||
)
|
||||
|
||||
def api_base_without_login(self) -> str:
|
||||
return self.authenticator.get_api_base()
|
||||
|
|
@ -32,16 +56,16 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
custom_llm_provider: str,
|
||||
litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None,
|
||||
) -> tuple[str | None, str | None, str]:
|
||||
dynamic_api_base: Final = self.api_base_without_login()
|
||||
try:
|
||||
dynamic_api_key: Final = self.authenticator.get_access_token()
|
||||
except GetAccessTokenError as e:
|
||||
raise AuthenticationError(
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
message=str(e),
|
||||
)
|
||||
client_credential: Final = get_chatgpt_request_credential(litellm_params)
|
||||
if client_credential is not None:
|
||||
return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider
|
||||
if api_key:
|
||||
return api_base or self.api_base_without_login(), api_key, custom_llm_provider
|
||||
authenticator: Final = self._resolve_authenticator(litellm_params)
|
||||
dynamic_api_base: Final = authenticator.get_api_base()
|
||||
dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider)
|
||||
return dynamic_api_base, dynamic_api_key, custom_llm_provider
|
||||
|
||||
def get_openai_compatible_provider_info(
|
||||
|
|
@ -50,26 +74,49 @@ class ChatGPTConfig(OpenAIConfig):
|
|||
api_base: str | None,
|
||||
api_key: str | None,
|
||||
custom_llm_provider: str,
|
||||
litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None,
|
||||
) -> tuple[str | None, str | None, str]:
|
||||
return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider)
|
||||
return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider, litellm_params)
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
messages: list[AllMessageValues],
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
litellm_params: dict[str, object],
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
client_credential: Final = get_chatgpt_client_credential(headers)
|
||||
if client_credential is not None:
|
||||
return dict(
|
||||
get_chatgpt_default_headers(
|
||||
client_credential.access_token,
|
||||
client_credential.account_id,
|
||||
ensure_chatgpt_session_id(litellm_params),
|
||||
),
|
||||
**strip_chatgpt_client_credential_headers(headers),
|
||||
)
|
||||
authenticator: Final = self._resolve_authenticator(litellm_params)
|
||||
resolved_api_key: Final = (
|
||||
self._get_access_token_or_raise(authenticator, model, "chatgpt")
|
||||
if get_chatgpt_auth_file(litellm_params)
|
||||
else api_key
|
||||
)
|
||||
validated_headers: Final = super().validate_environment(
|
||||
headers, model, messages, optional_params, litellm_params, api_key, api_base
|
||||
dict(strip_chatgpt_client_credential_headers(headers)),
|
||||
model,
|
||||
messages,
|
||||
optional_params,
|
||||
litellm_params,
|
||||
resolved_api_key,
|
||||
api_base,
|
||||
)
|
||||
|
||||
account_id: Final = self.authenticator.get_account_id()
|
||||
account_id: Final = authenticator.get_account_id(resolved_api_key)
|
||||
session_id: Final = ensure_chatgpt_session_id(litellm_params)
|
||||
default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id)
|
||||
default_headers: Final = get_chatgpt_default_headers(resolved_api_key or "", account_id, session_id)
|
||||
return {**default_headers, **validated_headers}
|
||||
|
||||
def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer:
|
||||
|
|
|
|||
|
|
@ -2,14 +2,32 @@
|
|||
Constants and helpers for ChatGPT subscription OAuth.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
from typing import Any, Final
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeAlias
|
||||
from urllib.parse import urlsplit
|
||||
from uuid import uuid4
|
||||
|
||||
import httpx
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils
|
||||
from litellm.llms.base_llm.chat.transformation import BaseLLMException
|
||||
from litellm.types.utils import ProviderSpecificHeader
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter(
|
||||
ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None
|
||||
)
|
||||
_STRING_HEADERS: Final = TypeAdapter(Mapping[str, str])
|
||||
_PARAMS: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
# OAuth + API constants (derived from openai/codex)
|
||||
CHATGPT_AUTH_BASE: Final = "https://auth.openai.com"
|
||||
|
|
@ -19,6 +37,12 @@ CHATGPT_OAUTH_TOKEN_URL: Final = f"{CHATGPT_AUTH_BASE}/oauth/token"
|
|||
CHATGPT_DEVICE_VERIFY_URL: Final = f"{CHATGPT_AUTH_BASE}/codex/device"
|
||||
CHATGPT_API_BASE: Final = "https://chatgpt.com/backend-api/codex"
|
||||
CHATGPT_CLIENT_ID: Final = "app_EMoamEEZ73f0CkXaXp7hrann"
|
||||
CHATGPT_ACCOUNT_ID_HEADER: Final = "ChatGPT-Account-Id"
|
||||
OPENAI_AUTH_CLAIM_KEY: Final = "https://api.openai.com/auth"
|
||||
|
||||
JsonObject: TypeAlias = Mapping[str, JsonValue]
|
||||
_JSON_OBJECT_ADAPTER: Final = TypeAdapter(JsonObject)
|
||||
_EMPTY_CLAIMS: Final[JsonObject] = MappingProxyType({})
|
||||
|
||||
DEFAULT_ORIGINATOR: Final = "codex_cli_rs"
|
||||
DEFAULT_USER_AGENT: Final = "codex_cli_rs/0.0.0 (Unknown 0; unknown) unknown"
|
||||
|
|
@ -229,7 +253,7 @@ def get_chatgpt_default_headers(
|
|||
access_token: str,
|
||||
account_id: str | None,
|
||||
session_id: str | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
originator: Final = get_chatgpt_originator()
|
||||
user_agent: Final = get_chatgpt_user_agent(originator)
|
||||
headers: Final = {
|
||||
|
|
@ -242,30 +266,133 @@ def get_chatgpt_default_headers(
|
|||
if session_id:
|
||||
headers["session_id"] = session_id
|
||||
if account_id:
|
||||
headers["ChatGPT-Account-Id"] = account_id
|
||||
headers[CHATGPT_ACCOUNT_ID_HEADER] = account_id
|
||||
return headers
|
||||
|
||||
|
||||
def decode_jwt_claims(token: str) -> JsonObject:
|
||||
try:
|
||||
parts: Final = token.split(".")
|
||||
if len(parts) < 2:
|
||||
return _EMPTY_CLAIMS
|
||||
payload_b64: Final = parts[1] + "=" * (-len(parts[1]) % 4)
|
||||
payload_bytes: Final = base64.urlsafe_b64decode(payload_b64)
|
||||
return _JSON_OBJECT_ADAPTER.validate_python(json.loads(payload_bytes.decode("utf-8")))
|
||||
except Exception:
|
||||
return _EMPTY_CLAIMS
|
||||
|
||||
|
||||
def extract_chatgpt_account_id(token: str | None) -> str | None:
|
||||
if not token:
|
||||
return None
|
||||
auth_claims: Final = decode_jwt_claims(token).get(OPENAI_AUTH_CLAIM_KEY)
|
||||
if not isinstance(auth_claims, dict):
|
||||
return None
|
||||
account_id: Final = auth_claims.get("chatgpt_account_id")
|
||||
return account_id if isinstance(account_id, str) and account_id else None
|
||||
|
||||
|
||||
def _bearer_token(value: str) -> str:
|
||||
scheme, separator, token = value.strip().partition(" ")
|
||||
if not separator:
|
||||
return scheme
|
||||
return token.strip() if scheme.lower() == "bearer" else ""
|
||||
|
||||
|
||||
def is_chatgpt_oauth_key(value: str | None) -> bool:
|
||||
if value is None:
|
||||
return False
|
||||
return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(_bearer_token(value))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ChatGPTClientCredential:
|
||||
access_token: str
|
||||
account_id: str | None
|
||||
|
||||
|
||||
def _header_value(headers: Mapping[str, str], name: str) -> str | None:
|
||||
return next((value for key, value in headers.items() if key.lower() == name), None)
|
||||
|
||||
|
||||
def get_chatgpt_client_credential(headers: Mapping[str, str]) -> ChatGPTClientCredential | None:
|
||||
auth_header: Final = _header_value(headers, "authorization")
|
||||
if auth_header is None or not is_chatgpt_oauth_key(auth_header):
|
||||
return None
|
||||
access_token: Final = _bearer_token(auth_header)
|
||||
account_id: Final = _header_value(headers, CHATGPT_ACCOUNT_ID_HEADER.lower()) or extract_chatgpt_account_id(
|
||||
access_token
|
||||
)
|
||||
return ChatGPTClientCredential(access_token=access_token, account_id=account_id)
|
||||
|
||||
|
||||
def select_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
if get_chatgpt_client_credential(headers) is None:
|
||||
return MappingProxyType({})
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in headers.items()
|
||||
if (key.lower() == "authorization" and is_chatgpt_oauth_key(value))
|
||||
or key.lower() == CHATGPT_ACCOUNT_ID_HEADER.lower()
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def strip_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
return MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in headers.items()
|
||||
if key.lower() not in ("authorization", CHATGPT_ACCOUNT_ID_HEADER.lower())
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_chatgpt_request_credential(
|
||||
params: "Mapping[str, object] | GenericLiteLLMParams | None",
|
||||
) -> ChatGPTClientCredential | None:
|
||||
normalized: Final = _normalize_litellm_params(params)
|
||||
scoped: Final = normalized.get("provider_specific_header")
|
||||
extra: Final = normalized.get("extra_headers")
|
||||
headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers(
|
||||
provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt"
|
||||
)
|
||||
return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential(
|
||||
_STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({})
|
||||
)
|
||||
|
||||
|
||||
def chatgpt_client_api_base(api_base: str | None) -> str:
|
||||
base: Final = api_base or CHATGPT_API_BASE
|
||||
parsed: Final = urlsplit(base)
|
||||
if (
|
||||
parsed.scheme != "https"
|
||||
or parsed.hostname != "chatgpt.com"
|
||||
or parsed.port not in (None, 443)
|
||||
or parsed.username
|
||||
or parsed.password
|
||||
):
|
||||
raise ChatGPTAuthError(
|
||||
status_code=400, message="Client ChatGPT credentials require the official HTTPS ChatGPT endpoint"
|
||||
)
|
||||
return base
|
||||
|
||||
|
||||
def get_chatgpt_default_instructions() -> str:
|
||||
return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS
|
||||
|
||||
|
||||
def _normalize_litellm_params(litellm_params: Any | None) -> dict:
|
||||
if litellm_params is None:
|
||||
return {}
|
||||
if isinstance(litellm_params, dict):
|
||||
return litellm_params
|
||||
if hasattr(litellm_params, "model_dump"):
|
||||
try:
|
||||
return litellm_params.model_dump()
|
||||
except Exception:
|
||||
return {}
|
||||
if hasattr(litellm_params, "dict"):
|
||||
try:
|
||||
return litellm_params.dict()
|
||||
except Exception:
|
||||
return {}
|
||||
return {}
|
||||
def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]:
|
||||
if isinstance(litellm_params, Mapping):
|
||||
return _PARAMS.validate_python(litellm_params)
|
||||
serializer: Final = getattr(litellm_params, "model_dump", None) or getattr(litellm_params, "dict", None)
|
||||
if not callable(serializer):
|
||||
return MappingProxyType({})
|
||||
try:
|
||||
return _PARAMS.validate_python(serializer())
|
||||
except Exception:
|
||||
return MappingProxyType({})
|
||||
|
||||
|
||||
def get_chatgpt_session_id(litellm_params: object) -> str | None:
|
||||
|
|
@ -274,8 +401,8 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None:
|
|||
value = params.get(key)
|
||||
if value:
|
||||
return str(value)
|
||||
metadata: Final = params.get("metadata")
|
||||
if isinstance(metadata, dict):
|
||||
metadata: Final = _normalize_litellm_params(params.get("metadata"))
|
||||
if metadata:
|
||||
value = metadata.get("session_id")
|
||||
if value:
|
||||
return str(value)
|
||||
|
|
|
|||
|
|
@ -25,13 +25,15 @@ from litellm.types.llms.openai import (
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
from ..authenticator import Authenticator
|
||||
from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file
|
||||
from ..common_utils import (
|
||||
CHATGPT_API_BASE,
|
||||
GetAccessTokenError,
|
||||
ensure_chatgpt_session_id,
|
||||
get_chatgpt_client_credential,
|
||||
get_chatgpt_default_headers,
|
||||
get_chatgpt_default_instructions,
|
||||
strip_chatgpt_client_credential_headers,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -49,14 +51,35 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
def custom_llm_provider(self) -> LlmProviders:
|
||||
return LlmProviders.CHATGPT
|
||||
|
||||
def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator:
|
||||
auth_file: Final = get_chatgpt_auth_file(litellm_params)
|
||||
return get_cached_authenticator(auth_file) if auth_file else self.authenticator
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: dict,
|
||||
headers: dict[str, str],
|
||||
model: str,
|
||||
litellm_params: GenericLiteLLMParams | None,
|
||||
) -> dict:
|
||||
) -> dict[str, str]:
|
||||
session_id: Final = ensure_chatgpt_session_id(litellm_params)
|
||||
client_credential: Final = get_chatgpt_client_credential(headers)
|
||||
if client_credential is not None:
|
||||
return {
|
||||
**get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id),
|
||||
**strip_chatgpt_client_credential_headers(headers),
|
||||
}
|
||||
authenticator: Final = self._resolve_authenticator(litellm_params)
|
||||
safe_headers: Final = strip_chatgpt_client_credential_headers(headers)
|
||||
configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None
|
||||
if configured_api_key and not get_chatgpt_auth_file(litellm_params):
|
||||
return {
|
||||
**get_chatgpt_default_headers(
|
||||
configured_api_key, authenticator.get_account_id(configured_api_key), session_id
|
||||
),
|
||||
**safe_headers,
|
||||
}
|
||||
try:
|
||||
access_token: Final = self.authenticator.get_access_token()
|
||||
access_token: Final = authenticator.get_access_token()
|
||||
except GetAccessTokenError as e:
|
||||
raise AuthenticationError(
|
||||
model=model,
|
||||
|
|
@ -64,10 +87,9 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
message=str(e),
|
||||
)
|
||||
|
||||
account_id: Final = self.authenticator.get_account_id()
|
||||
session_id: Final = ensure_chatgpt_session_id(litellm_params)
|
||||
account_id: Final = authenticator.get_account_id()
|
||||
default_headers: Final = get_chatgpt_default_headers(access_token, account_id, session_id)
|
||||
return {**default_headers, **headers}
|
||||
return {**default_headers, **safe_headers}
|
||||
|
||||
def transform_responses_api_request(
|
||||
self,
|
||||
|
|
@ -79,7 +101,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
) -> dict:
|
||||
request: Final = super().transform_responses_api_request(
|
||||
model,
|
||||
input,
|
||||
[{"role": "user", "content": input}] if isinstance(input, str) else input,
|
||||
response_api_optional_request_params,
|
||||
litellm_params,
|
||||
headers,
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ from aiohttp import ClientSession, DummyCookieJar, TCPConnector
|
|||
from httpx import USE_CLIENT_DEFAULT, AsyncHTTPTransport, HTTPTransport
|
||||
from httpx._types import CertTypes, RequestFiles
|
||||
from httpx._utils import get_environment_proxies
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -566,6 +567,34 @@ async def _raise_masked_async_error(e: httpx.HTTPStatusError, stream: bool) -> N
|
|||
raise MaskedHTTPStatusError(e, message=_text, text=_text) from None
|
||||
|
||||
|
||||
_TIMEOUT_PHASES: Final[Mapping[type[httpx.TimeoutException], tuple[str, str]]] = MappingProxyType(
|
||||
{
|
||||
httpx.ConnectTimeout: ("Connection", "connect"),
|
||||
httpx.ReadTimeout: ("Read", "read"),
|
||||
httpx.WriteTimeout: ("Write", "write"),
|
||||
httpx.PoolTimeout: ("Connection pool", "pool"),
|
||||
}
|
||||
)
|
||||
_UNKNOWN_TIMEOUT_PHASE: Final = ("Request", "read")
|
||||
_TIMEOUT_EXTENSION: Final = TypeAdapter(dict[str, float | None])
|
||||
|
||||
|
||||
def _timeout_phase_seconds(error: httpx.TimeoutException, phase: str) -> float | None:
|
||||
"""The limit httpx enforced for ``phase`` on the request that raised ``error``, from the request itself."""
|
||||
return _TIMEOUT_EXTENSION.validate_python(error.request.extensions["timeout"]).get(phase)
|
||||
|
||||
|
||||
def _raise_timeout(error: httpx.TimeoutException, started_at: float) -> NoReturn:
|
||||
label, phase = _TIMEOUT_PHASES.get(type(error), _UNKNOWN_TIMEOUT_PHASE)
|
||||
seconds: Final = _timeout_phase_seconds(error, phase)
|
||||
time_taken: Final = round(time.time() - started_at, 3)
|
||||
raise litellm.Timeout(
|
||||
message=f"{label} timed out. Timeout passed={seconds}, time taken={time_taken} seconds",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
|
||||
|
||||
class HTTPResponseLimitError(ValueError):
|
||||
pass
|
||||
|
||||
|
|
@ -870,20 +899,7 @@ class AsyncHTTPHandler:
|
|||
finally:
|
||||
await new_client.aclose()
|
||||
except httpx.TimeoutException as e:
|
||||
end_time: Final = time.time()
|
||||
time_delta: Final = round(end_time - start_time, 3)
|
||||
headers = {}
|
||||
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
|
||||
if error_response is not None:
|
||||
for key, value in error_response.headers.items():
|
||||
headers[f"response_headers-{key}"] = value
|
||||
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out. Timeout passed={timeout}, time taken={time_delta} seconds",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
headers=headers,
|
||||
)
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -902,6 +918,7 @@ class AsyncHTTPHandler:
|
|||
follow_redirects: bool | None = None,
|
||||
):
|
||||
_follow_redirects: Final = follow_redirects if follow_redirects is not None else USE_CLIENT_DEFAULT
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
|
|
@ -946,18 +963,7 @@ class AsyncHTTPHandler:
|
|||
finally:
|
||||
await new_client.aclose()
|
||||
except httpx.TimeoutException as e:
|
||||
headers = {}
|
||||
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
|
||||
if error_response is not None:
|
||||
for key, value in error_response.headers.items():
|
||||
headers[f"response_headers-{key}"] = value
|
||||
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
headers=headers,
|
||||
)
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -974,6 +980,7 @@ class AsyncHTTPHandler:
|
|||
stream: bool = False,
|
||||
content: _RequestContent | None = None,
|
||||
):
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
if timeout is None:
|
||||
timeout = self.timeout
|
||||
|
|
@ -1012,18 +1019,7 @@ class AsyncHTTPHandler:
|
|||
finally:
|
||||
await new_client.aclose()
|
||||
except httpx.TimeoutException as e:
|
||||
headers = {}
|
||||
error_response: Final[httpx.Response | None] = getattr(e, "response", None)
|
||||
if error_response is not None:
|
||||
for key, value in error_response.headers.items():
|
||||
headers[f"response_headers-{key}"] = value
|
||||
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
headers=headers,
|
||||
)
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
await _raise_masked_async_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -1514,6 +1510,7 @@ class HTTPHandler:
|
|||
content: _RequestContent | None = None,
|
||||
logging_obj: LiteLLMLoggingObject | None = None,
|
||||
):
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
|
||||
request_data, request_content = _prepare_request_data_and_content(data, content)
|
||||
|
|
@ -1546,12 +1543,8 @@ class HTTPHandler:
|
|||
_anchor_handler_to(response, self)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.TimeoutException:
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.TimeoutException as e:
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -1568,6 +1561,7 @@ class HTTPHandler:
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
content: _RequestContent | None = None,
|
||||
):
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
|
||||
request_data, request_content = _prepare_request_data_and_content(data, content)
|
||||
|
|
@ -1598,12 +1592,8 @@ class HTTPHandler:
|
|||
_anchor_handler_to(response, self)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.TimeoutException:
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.TimeoutException as e:
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -1620,6 +1610,7 @@ class HTTPHandler:
|
|||
timeout: float | httpx.Timeout | None = None,
|
||||
content: _RequestContent | None = None,
|
||||
):
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
|
||||
request_data, request_content = _prepare_request_data_and_content(data, content)
|
||||
|
|
@ -1649,12 +1640,8 @@ class HTTPHandler:
|
|||
if stream:
|
||||
_anchor_handler_to(response, self)
|
||||
return response
|
||||
except httpx.TimeoutException:
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.TimeoutException as e:
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
@ -1671,6 +1658,7 @@ class HTTPHandler:
|
|||
stream: bool = False,
|
||||
content: _RequestContent | None = None,
|
||||
):
|
||||
start_time: Final = time.time()
|
||||
try:
|
||||
# Prepare data/content parameters to prevent httpx DeprecationWarning (memory leak fix)
|
||||
request_data, request_content = _prepare_request_data_and_content(data, content)
|
||||
|
|
@ -1701,12 +1689,8 @@ class HTTPHandler:
|
|||
_anchor_handler_to(response, self)
|
||||
response.raise_for_status()
|
||||
return response
|
||||
except httpx.TimeoutException:
|
||||
raise litellm.Timeout(
|
||||
message=f"Connection timed out after {timeout} seconds.",
|
||||
model="default-model-name",
|
||||
llm_provider="litellm-httpx-handler",
|
||||
)
|
||||
except httpx.TimeoutException as e:
|
||||
_raise_timeout(e, start_time)
|
||||
except httpx.HTTPStatusError as e:
|
||||
_raise_masked_sync_error(e, stream)
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -198,6 +198,26 @@ from litellm.utils import (
|
|||
)
|
||||
|
||||
|
||||
def _merge_headers_case_insensitive(
|
||||
headers: Mapping[str, object], extra_headers: Mapping[str, object] | None
|
||||
) -> dict[str, str]: # mutable-ok: legacy HTTP handler methods require a dict of headers
|
||||
overridden: Final = frozenset(name.lower() for name in extra_headers or {})
|
||||
return {
|
||||
**{name: str(value) for name, value in headers.items() if name.lower() not in overridden},
|
||||
**{name: str(value) for name, value in (extra_headers or {}).items()},
|
||||
}
|
||||
|
||||
|
||||
def _client_request_headers(
|
||||
response_api_optional_request_params: Mapping[str, object],
|
||||
extra_headers: Mapping[str, object] | None,
|
||||
) -> dict[str, str]: # mutable-ok: provider validate_environment methods require a dict
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
|
||||
optional_headers: Final = as_str_mapping(response_api_optional_request_params.get("extra_headers"))
|
||||
return _merge_headers_case_insensitive(optional_headers or {}, extra_headers)
|
||||
|
||||
|
||||
def _rust_responses_websocket_enabled(
|
||||
custom_llm_provider: str | None,
|
||||
) -> bool:
|
||||
|
|
@ -2906,14 +2926,18 @@ class BaseLLMHTTPHandler:
|
|||
else:
|
||||
sync_httpx_client = client
|
||||
|
||||
headers = responses_api_provider_config.validate_environment(
|
||||
headers=response_api_optional_request_params.get("extra_headers", {}) or {},
|
||||
model=model,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
if extra_headers:
|
||||
headers.update(extra_headers)
|
||||
headers = { # rebind-ok: validation replaces the method's incoming headers
|
||||
**responses_api_provider_config.validate_environment(
|
||||
headers=(
|
||||
_client_request_headers(response_api_optional_request_params, extra_headers)
|
||||
if custom_llm_provider == "chatgpt"
|
||||
else response_api_optional_request_params.get("extra_headers", {}) or {}
|
||||
),
|
||||
model=model,
|
||||
litellm_params=litellm_params,
|
||||
),
|
||||
**((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}),
|
||||
}
|
||||
|
||||
# Check if streaming is requested
|
||||
stream = response_api_optional_request_params.get("stream", False)
|
||||
|
|
@ -3104,11 +3128,15 @@ class BaseLLMHTTPHandler:
|
|||
validated_headers: Final = {
|
||||
**await asyncio.to_thread(
|
||||
responses_api_provider_config.validate_environment,
|
||||
headers=response_api_optional_request_params.get("extra_headers", {}) or {},
|
||||
headers=(
|
||||
_client_request_headers(response_api_optional_request_params, extra_headers)
|
||||
if custom_llm_provider == "chatgpt"
|
||||
else response_api_optional_request_params.get("extra_headers", {}) or {}
|
||||
),
|
||||
model=model,
|
||||
litellm_params=litellm_params,
|
||||
),
|
||||
**(extra_headers or {}),
|
||||
**((extra_headers or {}) if custom_llm_provider != "chatgpt" else {}),
|
||||
}
|
||||
|
||||
# Check if streaming is requested
|
||||
|
|
|
|||
|
|
@ -56,9 +56,9 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
|
|||
# Response format mappings from OpenAI to ElevenLabs
|
||||
FORMAT_MAPPINGS = {
|
||||
"mp3": "mp3_44100_128",
|
||||
"pcm": "pcm_44100",
|
||||
"pcm": "pcm_24000",
|
||||
"opus": "opus_48000_128",
|
||||
# ElevenLabs does not support WAV, AAC, or FLAC formats.
|
||||
"wav": "wav_24000",
|
||||
}
|
||||
|
||||
ELEVENLABS_QUERY_PARAMS_KEY = "__elevenlabs_query_params__"
|
||||
|
|
|
|||
|
|
@ -90,7 +90,7 @@ class PassThroughEndpointHandler(BaseTranslation):
|
|||
payload_to_check: Final = {
|
||||
k: v
|
||||
for k, v in without_server_streaming_classification(data).items()
|
||||
if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj")
|
||||
if not k.startswith("_") and k not in ("metadata", "litellm_metadata", "litellm_logging_obj")
|
||||
}
|
||||
verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail")
|
||||
return safe_dumps(payload_to_check)
|
||||
|
|
|
|||
|
|
@ -726,6 +726,9 @@ def gemini_convert_messages_with_history(
|
|||
from .vertex_and_google_ai_studio_gemini import VertexGeminiConfig
|
||||
|
||||
forward_function_call_id: Final = VertexGeminiConfig.forward_gemini_function_call_id(model or "")
|
||||
supports_multimodal_function_responses: Final = model is None or (
|
||||
VertexGeminiConfig.supports_multimodal_function_responses(model)
|
||||
)
|
||||
|
||||
try:
|
||||
while msg_i < len(messages):
|
||||
|
|
@ -1029,6 +1032,7 @@ def gemini_convert_messages_with_history(
|
|||
messages[msg_i],
|
||||
last_message_with_tool_calls,
|
||||
forward_function_call_id=forward_function_call_id,
|
||||
supports_multimodal_function_responses=supports_multimodal_function_responses,
|
||||
)
|
||||
msg_i += 1
|
||||
# Handle both single part and list of parts (for Computer Use with images)
|
||||
|
|
|
|||
|
|
@ -689,6 +689,10 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
|
|||
) -> bool:
|
||||
return cls._forward_gemini_function_call_id(model)
|
||||
|
||||
@classmethod
|
||||
def supports_multimodal_function_responses(cls, model: str) -> bool:
|
||||
return cls.is_gemini_3_or_newer(model)
|
||||
|
||||
def _supports_penalty_parameters(self, model: str) -> bool:
|
||||
# Gemini 3 models do not support penalty parameters
|
||||
if VertexGeminiConfig._is_gemini_3_or_newer(model):
|
||||
|
|
|
|||
|
|
@ -7,12 +7,14 @@ Handles Authentication and generating request urls for Vertex AI and Google AI S
|
|||
import asyncio
|
||||
import json
|
||||
import threading
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_MAX_LRU_CACHE_SIZE
|
||||
from litellm.litellm_core_utils.asyncify import asyncify
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
|
@ -70,16 +72,20 @@ class _VertexCredentialsObject(Protocol):
|
|||
def refresh(self, request: object) -> None: ...
|
||||
|
||||
|
||||
_VertexCredentialCacheKey = tuple[str | None, str | None]
|
||||
|
||||
|
||||
class VertexBase:
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.access_token: str | None = None
|
||||
self.refresh_token: str | None = None
|
||||
self._credentials: GoogleCredentialsObject | None = None
|
||||
self._credentials_project_mapping: dict[
|
||||
tuple[VERTEX_CREDENTIALS_TYPES | None, str | None],
|
||||
self._credentials_project_mapping: OrderedDict[
|
||||
_VertexCredentialCacheKey,
|
||||
tuple[_VertexCredentialsObject, str | None],
|
||||
] = {}
|
||||
] = OrderedDict()
|
||||
self._credentials_cache_lock = threading.RLock()
|
||||
self.project_id: str | None = None
|
||||
self.async_handler: AsyncHTTPHandler | None = None
|
||||
# Per-credential-key asyncio.Lock for single-flight async refresh.
|
||||
|
|
@ -91,10 +97,10 @@ class VertexBase:
|
|||
# each lock; the entry is pruned when the count reaches zero, so the
|
||||
# dict stays bounded even in long-running high-cardinality deployments
|
||||
# without depending on any private asyncio internals.
|
||||
self._async_refresh_locks: dict[tuple, asyncio.Lock] = {}
|
||||
self._async_refresh_lock_refcounts: dict[tuple, int] = {}
|
||||
self._async_refresh_locks: dict[_VertexCredentialCacheKey, asyncio.Lock] = {}
|
||||
self._async_refresh_lock_refcounts: dict[_VertexCredentialCacheKey, int] = {}
|
||||
# Tracks in-flight background refresh tasks to avoid duplicate refreshes.
|
||||
self._background_refresh_tasks: dict[tuple, asyncio.Task] = {}
|
||||
self._background_refresh_tasks: dict[_VertexCredentialCacheKey, asyncio.Task] = {}
|
||||
# Protects the sync get_access_token refresh path.
|
||||
# Use RLock so that the reauthentication retry path (which calls
|
||||
# back into get_access_token while still holding the lock) can
|
||||
|
|
@ -378,7 +384,7 @@ class VertexBase:
|
|||
with self._sync_refresh_lock:
|
||||
credentials.refresh(Request())
|
||||
|
||||
def _acquire_async_refresh_lock(self, credential_cache_key: tuple) -> asyncio.Lock:
|
||||
def _acquire_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey) -> asyncio.Lock:
|
||||
"""Increment the refcount and return the lock for ``credential_cache_key``.
|
||||
|
||||
Every call must be paired with ``_release_async_refresh_lock`` once the
|
||||
|
|
@ -391,7 +397,7 @@ class VertexBase:
|
|||
)
|
||||
return lock
|
||||
|
||||
def _release_async_refresh_lock(self, credential_cache_key: tuple, lock: asyncio.Lock) -> None:
|
||||
def _release_async_refresh_lock(self, credential_cache_key: _VertexCredentialCacheKey, lock: asyncio.Lock) -> None:
|
||||
"""Decrement the refcount and drop the lock entry when it reaches zero.
|
||||
|
||||
Must be called only after the caller has released ``lock`` (i.e. once
|
||||
|
|
@ -409,7 +415,7 @@ class VertexBase:
|
|||
|
||||
def _try_get_cached_token(
|
||||
self,
|
||||
credential_cache_key: tuple,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
project_id: str | None,
|
||||
) -> tuple[str, str] | None:
|
||||
"""
|
||||
|
|
@ -432,7 +438,7 @@ class VertexBase:
|
|||
|
||||
def _try_get_usable_cached_token(
|
||||
self,
|
||||
credential_cache_key: tuple,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
project_id: str | None,
|
||||
) -> tuple[str, str, "TokenState", _VertexCredentialsObject, str | None] | None:
|
||||
"""
|
||||
|
|
@ -458,19 +464,37 @@ class VertexBase:
|
|||
return creds.token, resolved_project, token_state, creds, cached_project_id
|
||||
|
||||
def _unpack_cached_credentials(
|
||||
self, credential_cache_key: tuple
|
||||
self, credential_cache_key: _VertexCredentialCacheKey
|
||||
) -> tuple[_VertexCredentialsObject | None, str | None]:
|
||||
"""
|
||||
Return (credentials, project_id) from the cache, or (None, None) if
|
||||
not cached. Handles both tuple and legacy cache formats.
|
||||
"""
|
||||
if credential_cache_key not in self._credentials_project_mapping:
|
||||
return None, None
|
||||
cached_entry: Final = self._credentials_project_mapping[credential_cache_key]
|
||||
with self._credentials_cache_lock:
|
||||
cached_entry: Final = self._credentials_project_mapping.get(credential_cache_key)
|
||||
if cached_entry is None:
|
||||
return None, None
|
||||
self._credentials_project_mapping.move_to_end(credential_cache_key)
|
||||
if isinstance(cached_entry, tuple):
|
||||
return cached_entry
|
||||
return cached_entry, cached_entry.quota_project_id or getattr(cached_entry, "project_id", None)
|
||||
|
||||
def _cache_credentials(
|
||||
self,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
credentials: _VertexCredentialsObject,
|
||||
project_id: str | None,
|
||||
*,
|
||||
replace: bool = True,
|
||||
) -> None:
|
||||
with self._credentials_cache_lock:
|
||||
if not replace and credential_cache_key in self._credentials_project_mapping:
|
||||
return
|
||||
self._credentials_project_mapping[credential_cache_key] = (credentials, project_id)
|
||||
self._credentials_project_mapping.move_to_end(credential_cache_key)
|
||||
while len(self._credentials_project_mapping) > max(0, DEFAULT_MAX_LRU_CACHE_SIZE):
|
||||
self._credentials_project_mapping.popitem(last=False)
|
||||
|
||||
def _get_token_state(self, credentials: _VertexCredentialsObject) -> "TokenState":
|
||||
"""
|
||||
Return the token state using google-auth's TokenState enum.
|
||||
|
|
@ -494,7 +518,7 @@ class VertexBase:
|
|||
self,
|
||||
credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
project_id: str | None,
|
||||
credential_cache_key: tuple,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
) -> tuple[_VertexCredentialsObject, str | None]:
|
||||
"""Load credentials via load_auth (in thread) and cache the result."""
|
||||
try:
|
||||
|
|
@ -507,17 +531,12 @@ class VertexBase:
|
|||
raise
|
||||
if _credentials is None:
|
||||
raise ValueError("Could not resolve credentials")
|
||||
self._credentials_project_mapping[credential_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
self._cache_credentials(credential_cache_key, _credentials, credential_project_id)
|
||||
return _credentials, credential_project_id
|
||||
|
||||
async def _background_refresh_credentials(
|
||||
self,
|
||||
credentials: _VertexCredentialsObject,
|
||||
credential_cache_key: tuple,
|
||||
credential_project_id: str | None,
|
||||
) -> None:
|
||||
"""
|
||||
Refresh credentials in the background without blocking the calling request.
|
||||
|
|
@ -528,25 +547,13 @@ class VertexBase:
|
|||
try:
|
||||
verbose_logger.debug("Background proactive credential refresh")
|
||||
await asyncify(self.refresh_auth)(credentials)
|
||||
# Only update the cache if it still points at the credentials
|
||||
# object we just refreshed. The per-key async lock is not held
|
||||
# here, so a concurrent INVALID path may have already replaced
|
||||
# this entry (e.g. via _handle_reauthentication_async, which
|
||||
# creates a fresh credentials object). In that case our write
|
||||
# would clobber the newer entry with a stale reference.
|
||||
cached_creds, _ = self._unpack_cached_credentials(credential_cache_key)
|
||||
if cached_creds is credentials:
|
||||
self._credentials_project_mapping[credential_cache_key] = (
|
||||
credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
except Exception:
|
||||
verbose_logger.debug(
|
||||
"Background credential refresh failed, will retry on next request",
|
||||
exc_info=True,
|
||||
)
|
||||
|
||||
async def _await_in_flight_background_refresh(self, credential_cache_key: tuple) -> None:
|
||||
async def _await_in_flight_background_refresh(self, credential_cache_key: _VertexCredentialCacheKey) -> None:
|
||||
"""Wait for an in-flight background refresh to finish, if any.
|
||||
|
||||
google-auth's ``Credentials.refresh()`` is not safe to invoke
|
||||
|
|
@ -568,8 +575,7 @@ class VertexBase:
|
|||
def _schedule_background_refresh(
|
||||
self,
|
||||
credentials: _VertexCredentialsObject,
|
||||
credential_cache_key: tuple,
|
||||
credential_project_id: str | None,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
) -> None:
|
||||
"""Kick off a single background refresh for ``credential_cache_key``.
|
||||
|
||||
|
|
@ -581,9 +587,7 @@ class VertexBase:
|
|||
if existing is not None and not existing.done():
|
||||
return
|
||||
self._background_refresh_tasks.pop(credential_cache_key, None)
|
||||
task: Final = asyncio.create_task(
|
||||
self._background_refresh_credentials(credentials, credential_cache_key, credential_project_id)
|
||||
)
|
||||
task: Final = asyncio.create_task(self._background_refresh_credentials(credentials))
|
||||
|
||||
def _drop_background_refresh_task(_fut: asyncio.Future[None]) -> None:
|
||||
if self._background_refresh_tasks.get(credential_cache_key) is _fut:
|
||||
|
|
@ -796,7 +800,7 @@ class VertexBase:
|
|||
self,
|
||||
credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
project_id: str | None,
|
||||
credential_cache_key: tuple,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
error: Exception,
|
||||
) -> tuple[str, str]:
|
||||
"""
|
||||
|
|
@ -822,8 +826,8 @@ class VertexBase:
|
|||
)
|
||||
|
||||
# Clear the cached credentials
|
||||
if credential_cache_key in self._credentials_project_mapping:
|
||||
del self._credentials_project_mapping[credential_cache_key]
|
||||
with self._credentials_cache_lock:
|
||||
self._credentials_project_mapping.pop(credential_cache_key, None)
|
||||
|
||||
# Retry once with _retry_reauth=True to prevent infinite recursion
|
||||
try:
|
||||
|
|
@ -846,7 +850,7 @@ class VertexBase:
|
|||
self,
|
||||
credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
project_id: str | None,
|
||||
credential_cache_key: tuple,
|
||||
credential_cache_key: _VertexCredentialCacheKey,
|
||||
error: Exception,
|
||||
) -> tuple[str, str]:
|
||||
"""
|
||||
|
|
@ -856,7 +860,8 @@ class VertexBase:
|
|||
"Handling async reauthentication for project_id: %s. Clearing cache and retrying once.", project_id
|
||||
)
|
||||
|
||||
self._credentials_project_mapping.pop(credential_cache_key, None)
|
||||
with self._credentials_cache_lock:
|
||||
self._credentials_project_mapping.pop(credential_cache_key, None)
|
||||
|
||||
try:
|
||||
(
|
||||
|
|
@ -875,10 +880,7 @@ class VertexBase:
|
|||
# references the OLD credentials object we just replaced, and
|
||||
# leaving it would force the next request to do a redundant
|
||||
# refresh/reauth before realizing the cached creds are stale.
|
||||
self._credentials_project_mapping[resolved_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id)
|
||||
|
||||
if _credentials.token is None or not isinstance(_credentials.token, str):
|
||||
raise ValueError(
|
||||
|
|
@ -924,21 +926,11 @@ class VertexBase:
|
|||
# Convert dict credentials to string for caching
|
||||
cache_credentials: Final = json.dumps(credentials) if isinstance(credentials, dict) else credentials
|
||||
credential_cache_key: Final = (cache_credentials, project_id)
|
||||
_credentials: _VertexCredentialsObject | None = None
|
||||
|
||||
verbose_logger.debug("Checking cached credentials for project_id: %s", project_id)
|
||||
|
||||
if credential_cache_key in self._credentials_project_mapping:
|
||||
_credentials, credential_project_id = self._unpack_cached_credentials(credential_cache_key)
|
||||
if _credentials is not None:
|
||||
verbose_logger.debug("Cached credentials found for project_id: %s.", project_id)
|
||||
# Retrieve both credentials and cached project_id
|
||||
cached_entry: Final = self._credentials_project_mapping[credential_cache_key]
|
||||
verbose_logger.debug("cached_entry: %s", cached_entry)
|
||||
if isinstance(cached_entry, tuple):
|
||||
_credentials, credential_project_id = cached_entry
|
||||
else:
|
||||
# Backward compatibility with old cache format
|
||||
_credentials = cached_entry
|
||||
credential_project_id = _credentials.quota_project_id or getattr(_credentials, "project_id", None)
|
||||
verbose_logger.debug(
|
||||
"Using cached credentials for project_id: %s",
|
||||
credential_project_id,
|
||||
|
|
@ -963,10 +955,7 @@ class VertexBase:
|
|||
f"Could not resolve credentials - either dynamically or from environment, for project_id: {project_id}"
|
||||
)
|
||||
# Cache the project_id and credentials from load_auth result (resolved project_id)
|
||||
self._credentials_project_mapping[credential_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
self._cache_credentials(credential_cache_key, _credentials, credential_project_id)
|
||||
|
||||
## VALIDATE CREDENTIALS
|
||||
verbose_logger.debug("Validating credentials")
|
||||
|
|
@ -974,11 +963,7 @@ class VertexBase:
|
|||
project_id = credential_project_id
|
||||
# Update cache with resolved project_id for future lookups
|
||||
resolved_cache_key: Final = (cache_credentials, project_id)
|
||||
if resolved_cache_key not in self._credentials_project_mapping:
|
||||
self._credentials_project_mapping[resolved_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id, replace=False)
|
||||
|
||||
# Check if credentials are None before accessing attributes
|
||||
if _credentials is None:
|
||||
|
|
@ -991,10 +976,6 @@ class VertexBase:
|
|||
try:
|
||||
verbose_logger.debug("Credentials expired, refreshing")
|
||||
self.refresh_auth(_credentials)
|
||||
self._credentials_project_mapping[credential_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
except Exception as e:
|
||||
# if refresh fails, it's possible the user has re-authenticated via `gcloud auth application-default login`
|
||||
# in this case, we should try to reload the credentials by clearing the cache and retrying
|
||||
|
|
@ -1043,9 +1024,9 @@ class VertexBase:
|
|||
# callers on the lock just to schedule that refresh.
|
||||
usable: Final = self._try_get_usable_cached_token(credential_cache_key, project_id)
|
||||
if usable is not None:
|
||||
cached_token, resolved_project, token_state, creds, cached_project_id = usable
|
||||
cached_token, resolved_project, token_state, creds, _cached_project_id = usable
|
||||
if token_state == TokenState.STALE:
|
||||
self._schedule_background_refresh(creds, credential_cache_key, cached_project_id)
|
||||
self._schedule_background_refresh(creds, credential_cache_key)
|
||||
return cached_token, resolved_project
|
||||
|
||||
# === SLOW PATH (per-key lock) ===
|
||||
|
|
@ -1075,10 +1056,7 @@ class VertexBase:
|
|||
# reauth that only repopulated the unresolved key), which
|
||||
# would force the next request through an unnecessary
|
||||
# refresh/reauth cycle.
|
||||
self._credentials_project_mapping[resolved_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
self._cache_credentials(resolved_cache_key, _credentials, credential_project_id)
|
||||
|
||||
# Use google-auth's token_state to decide refresh strategy:
|
||||
# - STALE: token is usable but within REFRESH_THRESHOLD (3:45) of
|
||||
|
|
@ -1098,7 +1076,6 @@ class VertexBase:
|
|||
self._schedule_background_refresh(
|
||||
_credentials,
|
||||
credential_cache_key,
|
||||
credential_project_id,
|
||||
)
|
||||
return current_token, project_id
|
||||
|
||||
|
|
@ -1117,10 +1094,6 @@ class VertexBase:
|
|||
try:
|
||||
verbose_logger.debug("Credentials expired, refreshing")
|
||||
await asyncify(self.refresh_auth)(_credentials)
|
||||
self._credentials_project_mapping[credential_cache_key] = (
|
||||
_credentials,
|
||||
credential_project_id,
|
||||
)
|
||||
except Exception as e:
|
||||
if "Reauthentication is needed" in str(e):
|
||||
verbose_logger.debug("Reauthentication needed, clearing cache and retrying")
|
||||
|
|
|
|||
|
|
@ -677,7 +677,11 @@ async def acompletion(
|
|||
}
|
||||
if custom_llm_provider is None:
|
||||
_supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects
|
||||
"dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs}
|
||||
"dict[str, object]",
|
||||
{
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
},
|
||||
)
|
||||
_, custom_llm_provider, _, _ = get_llm_provider(
|
||||
model=model,
|
||||
|
|
@ -5678,7 +5682,10 @@ def completion(
|
|||
if deployment_id is not None: # azure llms
|
||||
model = deployment_id
|
||||
custom_llm_provider = "azure"
|
||||
_supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs}
|
||||
_supplemental_provider_params: Final = {
|
||||
**{k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs},
|
||||
"extra_headers": extra_headers,
|
||||
}
|
||||
model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider(
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
|
|
@ -5998,6 +6005,7 @@ def completion(
|
|||
tpm=kwargs.get("tpm"),
|
||||
rpm=kwargs.get("rpm"),
|
||||
use_xai_oauth=kwargs.get("use_xai_oauth", False),
|
||||
chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"),
|
||||
gigachat_scope=kwargs.get("gigachat_scope"),
|
||||
gigachat_auth_url=kwargs.get("gigachat_auth_url"),
|
||||
gigachat_access_token=kwargs.get("gigachat_access_token"),
|
||||
|
|
@ -9140,6 +9148,7 @@ async def ahealth_check(
|
|||
custom_llm_provider=custom_llm_provider_from_params,
|
||||
api_base=api_base_from_params,
|
||||
api_key=api_key_from_params,
|
||||
litellm_params=GenericLiteLLMParams(**model_params),
|
||||
)
|
||||
|
||||
model_params["cache"] = {"no-cache": True} # don't used cached responses for making health check calls
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -27,6 +27,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import (
|
|||
get_passthrough_resource_metadata_url,
|
||||
get_passthrough_www_authenticate,
|
||||
get_request_base_url,
|
||||
get_route_relative_request_path,
|
||||
well_known_root_suffix,
|
||||
)
|
||||
from litellm.proxy._experimental.mcp_server.outbound_credentials.bridge_credentials import (
|
||||
|
|
@ -387,7 +388,7 @@ def _gateway_dcr_challenge(
|
|||
"""The RFC 9728 challenge pointing the client at the protected-resource metadata
|
||||
matching the scope it requested: the per-server document (same URL spelling the
|
||||
request arrived on) when the single target advertises gateway-owned sign-in,
|
||||
else the gateway's aggregate document. Either way the client discovers the gateway
|
||||
else a gateway document containing the public resource. Either way the client discovers the gateway
|
||||
as its authorization server and starts the same sign-in flow.
|
||||
|
||||
``invalid_token`` adds the RFC 6750 error code for a request that DID
|
||||
|
|
@ -397,10 +398,13 @@ def _gateway_dcr_challenge(
|
|||
client_ip: Final = IPAddressUtils.get_mcp_client_ip(request)
|
||||
target: Final = _gateway_dcr_challenge_target(route, mcp_servers, client_ip)
|
||||
server_scope: Final = gateway_dcr_challenge_scope(route, mcp_servers, client_ip) if target is None else None
|
||||
public_path: Final = get_route_relative_request_path(request.scope).rstrip("/")
|
||||
aggregate_resource: Final = public_path == "/mcp" or public_path.startswith("/mcp/")
|
||||
metadata_suffix: Final = f"{well_known_root_suffix()}/mcp" if aggregate_resource else ""
|
||||
resource_metadata_url: Final = (
|
||||
get_passthrough_resource_metadata_url(request.scope, target)
|
||||
if target is not None
|
||||
else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{well_known_root_suffix()}/mcp"
|
||||
else f"{get_request_base_url(request)}/.well-known/oauth-protected-resource{metadata_suffix}"
|
||||
)
|
||||
error_attr: Final = 'error="invalid_token", ' if invalid_token else ""
|
||||
scope_attr: Final = f', scope="{server_scope}"' if server_scope is not None else ""
|
||||
|
|
|
|||
|
|
@ -472,15 +472,15 @@ def redeem_passthrough_authorization_code(
|
|||
return sealed
|
||||
|
||||
|
||||
def _session_cookie_user_id(request: Request) -> str | None:
|
||||
async def _session_cookie_user_id(request: Request) -> str | None:
|
||||
"""The signed-in litellm user for a browser request, or ``None``. Thin wrapper so the
|
||||
aggregate DCR flow's verbs receive the identity as a plain value instead of parsing
|
||||
cookies themselves."""
|
||||
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # circular import at module load
|
||||
user_id_from_session_cookie,
|
||||
get_authenticated_browser_user_id,
|
||||
)
|
||||
|
||||
return user_id_from_session_cookie(request)
|
||||
return await get_authenticated_browser_user_id(request)
|
||||
|
||||
|
||||
def _redirect_to_litellm_login(request: Request) -> RedirectResponse:
|
||||
|
|
@ -996,17 +996,13 @@ async def _resolve_oauth_authorization_user(
|
|||
enforce_binding: bool,
|
||||
) -> str | RedirectResponse:
|
||||
"""Resolve the authorization subject without replacing denied credentials with cookie grants."""
|
||||
from litellm.proxy._experimental.mcp_server.byok_oauth_endpoints import ( # noqa: PLC0415 # proxy import cycle
|
||||
user_id_from_session_cookie,
|
||||
)
|
||||
|
||||
use_gateway_credential: Final = enforce_binding and await oauth_authorization_uses_gateway_credential(request)
|
||||
request_user_id: Final = (
|
||||
await authorize_oauth_credential_request(request, mcp_server.server_id) if use_gateway_credential else None
|
||||
)
|
||||
if use_gateway_credential and request_user_id is None:
|
||||
return _bridge_access_denied_redirect(redirect_uri, state, mcp_server)
|
||||
user_id: Final = request_user_id or user_id_from_session_cookie(request)
|
||||
user_id: Final = request_user_id or await _session_cookie_user_id(request)
|
||||
if user_id is None:
|
||||
return _redirect_to_litellm_login(request)
|
||||
if not await _user_can_reach_mcp_server(user_id, mcp_server.server_id):
|
||||
|
|
@ -1024,7 +1020,7 @@ async def _managed_authorization_scope(
|
|||
request
|
||||
)
|
||||
session_user: Final = (
|
||||
_session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None
|
||||
await _session_cookie_user_id(request) if authorized_user_id is None and not gateway_credential else None
|
||||
)
|
||||
scope_user_id: Final = authorized_user_id or (
|
||||
await authorize_oauth_credential_request(request, server.server_id)
|
||||
|
|
@ -2359,7 +2355,7 @@ async def authorize_mcp_session(
|
|||
code_challenge=code_challenge,
|
||||
code_challenge_method=code_challenge_method,
|
||||
response_type=response_type,
|
||||
session_user_id=_session_cookie_user_id(request),
|
||||
session_user_id=await _session_cookie_user_id(request),
|
||||
resource=resource,
|
||||
scope=scope,
|
||||
)
|
||||
|
|
@ -2391,7 +2387,7 @@ async def authorize(
|
|||
code_challenge=code_challenge,
|
||||
code_challenge_method=code_challenge_method,
|
||||
response_type=response_type,
|
||||
session_user_id=_session_cookie_user_id(request),
|
||||
session_user_id=await _session_cookie_user_id(request),
|
||||
lookup_consent_teams=lookup_consent_teams,
|
||||
)
|
||||
return aggregate_authorize(
|
||||
|
|
@ -2402,7 +2398,7 @@ async def authorize(
|
|||
code_challenge=code_challenge,
|
||||
code_challenge_method=code_challenge_method,
|
||||
response_type=response_type,
|
||||
session_user_id=_session_cookie_user_id(request),
|
||||
session_user_id=await _session_cookie_user_id(request),
|
||||
resource=resource,
|
||||
scope=scope,
|
||||
)
|
||||
|
|
@ -2560,7 +2556,7 @@ async def authorize_flow(request: Request, flow: str) -> Response:
|
|||
return await describe_connect_flow(
|
||||
request=request,
|
||||
flow_handle=flow,
|
||||
session_user_id=_session_cookie_user_id(request),
|
||||
session_user_id=await _session_cookie_user_id(request),
|
||||
lookup_vendor_credential=_vendor_credential_state,
|
||||
lookup_server_reachability=_user_can_reach_mcp_server,
|
||||
)
|
||||
|
|
@ -2586,7 +2582,7 @@ async def authorize_complete(
|
|||
return await complete_connect_flow(
|
||||
request=request,
|
||||
flow_handle=flow,
|
||||
session_user_id=_session_cookie_user_id(request),
|
||||
session_user_id=await _session_cookie_user_id(request),
|
||||
cache=user_api_key_cache,
|
||||
delivery=delivery,
|
||||
team_id=team_id,
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
The aggregate listing deliberately keeps serving the healthy subset when one server fails, but a
|
||||
failed server must contribute a classified outcome instead of silently shrinking the list: an empty
|
||||
contribution with no signal makes a broken upstream indistinguishable from a healthy server with no
|
||||
tools. Outcomes carry only machine fields (category and status code) so nothing from an upstream
|
||||
body crosses the trust boundary; classification is total, so any exception out of a server fetch
|
||||
tools. Outcomes carry machine fields and controlled diagnostics so nothing from an upstream body
|
||||
crosses the trust boundary; classification is total, so any exception out of a server fetch
|
||||
becomes an outcome, never a second failure.
|
||||
"""
|
||||
|
||||
|
|
@ -21,8 +21,10 @@ from urllib.request import parse_http_list, parse_keqv_list
|
|||
import anyio
|
||||
import httpx
|
||||
import httpx2
|
||||
from mcp import MCPError
|
||||
from mcp.types import CONNECTION_CLOSED
|
||||
from mcp.types import Tool as MCPTool
|
||||
from pydantic import ConfigDict, TypeAdapter
|
||||
from pydantic import ConfigDict, PositiveInt, TypeAdapter
|
||||
from typing_extensions import assert_never
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
|
|
@ -65,6 +67,16 @@ class ServerListFault(LiteLLMBaseModel):
|
|||
model_config = ConfigDict(frozen=True)
|
||||
tag: ListFaultCategory
|
||||
status_code: int | None = None
|
||||
sse_event_limit_bytes: PositiveInt | None = None
|
||||
|
||||
@property
|
||||
def public_message(self) -> str | None:
|
||||
if self.sse_event_limit_bytes is None:
|
||||
return None
|
||||
return (
|
||||
f"Upstream MCP SSE event exceeded the {self.sse_event_limit_bytes} byte per-event limit. "
|
||||
"Paginate tools/list responses so each event stays below the limit."
|
||||
)
|
||||
|
||||
|
||||
ServerOutcome: TypeAlias = ServerListOk | ServerListFault
|
||||
|
|
@ -155,6 +167,18 @@ def raise_classified_list_failure(
|
|||
raise MCPServerListError(classify_list_exception(exc), server_name) from exc
|
||||
|
||||
|
||||
def _sse_event_limit(exc: BaseException) -> int | None:
|
||||
"""The SDK replaces decoder exceptions with CONNECTION_CLOSED and a fixed diagnostic."""
|
||||
if not isinstance(exc, (httpx2.SSEError, MCPError)):
|
||||
return None
|
||||
if isinstance(exc, MCPError) and exc.error.code != CONNECTION_CLOSED:
|
||||
return None
|
||||
prefix: Final = "SSE stream failed: " if isinstance(exc, MCPError) else ""
|
||||
message: Final = exc.error.message if isinstance(exc, MCPError) else str(exc)
|
||||
matched: Final = re.fullmatch(prefix + r"Server-sent event exceeded the ([1-9][0-9]{0,18}) byte limit\.", message)
|
||||
return int(matched[1]) if matched is not None else None
|
||||
|
||||
|
||||
def classify_list_exception(exc: BaseException) -> ServerListFault:
|
||||
"""Classify a per-server listing failure into exactly one outcome. Total: an exception this
|
||||
function cannot recognize is the gateway's own fault (``internal``), never a re-raise."""
|
||||
|
|
@ -179,6 +203,11 @@ def classify_list_exception(exc: BaseException) -> ServerListFault:
|
|||
response: Final = _find_upstream_response(exc)
|
||||
if response is not None:
|
||||
return ServerListFault(tag="upstream_error", status_code=response.status_code)
|
||||
sse_event_limit: Final = next(
|
||||
(limit for error in iter_exception_tree(exc) if (limit := _sse_event_limit(error)) is not None), None
|
||||
)
|
||||
if sse_event_limit is not None:
|
||||
return ServerListFault(tag="upstream_error", sse_event_limit_bytes=sse_event_limit)
|
||||
if isinstance(exc, (httpx.TimeoutException, httpx2.TimeoutException)):
|
||||
return ServerListFault(tag="timeout")
|
||||
if isinstance(exc, (httpx.TransportError, httpx2.TransportError)):
|
||||
|
|
@ -295,7 +324,7 @@ def listing_failed_outright(outcomes: Mapping[str, ServerOutcome]) -> bool:
|
|||
|
||||
def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
|
||||
"""The client-visible form of one outcome, for the tools/list result ``_meta`` and the REST
|
||||
response: category plus status code only, never upstream prose or URLs."""
|
||||
response: category, status code and controlled diagnostics, never upstream prose or URLs."""
|
||||
match outcome.tag:
|
||||
case "ok":
|
||||
return {"status": "ok", "tool_count": outcome.tool_count}
|
||||
|
|
@ -303,6 +332,15 @@ def outcome_wire_value(outcome: ServerOutcome) -> dict[str, object]:
|
|||
return {
|
||||
"status": outcome.tag,
|
||||
**({"http_status": outcome.status_code} if outcome.status_code is not None else {}),
|
||||
**(
|
||||
{
|
||||
"reason": "sse_event_too_large",
|
||||
"limit_bytes": outcome.sse_event_limit_bytes,
|
||||
"message": outcome.public_message,
|
||||
}
|
||||
if outcome.sse_event_limit_bytes is not None
|
||||
else {}
|
||||
),
|
||||
}
|
||||
case _:
|
||||
assert_never(outcome.tag)
|
||||
|
|
|
|||
|
|
@ -957,7 +957,7 @@ if MCP_AVAILABLE:
|
|||
status_code=list_fault_http_status(fault),
|
||||
detail={
|
||||
"error": fault.tag,
|
||||
"message": f"Failed to list tools from server {get_server_prefix(server)}",
|
||||
"message": fault.public_message or f"Failed to list tools from server {get_server_prefix(server)}",
|
||||
},
|
||||
) from e
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -222,7 +222,7 @@ def _select_model_by_priority(
|
|||
input_cost = info.get("input_cost_per_token") or 0.0
|
||||
output_cost = info.get("output_cost_per_token") or 0.0
|
||||
total_cost = input_cost + output_cost
|
||||
max_output = info.get("max_output_tokens") or info.get("max_tokens") or 0
|
||||
max_output = info.get("max_output_tokens") or 0
|
||||
output_tps = info.get("output_tokens_per_second") or 0.0
|
||||
scored.append(
|
||||
_ScoredModel(
|
||||
|
|
|
|||
|
|
@ -33,9 +33,8 @@ class OAuthScopeResponse:
|
|||
self._request_path = request_path
|
||||
self._ready = asyncio.Event()
|
||||
self._lock = asyncio.Lock()
|
||||
self._start: Message | None = None
|
||||
self._challenge: MCPUpstreamAuthError | None = None
|
||||
self._refused = False
|
||||
self._closed = False
|
||||
|
||||
@property
|
||||
def pending(self) -> bool:
|
||||
|
|
@ -50,31 +49,33 @@ class OAuthScopeResponse:
|
|||
self._ready.set()
|
||||
|
||||
async def __call__(self, message: Message) -> None:
|
||||
if message["type"] == "http.response.start":
|
||||
self._start = message
|
||||
if message["status"] != 200:
|
||||
self.allow()
|
||||
return
|
||||
await self._ready.wait()
|
||||
async with self._lock:
|
||||
if self._refused:
|
||||
if self._closed:
|
||||
return
|
||||
if self._start is not None:
|
||||
start: Final = self._start
|
||||
self._start = None
|
||||
if self._challenge is not None:
|
||||
error: Final = self._challenge.to_http_exception(self._base_url, self._request_path)
|
||||
response: Final = JSONResponse(
|
||||
status_code=error.status_code, content={"detail": error.detail}, headers=error.headers
|
||||
)
|
||||
self._refused = True
|
||||
await self._send(
|
||||
{"type": "http.response.start", "status": response.status_code, "headers": response.raw_headers}
|
||||
)
|
||||
await self._send({"type": "http.response.body", "body": bytes(response.body)})
|
||||
return
|
||||
await self._send(start)
|
||||
await self._send(message)
|
||||
try:
|
||||
if message["type"] == "http.response.start":
|
||||
if message["status"] != 200:
|
||||
self.allow()
|
||||
await self._ready.wait()
|
||||
if self._challenge is not None:
|
||||
error: Final = self._challenge.to_http_exception(self._base_url, self._request_path)
|
||||
response: Final = JSONResponse(
|
||||
status_code=error.status_code, content={"detail": error.detail}, headers=error.headers
|
||||
)
|
||||
self._closed = True
|
||||
await self._send(
|
||||
{
|
||||
"type": "http.response.start",
|
||||
"status": response.status_code,
|
||||
"headers": response.raw_headers,
|
||||
}
|
||||
)
|
||||
await self._send({"type": "http.response.body", "body": bytes(response.body)})
|
||||
return
|
||||
await self._send(message)
|
||||
except BaseException:
|
||||
self._closed = True
|
||||
raise
|
||||
|
||||
|
||||
def scope_response_from_request(request: object) -> OAuthScopeResponse | None:
|
||||
|
|
|
|||
|
|
@ -207,6 +207,11 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
module_path="litellm.proxy.management_endpoints.config_override_endpoints",
|
||||
path_prefixes=("/config_overrides",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="codex_passthrough",
|
||||
module_path="litellm.proxy.pass_through_endpoints.codex_passthrough_endpoints",
|
||||
path_prefixes=("/backend-api/codex/",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="llm_passthrough",
|
||||
module_path="litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints",
|
||||
|
|
|
|||
|
|
@ -8673,6 +8673,184 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"codex_passthrough": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
"HTTPValidationError": {
|
||||
"properties": {
|
||||
"detail": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ValidationError"
|
||||
},
|
||||
"title": "Detail",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"title": "HTTPValidationError",
|
||||
"type": "object"
|
||||
},
|
||||
"ValidationError": {
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"title": "Context",
|
||||
"type": "object"
|
||||
},
|
||||
"input": {
|
||||
"title": "Input"
|
||||
},
|
||||
"loc": {
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Location",
|
||||
"type": "array"
|
||||
},
|
||||
"msg": {
|
||||
"title": "Message",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"title": "Error Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"loc",
|
||||
"msg",
|
||||
"type"
|
||||
],
|
||||
"title": "ValidationError",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"paths": {
|
||||
"/backend-api/codex/{endpoint}": {
|
||||
"get": {
|
||||
"description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.",
|
||||
"operationId": "codex_passthrough_route_backend_api_codex__endpoint__get",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"models",
|
||||
"responses"
|
||||
],
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Codex Passthrough Route",
|
||||
"tags": [
|
||||
"codex_passthrough"
|
||||
]
|
||||
},
|
||||
"post": {
|
||||
"description": "Forward a client's ChatGPT subscription with separate LiteLLM authentication.",
|
||||
"operationId": "codex_passthrough_route_backend_api_codex__endpoint__post",
|
||||
"parameters": [
|
||||
{
|
||||
"in": "path",
|
||||
"name": "endpoint",
|
||||
"required": true,
|
||||
"schema": {
|
||||
"enum": [
|
||||
"models",
|
||||
"responses"
|
||||
],
|
||||
"title": "Endpoint",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
],
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"additionalProperties": true,
|
||||
"type": "object"
|
||||
}
|
||||
},
|
||||
"text/event-stream": {
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Unmodified Codex response or server-sent events"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Codex Passthrough Route",
|
||||
"tags": [
|
||||
"codex_passthrough"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"compliance": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
|
|
|
|||
|
|
@ -391,6 +391,7 @@ class LiteLLMRoutes(enum.Enum):
|
|||
]
|
||||
|
||||
mapped_pass_through_routes = [
|
||||
"/backend-api/codex",
|
||||
"/bedrock",
|
||||
"/comprehendmedical",
|
||||
"/azure_speech",
|
||||
|
|
@ -1766,8 +1767,10 @@ class BudgetNewRequest(LiteLLMPydanticObjectBase):
|
|||
max_parallel_requests: int | None = Field(
|
||||
default=None, description="Max concurrent requests allowed for this budget id."
|
||||
)
|
||||
tpm_limit: int | None = Field(default=None, description="Max tokens per minute, allowed for this budget id.")
|
||||
rpm_limit: int | None = Field(default=None, description="Max requests per minute, allowed for this budget id.")
|
||||
tpm_limit: int | None = Field(default=None, ge=0, description="Max tokens per minute, allowed for this budget id.")
|
||||
rpm_limit: int | None = Field(
|
||||
default=None, ge=0, description="Max requests per minute, allowed for this budget id."
|
||||
)
|
||||
tpd_limit: int | None = Field(
|
||||
default=None, description="Max tokens per day, charged by batch submissions, allowed for this budget id."
|
||||
)
|
||||
|
|
@ -1878,6 +1881,8 @@ RouterSettingsDict = Annotated[
|
|||
|
||||
|
||||
class NewTeamRequest(TeamBase):
|
||||
tpm_limit: int | None = Field(default=None, ge=0)
|
||||
rpm_limit: int | None = Field(default=None, ge=0)
|
||||
router_settings: RouterSettingsDict | None = None
|
||||
model_aliases: dict | None = None
|
||||
model_max_budget: GenericBudgetConfigType | None = Field(
|
||||
|
|
@ -1957,8 +1962,8 @@ class UpdateTeamRequest(LiteLLMPydanticObjectBase):
|
|||
team_alias: str | None = None
|
||||
organization_id: str | None = None
|
||||
metadata: dict | None = None
|
||||
tpm_limit: int | None = None
|
||||
rpm_limit: int | None = None
|
||||
tpm_limit: int | None = Field(default=None, ge=0)
|
||||
rpm_limit: int | None = Field(default=None, ge=0)
|
||||
tpd_limit: int | None = None
|
||||
max_budget: float | None = None
|
||||
soft_budget: float | None = None
|
||||
|
|
@ -3272,7 +3277,7 @@ class LiteLLM_OrganizationTableWithMembers(LiteLLM_OrganizationTable):
|
|||
|
||||
members: list[LiteLLM_OrganizationMembershipTable] = []
|
||||
teams: list[LiteLLM_TeamTable] = []
|
||||
litellm_budget_table: LiteLLM_BudgetTable | None = None
|
||||
litellm_budget_table: LiteLLM_BudgetTableFull | None = None
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
|
|
|
|||
|
|
@ -6461,8 +6461,29 @@ def _search_tool_deny_by_default(general_settings: Mapping[str, object]) -> bool
|
|||
return True
|
||||
|
||||
|
||||
def _is_dashboard_session(valid_token: UserAPIKeyAuth | None) -> bool:
|
||||
return valid_token is not None and valid_token.team_id == UI_TEAM_ID
|
||||
|
||||
|
||||
def _is_search_tool_strict_identity(valid_token: UserAPIKeyAuth) -> bool:
|
||||
if not _is_dashboard_session(valid_token):
|
||||
return _is_strict_grant_identity(valid_token)
|
||||
return valid_token.user_role not in (
|
||||
LitellmUserRoles.PROXY_ADMIN,
|
||||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
)
|
||||
|
||||
|
||||
def _search_tool_strict_layers(
|
||||
valid_token: UserAPIKeyAuth, team_object: LiteLLM_TeamTable | None
|
||||
) -> frozenset[GrantLayer]:
|
||||
if _is_dashboard_session(valid_token):
|
||||
return frozenset(("user",))
|
||||
return _strict_grant_layers(True, valid_token, team_object)
|
||||
|
||||
|
||||
def is_search_tool_deny_by_default_applied(valid_token: UserAPIKeyAuth, general_settings: Mapping[str, object]) -> bool:
|
||||
return _search_tool_deny_by_default(general_settings) and _is_strict_grant_identity(valid_token)
|
||||
return _search_tool_deny_by_default(general_settings) and _is_search_tool_strict_identity(valid_token)
|
||||
|
||||
|
||||
def _search_tool_denied(object_type: GrantLayer, message: str) -> ProxyException:
|
||||
|
|
@ -6506,12 +6527,12 @@ async def resolve_search_tool_grants(
|
|||
load_team_object: TeamObjectLoader,
|
||||
) -> SearchToolGrants:
|
||||
"""
|
||||
Without `general_settings.search_tool_deny_by_default`, or for the master key and dashboard sessions,
|
||||
Without `general_settings.search_tool_deny_by_default`, or for the master key and admin dashboard sessions,
|
||||
the key and team lists only restrict when nonempty. With it, the identities `_strict_grant_layers` names
|
||||
must each list the tool, and a missing record, `null`, `[]`, or a team or user that fails to load grants nothing
|
||||
"""
|
||||
if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
|
||||
team_object: Final = await load_team_object()
|
||||
team_object: Final = await load_team_object() if not _is_dashboard_session(valid_token) else None
|
||||
return SearchToolGrants(
|
||||
grants=MappingProxyType(
|
||||
{"key": valid_token.object_permission, "team": team_object.object_permission if team_object else None}
|
||||
|
|
@ -6521,8 +6542,10 @@ async def resolve_search_tool_grants(
|
|||
|
||||
from litellm.proxy.proxy_server import prisma_client, user_api_key_cache
|
||||
|
||||
strict_team_object: Final = await _strict_team_object(load_team_object)
|
||||
strict_layers: Final = _strict_grant_layers(True, valid_token, strict_team_object)
|
||||
strict_team_object: Final = (
|
||||
await _strict_team_object(load_team_object) if not _is_dashboard_session(valid_token) else None
|
||||
)
|
||||
strict_layers: Final = _search_tool_strict_layers(valid_token, strict_team_object)
|
||||
if prisma_client is None:
|
||||
return SearchToolGrants(grants=MappingProxyType({}), strict_layers=strict_layers)
|
||||
user_object: Final = (
|
||||
|
|
@ -6607,7 +6630,7 @@ def check_unregistered_search_fallback(
|
|||
"""The unregistered provider fallback names no search tool that a grant could list, so deny-by-default denies it"""
|
||||
if not is_search_tool_deny_by_default_applied(valid_token, general_settings):
|
||||
return True
|
||||
layer: Final = min(_strict_grant_layers(True, valid_token, None), key=("key", "team", "user").index)
|
||||
layer: Final = min(_search_tool_strict_layers(valid_token, None), key=("key", "team", "user").index)
|
||||
raise _search_tool_denied(
|
||||
layer,
|
||||
"No registered search tool is available and search_tool_deny_by_default is enabled",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ import litellm
|
|||
from litellm._logging import verbose_proxy_logger, verbose_proxy_stdout_logger
|
||||
from litellm.constants import EMPTY_MAPPING
|
||||
from litellm.integrations.otel.runtime import seed_request_identity
|
||||
from litellm.litellm_core_utils.core_helpers import is_expected_client_error
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, is_expected_client_error
|
||||
from litellm.proxy._types import (
|
||||
LitellmUserRoles,
|
||||
ModelAccessDeniedProxyException,
|
||||
|
|
@ -139,7 +139,7 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
async def handle_authentication_error(
|
||||
e: Exception,
|
||||
request: Request,
|
||||
request_data: dict[str, object],
|
||||
request_data: object,
|
||||
route: str,
|
||||
parent_otel_span: Span | None,
|
||||
api_key: str,
|
||||
|
|
@ -190,6 +190,8 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
request_route=route,
|
||||
)
|
||||
else:
|
||||
request_mapping: Final = as_str_mapping(request_data)
|
||||
request_context: Final[dict[str, object]] = dict(request_mapping) if request_mapping is not None else {}
|
||||
# raise the exception to the caller
|
||||
requester_ip: Final = get_request_ip_address(
|
||||
request=request,
|
||||
|
|
@ -229,7 +231,7 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
# without this the failed trace would carry no team/key attributes.
|
||||
seed_request_identity(
|
||||
user_api_key_dict,
|
||||
model=request_data.get("model"),
|
||||
model=request_context.get("model"),
|
||||
)
|
||||
|
||||
# Budget checks live in tenant-scoped helpers (key / team / org / tag)
|
||||
|
|
@ -242,14 +244,14 @@ class UserAPIKeyAuthExceptionHandler:
|
|||
resolve_llm_provider_for_rate_limit,
|
||||
)
|
||||
|
||||
budget_model: Final = request_data.get("model")
|
||||
budget_model: Final = request_context.get("model")
|
||||
_, e.llm_provider = resolve_llm_provider_for_rate_limit(
|
||||
budget_model if isinstance(budget_model, str) else None
|
||||
)
|
||||
|
||||
# Allow callbacks to transform the error response
|
||||
transformed_exception: Final = await proxy_logging_obj.post_call_failure_hook(
|
||||
request_data=_with_client_context(request_data, requester_ip, user_agent),
|
||||
request_data=_with_client_context(request_context, requester_ip, user_agent),
|
||||
original_exception=e,
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
error_type=ProxyErrorTypes.auth_error,
|
||||
|
|
|
|||
|
|
@ -736,7 +736,9 @@ def is_request_body_safe(
|
|||
|
||||
def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None:
|
||||
if manages_deployments:
|
||||
reject_server_owned_wif_fields(litellm_params)
|
||||
reject_server_owned_wif_fields(
|
||||
MappingProxyType({key: value for key, value in litellm_params.items() if key != "chatgpt_auth_file"})
|
||||
)
|
||||
else:
|
||||
reject_server_owned_wif_params(litellm_params)
|
||||
reject_federated_credential_reference(litellm_params)
|
||||
|
|
|
|||
|
|
@ -119,6 +119,7 @@ from litellm.proxy.common_utils.sse_keepalive import (
|
|||
)
|
||||
from litellm.proxy.dd_span_tagger import DDSpanTagger
|
||||
from litellm.proxy.guardrails.auto_router_compression import arm_pre_call as _arm_auto_router_compression
|
||||
from litellm.proxy.hooks.parallel_request_limiter_v3 import get_or_create_request_stash
|
||||
from litellm.proxy.native_compaction import with_proxy_compaction_executor
|
||||
from litellm.proxy.route_llm_request import (
|
||||
route_request,
|
||||
|
|
@ -1416,6 +1417,8 @@ async def open_sse_before_first_byte(
|
|||
if interval is None:
|
||||
return await produce_response
|
||||
|
||||
# Share admission state with stream cleanup before the producer copies this context.
|
||||
get_or_create_request_stash()
|
||||
produce_task: Final = asyncio.ensure_future(produce_response)
|
||||
await asyncio.wait((produce_task,), timeout=interval)
|
||||
if produce_task.done():
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin
|
|||
|
||||
import orjson
|
||||
from fastapi import Request, UploadFile, status
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
from starlette._utils import get_route_path
|
||||
from starlette.requests import HTTPConnection
|
||||
from typing_extensions import NotRequired, ReadOnly, Required, assert_never
|
||||
|
|
@ -162,13 +162,11 @@ def coerce_numeric_form_fields(
|
|||
}
|
||||
|
||||
|
||||
def _parse_binary_body(body: bytes) -> dict:
|
||||
def _parse_binary_body(body: bytes) -> dict[str, object]:
|
||||
"""JSON sent under a binary content type still parses; real binary (protobuf) carries no params -> {}."""
|
||||
try:
|
||||
parsed: Final = _DECODED_JSON.validate_python(orjson.loads(body))
|
||||
if isinstance(parsed, dict):
|
||||
return parsed
|
||||
except orjson.JSONDecodeError:
|
||||
return dict(_JSON_REQUEST_BODY.validate_python(orjson.loads(body)))
|
||||
except (orjson.JSONDecodeError, ValidationError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
|
@ -299,8 +297,9 @@ async def read_request_body(request: Request | None) -> dict:
|
|||
body_str = re.sub(r"(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]", "", body_str)
|
||||
|
||||
try:
|
||||
parsed_body = json.loads(body_str)
|
||||
json.dumps(parsed_body, ensure_ascii=False).encode("utf-8")
|
||||
decoded_body: Final = _DECODED_JSON.validate_python(json.loads(body_str))
|
||||
json.dumps(decoded_body, ensure_ascii=False).encode("utf-8")
|
||||
parsed_body = dict(_JSON_REQUEST_BODY.validate_python(decoded_body))
|
||||
except (json.JSONDecodeError, UnicodeEncodeError):
|
||||
# json.loads accepts lone surrogate escapes that no provider can encode
|
||||
verbose_proxy_logger.error("Invalid JSON payload received: %s", e)
|
||||
|
|
@ -315,6 +314,34 @@ async def read_request_body(request: Request | None) -> dict:
|
|||
safe_set_request_parsed_body(request=request, parsed_body=parsed_body)
|
||||
return parsed_body
|
||||
|
||||
except ValidationError as e:
|
||||
from fastapi.routing import APIRoute
|
||||
|
||||
from litellm.proxy.auth.auth_utils import (
|
||||
request_dispatched_to_pass_through_endpoint,
|
||||
request_dispatched_to_provider_pass_through,
|
||||
)
|
||||
from litellm.proxy.lens.buffering import LensRoute
|
||||
|
||||
route: Final[object] = request.scope.get("route") if request is not None else None
|
||||
if request is not None and (
|
||||
request_dispatched_to_pass_through_endpoint(request)
|
||||
or request_dispatched_to_provider_pass_through(request)
|
||||
or isinstance(route, LensRoute)
|
||||
or (
|
||||
isinstance(route, APIRoute)
|
||||
and route.body_field is not None
|
||||
and not route.body_field.field_info.is_required()
|
||||
and (await request.body()).strip() == b"null"
|
||||
)
|
||||
):
|
||||
return {}
|
||||
raise ProxyException(
|
||||
message="Invalid JSON payload: expected an object",
|
||||
type="invalid_request_error",
|
||||
param="request_body",
|
||||
code=status.HTTP_400_BAD_REQUEST,
|
||||
) from e
|
||||
except (json.JSONDecodeError, orjson.JSONDecodeError, ProxyException) as e:
|
||||
# Re-raise ProxyException as-is
|
||||
verbose_proxy_logger.error("Invalid JSON payload received: %s", e)
|
||||
|
|
|
|||
|
|
@ -2304,7 +2304,15 @@ async def test_model_connection(
|
|||
)
|
||||
|
||||
selected_by_id: Final = deployment_by_id is not None
|
||||
reject_server_owned_wif_fields(request_litellm_params)
|
||||
reject_server_owned_wif_fields(
|
||||
MappingProxyType(
|
||||
{
|
||||
key: value
|
||||
for key, value in _OBJECT_MAPPING.validate_python(request_litellm_params).items()
|
||||
if key != "chatgpt_auth_file"
|
||||
}
|
||||
)
|
||||
)
|
||||
# Merge: config params (from proxy config) as base, request params override
|
||||
litellm_params = {
|
||||
**_config_base_for_health_check(
|
||||
|
|
|
|||
|
|
@ -1417,13 +1417,14 @@ def clean_headers(
|
|||
Cleaned headers dict
|
||||
"""
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
|
||||
from litellm.llms.chatgpt.common_utils import is_chatgpt_oauth_key
|
||||
|
||||
clean_headers: Final = {}
|
||||
litellm_key_lower: Final = litellm_key_header_name.lower() if litellm_key_header_name is not None else None
|
||||
for header, value in headers.items():
|
||||
header_lower = header.lower()
|
||||
|
||||
if header_lower == "authorization" and is_anthropic_oauth_key(value):
|
||||
if header_lower == "authorization" and (is_anthropic_oauth_key(value) or is_chatgpt_oauth_key(value)):
|
||||
if authenticated_with_header is None or authenticated_with_header.lower() != "authorization":
|
||||
clean_headers[header] = value
|
||||
continue
|
||||
|
|
@ -1452,7 +1453,7 @@ def clean_headers(
|
|||
|
||||
def _is_credential_header(header: str) -> bool:
|
||||
"""Whether `header` carries a caller credential rather than request context."""
|
||||
return header.lower() in _CREDENTIAL_HEADER_NAMES
|
||||
return header.lower().removeprefix("x-pass-") in _CREDENTIAL_HEADER_NAMES
|
||||
|
||||
|
||||
def redact_credential_headers(headers: Mapping[str, str]) -> Mapping[str, str]:
|
||||
|
|
@ -4282,6 +4283,7 @@ _NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
|||
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
|
||||
)
|
||||
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
|
||||
_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS: Final = LlmProviders.CHATGPT.value
|
||||
|
||||
|
||||
def add_provider_specific_headers_to_request(
|
||||
|
|
@ -4289,6 +4291,7 @@ def add_provider_specific_headers_to_request(
|
|||
headers: Mapping[str, str],
|
||||
) -> bool:
|
||||
from litellm.llms.anthropic.common_utils import is_anthropic_oauth_key
|
||||
from litellm.llms.chatgpt.common_utils import select_chatgpt_client_credential_headers
|
||||
|
||||
anthropic_api_headers: Final = {header: headers[header] for header in ANTHROPIC_API_HEADERS if header in headers}
|
||||
anthropic_oauth_credential_headers: Final = {
|
||||
|
|
@ -4296,6 +4299,8 @@ def add_provider_specific_headers_to_request(
|
|||
for header, value in headers.items()
|
||||
if header.lower() == "authorization" and is_anthropic_oauth_key(value)
|
||||
}
|
||||
chatgpt_oauth_credential_headers: Final = dict(select_chatgpt_client_credential_headers(headers))
|
||||
|
||||
typesafe_danger_zone_headers: Final = {
|
||||
header: value
|
||||
for header, value in headers.items()
|
||||
|
|
@ -4320,13 +4325,17 @@ def add_provider_specific_headers_to_request(
|
|||
ProviderSpecificHeader(
|
||||
custom_llm_provider=LlmProviders.TYPESAFE.value, extra_headers=typesafe_danger_zone_headers
|
||||
),
|
||||
ProviderSpecificHeader(
|
||||
custom_llm_provider=_CHATGPT_OAUTH_CREDENTIAL_PROVIDERS,
|
||||
extra_headers=chatgpt_oauth_credential_headers,
|
||||
),
|
||||
)
|
||||
if scoped_header["extra_headers"]
|
||||
]
|
||||
|
||||
if scoped_headers:
|
||||
data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers
|
||||
return bool(anthropic_oauth_credential_headers)
|
||||
return bool(anthropic_oauth_credential_headers or chatgpt_oauth_credential_headers)
|
||||
|
||||
|
||||
def _add_otel_traceparent_to_data(data: dict, request: Request):
|
||||
|
|
|
|||
|
|
@ -569,8 +569,8 @@ _TEAM_MEMBER_BUDGET_LIMIT_FIELDS: Final = (
|
|||
"temp_budget_expiry",
|
||||
)
|
||||
|
||||
_TEMP_BUDGET_FIELDS: Final = frozenset({"temp_budget_increase", "temp_budget_expiry"})
|
||||
|
||||
_BUDGET_COLUMNS: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
MEMBER_BUDGET_PATCH_FIELDS: Final = MappingProxyType(
|
||||
{
|
||||
|
|
@ -653,8 +653,8 @@ async def upsert_budget_and_membership(
|
|||
``shared_budget_ids`` extends that protection to any other row more than one
|
||||
membership points at, which a caller patching several members at once has
|
||||
already counted; a row listed there is cloned rather than written in place.
|
||||
A patch that only touches the temporary budget pair never copies permanent
|
||||
limits into a new row, so the member keeps inheriting the live team default.
|
||||
A patch whose every field already matches the shared row changes nothing and
|
||||
is a no-op rather than a clone.
|
||||
"""
|
||||
ignore_default_model_budget_clear: Final = (
|
||||
existing_budget_id is not None
|
||||
|
|
@ -685,7 +685,6 @@ async def upsert_budget_and_membership(
|
|||
is_shared_default: Final = existing_budget_id is not None and (
|
||||
existing_budget_id == team_default_budget_id or existing_budget_id in (shared_budget_ids or frozenset())
|
||||
)
|
||||
temp_only: Final = frozenset(write_data) <= _TEMP_BUDGET_FIELDS
|
||||
|
||||
async def _disconnect():
|
||||
await tx.litellm_teammembership.update(
|
||||
|
|
@ -707,11 +706,15 @@ async def upsert_budget_and_membership(
|
|||
return
|
||||
|
||||
source_row: Final = (
|
||||
await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id})
|
||||
if is_shared_default and not temp_only
|
||||
else None
|
||||
await tx.litellm_budgettable.find_unique(where={"budget_id": existing_budget_id}) if is_shared_default else None
|
||||
)
|
||||
source: Final[Mapping[str, object]] = source_row.model_dump() if source_row is not None else MappingProxyType({})
|
||||
source: Final[Mapping[str, object]] = (
|
||||
MappingProxyType(_BUDGET_COLUMNS.validate_python(source_row.model_dump()))
|
||||
if source_row is not None
|
||||
else MappingProxyType({})
|
||||
)
|
||||
if is_shared_default and all(source.get(field) == value for field, value in effective_patch.items()):
|
||||
return
|
||||
|
||||
create_data: Final[dict[str, object]] = { # mutable-ok: Prisma create payloads are dict-shaped
|
||||
"created_by": user_api_key_dict.user_id or "",
|
||||
|
|
@ -740,7 +743,7 @@ async def upsert_budget_and_membership(
|
|||
create_data.pop("budget_reset_at", None)
|
||||
|
||||
if not _has_meaningful_budget_limit(create_data):
|
||||
if existing_budget_id is not None and not temp_only:
|
||||
if existing_budget_id is not None:
|
||||
await _disconnect()
|
||||
return
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import Final, Literal, TypeAlias
|
|||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.litellm_core_utils.duration_parser import normalize_duration
|
||||
from litellm.models.team import BudgetLimitEntry
|
||||
from litellm.models.verification_token import LiteLLM_VerificationToken
|
||||
from litellm.proxy._types import UpdateKeyRequest
|
||||
|
|
@ -99,6 +100,30 @@ def _budget_limits_change(data: UpdateKeyRequest, existing: LiteLLM_Verification
|
|||
return "tightened" if kept_or_lowered else "loosened"
|
||||
|
||||
|
||||
def _reset_schedule(budget_duration: str | None) -> str | None:
|
||||
return None if budget_duration is None else normalize_duration(budget_duration)
|
||||
|
||||
|
||||
def _changes_budget_duration(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool:
|
||||
return "budget_duration" in data.model_fields_set and _reset_schedule(data.budget_duration) != _reset_schedule(
|
||||
existing.budget_duration
|
||||
)
|
||||
|
||||
|
||||
def _changes_budget_id(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> bool:
|
||||
return "budget_id" in data.model_fields_set and data.budget_id != existing.budget_id
|
||||
|
||||
|
||||
def _budget_duration_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> _FieldChange:
|
||||
"""
|
||||
Any new `budget_duration` restarts the reset schedule from now, so only clearing it tightens the
|
||||
stored `max_budget`. A key with no `max_budget` has no cap for the schedule to reset.
|
||||
"""
|
||||
if existing.max_budget is None or not _changes_budget_duration(data, existing):
|
||||
return "unchanged"
|
||||
return "tightened" if data.budget_duration is None else "loosened"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class EffectiveKeyBudget:
|
||||
max_budget: float | None
|
||||
|
|
@ -122,12 +147,23 @@ def effective_key_budget(data: UpdateKeyRequest, existing: LiteLLM_VerificationT
|
|||
|
||||
|
||||
def classify_key_budget_change(data: UpdateKeyRequest, existing: LiteLLM_VerificationToken) -> KeyBudgetChange:
|
||||
overrides_linked_budget: Final = (
|
||||
existing.budget_id is not None and _max_budget_change(data, existing) != "unchanged"
|
||||
overrides_linked_budget: Final = existing.budget_id is not None and (
|
||||
_max_budget_change(data, existing) != "unchanged" or _changes_budget_duration(data, existing)
|
||||
)
|
||||
if data.spend is not None or "soft_budget" in data.model_fields_set or overrides_linked_budget:
|
||||
if (
|
||||
data.spend is not None
|
||||
or "soft_budget" in data.model_fields_set
|
||||
or overrides_linked_budget
|
||||
or _changes_budget_id(data, existing)
|
||||
):
|
||||
return KeyBudgetAdminOnly()
|
||||
changes: Final = frozenset((_max_budget_change(data, existing), _budget_limits_change(data, existing)))
|
||||
changes: Final = frozenset(
|
||||
(
|
||||
_max_budget_change(data, existing),
|
||||
_budget_limits_change(data, existing),
|
||||
_budget_duration_change(data, existing),
|
||||
)
|
||||
)
|
||||
if "loosened" in changes:
|
||||
return KeyBudgetLoosened()
|
||||
if "tightened" in changes:
|
||||
|
|
|
|||
|
|
@ -504,9 +504,8 @@ async def new_organization(
|
|||
|
||||
_budget: Final = await _table(BudgetRepository(prisma_client)).create(
|
||||
data={
|
||||
**new_budget,
|
||||
**build_budget_write_data(new_budget, user_api_key_dict.user_id or litellm_proxy_admin_name),
|
||||
"created_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
|
||||
"updated_by": user_api_key_dict.user_id or litellm_proxy_admin_name,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,87 @@
|
|||
import hmac
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Request, Response
|
||||
|
||||
from litellm.llms.chatgpt.common_utils import CHATGPT_API_BASE
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.pass_through_endpoints.pass_through_endpoints import pass_through_request
|
||||
|
||||
router: Final = APIRouter()
|
||||
_HEADERS: Final = frozenset(
|
||||
{
|
||||
"authorization",
|
||||
"chatgpt-account-id",
|
||||
"content-type",
|
||||
"accept",
|
||||
"user-agent",
|
||||
"originator",
|
||||
"openai-beta",
|
||||
"session-id",
|
||||
"thread-id",
|
||||
"session_id",
|
||||
"conversation_id",
|
||||
"x-codex-turn-metadata",
|
||||
"x-codex-turn-state",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/backend-api/codex/{endpoint}", tags=["ChatGPT Pass-through", "pass-through"])
|
||||
@router.post(
|
||||
"/backend-api/codex/{endpoint}",
|
||||
tags=["ChatGPT Pass-through", "pass-through"],
|
||||
openapi_extra={
|
||||
"requestBody": {
|
||||
"required": True,
|
||||
"content": {"application/json": {"schema": {"type": "object", "additionalProperties": True}}},
|
||||
}
|
||||
},
|
||||
responses={
|
||||
200: {
|
||||
"description": "Unmodified Codex response or server-sent events",
|
||||
"content": {
|
||||
"application/json": {"schema": {"type": "object", "additionalProperties": True}},
|
||||
"text/event-stream": {"schema": {"type": "string"}},
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
async def codex_passthrough_route(
|
||||
endpoint: Literal["models", "responses"],
|
||||
request: Request,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
) -> Response:
|
||||
"""Forward a client's ChatGPT subscription with separate LiteLLM authentication."""
|
||||
expected_method: Final = "GET" if endpoint == "models" else "POST"
|
||||
if request.method != expected_method:
|
||||
raise HTTPException(status_code=405, detail="Method not allowed", headers={"Allow": expected_method})
|
||||
|
||||
authorization: Final = request.headers.get("authorization", "")
|
||||
scheme, _, token = authorization.partition(" ")
|
||||
gateway_key: Final = request.headers.get("x-litellm-api-key", "")
|
||||
gateway_parts: Final = gateway_key.split()
|
||||
gateway_token: Final = (
|
||||
gateway_parts[-1]
|
||||
if len(gateway_parts) == 1 or (len(gateway_parts) == 2 and gateway_parts[0].lower() == "bearer")
|
||||
else ""
|
||||
)
|
||||
if scheme.lower() != "bearer" or not token.strip() or not gateway_token:
|
||||
raise HTTPException(
|
||||
status_code=401,
|
||||
detail="Send the ChatGPT bearer in Authorization and the LiteLLM key in x-litellm-api-key",
|
||||
)
|
||||
if hmac.compare_digest(token.strip().encode(), gateway_token.encode()):
|
||||
raise HTTPException(status_code=401, detail="ChatGPT and LiteLLM credentials must be separate")
|
||||
|
||||
headers: Final = MappingProxyType({name: value for name, value in request.headers.items() if name in _HEADERS})
|
||||
return await pass_through_request(
|
||||
request=request,
|
||||
target=f"{CHATGPT_API_BASE}/{endpoint}",
|
||||
custom_headers=dict(headers),
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
custom_llm_provider="chatgpt",
|
||||
upstream_headers_filter=lambda _: headers,
|
||||
)
|
||||
|
|
@ -10,6 +10,7 @@ from typing import Final, Literal, TypeAlias
|
|||
from urllib.parse import urlparse
|
||||
|
||||
import httpx
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -95,6 +96,8 @@ def is_openai_compatible_url(url_route: str | None) -> bool:
|
|||
hostname: Final = parsed_url.hostname
|
||||
if not hostname:
|
||||
return False
|
||||
if hostname == "chatgpt.com" and parsed_url.path.startswith("/backend-api/codex/"):
|
||||
return True
|
||||
if _hostname_matches(hostname, _OPENAI_HOSTNAMES):
|
||||
return True
|
||||
if _hostname_matches(hostname, _AZURE_OPENAI_HOSTNAMES):
|
||||
|
|
@ -198,7 +201,7 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
if not url_route:
|
||||
return False
|
||||
parsed_url: Final = urlparse(url_route)
|
||||
return _is_openai_compatible_host(parsed_url.hostname) and (
|
||||
return is_openai_compatible_url(url_route) and (
|
||||
"/v1/responses" in parsed_url.path or "/responses" in parsed_url.path
|
||||
)
|
||||
|
||||
|
|
@ -424,7 +427,9 @@ class OpenAIPassthroughLoggingHandler(BasePassthroughLoggingHandler):
|
|||
) = None
|
||||
handler_instance: Final = OpenAIPassthroughLoggingHandler()
|
||||
|
||||
custom_llm_provider: Final = kwargs.get("custom_llm_provider", "openai")
|
||||
custom_llm_provider: Final = TypeAdapter(str).validate_python(
|
||||
kwargs.get("custom_llm_provider", logging_obj.model_call_details.get("custom_llm_provider", "openai"))
|
||||
)
|
||||
|
||||
if is_chat_completions:
|
||||
# Handle chat completions with existing logic
|
||||
|
|
|
|||
|
|
@ -118,6 +118,7 @@ from litellm.proxy.litellm_pre_call_utils import ( # noqa: F401 # legacy modul
|
|||
_key_or_team_allows_client_pricing_override, # pyright: ignore[reportPrivateUsage] # reuse the proxy's pricing trust policy
|
||||
_strip_client_pricing_overrides, # pyright: ignore[reportPrivateUsage] # sanitize before trusted hooks add guardrail costs
|
||||
get_dynamic_logging_metadata, # pyright: ignore[reportPrivateUsage] # shared proxy helper, same import style as _read_request_body above
|
||||
redact_credential_headers,
|
||||
)
|
||||
from litellm.proxy.route_llm_request import ProxyModelNotFoundError
|
||||
from litellm.proxy.route_priority import (
|
||||
|
|
@ -409,6 +410,8 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
@staticmethod
|
||||
def get_endpoint_type(url: str, custom_llm_provider: str | None = None) -> EndpointType:
|
||||
parsed_url: Final = urlparse(url)
|
||||
if custom_llm_provider == "chatgpt":
|
||||
return EndpointType.OPENAI
|
||||
if custom_llm_provider in ("typesafe", "laya", "bespoke") and parsed_url.path.removesuffix("/").endswith(
|
||||
"/v1/systemone"
|
||||
):
|
||||
|
|
@ -723,7 +726,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
|
|||
"url": str(request.url),
|
||||
"method": request.method,
|
||||
"body": copy.copy(_parsed_body), # use copy instead of deepcopy
|
||||
"headers": request.headers,
|
||||
"headers": redact_credential_headers(request.headers),
|
||||
},
|
||||
},
|
||||
"call_type": "pass_through_endpoint",
|
||||
|
|
@ -1490,7 +1493,7 @@ async def pass_through_request(
|
|||
additional_args={
|
||||
"complete_input_dict": _parsed_body,
|
||||
"api_base": str(logging_url),
|
||||
"headers": upstream_headers,
|
||||
"headers": redact_credential_headers(upstream_headers),
|
||||
},
|
||||
)
|
||||
stream = HttpPassThroughEndpointHelpers.update_stream_param_based_on_request_body(
|
||||
|
|
@ -1561,7 +1564,7 @@ async def pass_through_request(
|
|||
data=_parsed_body or {},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=relay_response,
|
||||
request_headers=dict(request.headers),
|
||||
request_headers=dict(redact_credential_headers(request.headers)),
|
||||
)
|
||||
if callback_headers:
|
||||
_response_headers.update(callback_headers)
|
||||
|
|
@ -1653,7 +1656,7 @@ async def pass_through_request(
|
|||
data=_parsed_body or {},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=detected_relay_response,
|
||||
request_headers=dict(request.headers),
|
||||
request_headers=dict(redact_credential_headers(request.headers)),
|
||||
)
|
||||
if callback_headers:
|
||||
_response_headers.update(callback_headers)
|
||||
|
|
@ -1703,7 +1706,7 @@ async def pass_through_request(
|
|||
data=_parsed_body or {},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=response,
|
||||
request_headers=dict(request.headers),
|
||||
request_headers=dict(redact_credential_headers(request.headers)),
|
||||
)
|
||||
if relay_callback_headers:
|
||||
relay_custom_headers.update(relay_callback_headers)
|
||||
|
|
@ -1878,7 +1881,7 @@ async def pass_through_request(
|
|||
data=_parsed_body or {},
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
response=response,
|
||||
request_headers=dict(request.headers),
|
||||
request_headers=dict(redact_credential_headers(request.headers)),
|
||||
)
|
||||
if callback_headers:
|
||||
custom_headers.update(callback_headers)
|
||||
|
|
|
|||
|
|
@ -21795,8 +21795,7 @@ async def toolset_mcp_route(toolset_name: str, request: Request):
|
|||
detail=f"Toolset '{toolset_name}' not found",
|
||||
)
|
||||
|
||||
scope: Final = dict(request.scope)
|
||||
scope["path"] = "/mcp"
|
||||
scope: Final = {**request.scope, "_original_path": request.scope.get("path", ""), "path": "/mcp"}
|
||||
|
||||
token: Final = mcp_active_toolset_id.set(toolset.toolset_id)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -308,10 +308,7 @@ def should_use_polling_for_request(
|
|||
# Otherwise, check ALL deployments for this model_name in router
|
||||
elif llm_router is not None:
|
||||
try:
|
||||
# Get all deployment indices for this model name
|
||||
indices: Final = llm_router.model_name_to_deployment_indices.get(model, [])
|
||||
for idx in indices:
|
||||
deployment_dict = llm_router.model_list[idx]
|
||||
for deployment_dict in llm_router.deployments_named(model) or ():
|
||||
litellm_params = deployment_dict.get("litellm_params", {})
|
||||
|
||||
# Check custom_llm_provider first
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue