mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
Merge remote-tracking branch 'origin/main' into litellm_partial_stream_live_budget_counters
This commit is contained in:
yucheng
commit
a5f1a42a86
1379 changed files with 75471 additions and 17012 deletions
|
|
@ -1291,7 +1291,7 @@ jobs:
|
|||
name: Install MCP SDK1 peer
|
||||
command: |
|
||||
uv venv --python 3.12 .venv-mcp-peer
|
||||
uv pip install --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
uv pip install --no-config --exclude-newer "3 days" --python .venv-mcp-peer 'mcp==1.28.1' 'langchain-mcp-adapters==0.2.1'
|
||||
echo "export MCP_TEST_PEER_PYTHON=$PWD/.venv-mcp-peer/bin/python" >> "$BASH_ENV"
|
||||
- when:
|
||||
condition:
|
||||
|
|
|
|||
|
|
@ -33,7 +33,7 @@ while IFS= read -r file || [ -n "$file" ]; do
|
|||
*) has_backend=true ;;
|
||||
esac
|
||||
case "$file" in
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json | model_prices_and_context_window.schema.json)
|
||||
model_prices_and_context_window.json | litellm/model_prices_and_context_window_backup.json)
|
||||
has_cost_map=true ;;
|
||||
tests/unit/proxy/*) : ;;
|
||||
*) outside_cost_map_set=true ;;
|
||||
|
|
|
|||
4
.github/CODEOWNERS
vendored
4
.github/CODEOWNERS
vendored
|
|
@ -1,4 +1,4 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
|
|
|
|||
|
|
@ -1,319 +0,0 @@
|
|||
import asyncio
|
||||
import aiohttp
|
||||
import json
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
# Asynchronously fetch data from a given URL
|
||||
async def fetch_data(url):
|
||||
try:
|
||||
# Create an asynchronous session
|
||||
async with aiohttp.ClientSession() as session:
|
||||
# Send a GET request to the URL
|
||||
async with session.get(url) as resp:
|
||||
# Raise an error if the response status is not OK
|
||||
resp.raise_for_status()
|
||||
# Parse the response JSON
|
||||
resp_json = await resp.json()
|
||||
print("Fetch the data from URL.")
|
||||
# Return the 'data' field from the JSON response
|
||||
return resp_json['data']
|
||||
except Exception as e:
|
||||
# Print an error message if fetching data fails
|
||||
print("Error fetching data from URL:", e)
|
||||
return None
|
||||
|
||||
|
||||
FRIENDLI_API_URL = "https://api.friendli.ai/serverless/v1/models"
|
||||
FRIENDLI_PROVIDER = "friendliai"
|
||||
|
||||
INHERITABLE_BASE_KEYS = (
|
||||
"supports_pdf_input",
|
||||
"supports_assistant_prefill",
|
||||
"supports_adaptive_thinking",
|
||||
"supports_output_config",
|
||||
)
|
||||
|
||||
REASONING_EFFORT_LEVEL_ORDER = ("none", "minimal", "low", "medium", "high", "xhigh", "max")
|
||||
|
||||
|
||||
def _find_base_model_entry(base_model: str, local_data: dict) -> str | None:
|
||||
if not base_model:
|
||||
return None
|
||||
bm_tail = base_model.split("/")[-1].lower()
|
||||
if base_model in local_data:
|
||||
return base_model
|
||||
for key in local_data:
|
||||
if key.startswith("sample_spec") or key == "fallback_generalizations":
|
||||
continue
|
||||
if key.split("/")[-1].lower() == bm_tail:
|
||||
return key
|
||||
return None
|
||||
|
||||
|
||||
def _reasoning_effort_levels(reasoning_options: list) -> list:
|
||||
offered = {
|
||||
val
|
||||
for opt in reasoning_options or []
|
||||
if opt.get("type") == "effort"
|
||||
for val in opt.get("values", [])
|
||||
}
|
||||
return [level for level in REASONING_EFFORT_LEVEL_ORDER if level in offered]
|
||||
|
||||
|
||||
def _valid_token_price(value: object) -> bool:
|
||||
try:
|
||||
price = float(value) # pyright: ignore[reportArgumentType] # non-numeric values are rejected via the except
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
return math.isfinite(price) and price >= 0
|
||||
|
||||
|
||||
def _has_valid_token_prices(pricing: dict | None) -> bool:
|
||||
prices = pricing or {}
|
||||
return _valid_token_price(prices.get("input")) and _valid_token_price(prices.get("output"))
|
||||
|
||||
|
||||
def _pricing(pricing: dict) -> dict:
|
||||
out: dict[str, Any] = {}
|
||||
if not pricing:
|
||||
return out
|
||||
if "input" in pricing:
|
||||
out["input_cost_per_token"] = float(pricing["input"])
|
||||
if "output" in pricing:
|
||||
out["output_cost_per_token"] = float(pricing["output"])
|
||||
if "input_cache_read" in pricing and pricing["input_cache_read"] is not None:
|
||||
out["cache_read_input_token_cost"] = float(pricing["input_cache_read"])
|
||||
return out
|
||||
|
||||
|
||||
def _modality_flags(input_mods: list) -> dict:
|
||||
mods = input_mods or []
|
||||
has_image = "image" in mods
|
||||
return {
|
||||
"supports_vision": has_image,
|
||||
"supports_image_input": has_image,
|
||||
"supports_video_input": "video" in mods,
|
||||
}
|
||||
|
||||
|
||||
def transform_friendli_data(data: list, local_data: dict) -> dict:
|
||||
transformed: dict[str, dict] = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for model in data:
|
||||
# An unpriced row must never wholesale-replace an already priced local entry:
|
||||
# missing prices cost-calculate as zero, silently zeroing tracked spend
|
||||
if not _has_valid_token_prices(model.get("pricing")):
|
||||
continue
|
||||
model_id = model["id"]
|
||||
base_model = model.get("base_model") or ""
|
||||
entry: dict[str, Any] = {
|
||||
"litellm_provider": FRIENDLI_PROVIDER,
|
||||
}
|
||||
|
||||
base_key = _find_base_model_entry(base_model, local_data)
|
||||
if base_key:
|
||||
base_entry = local_data[base_key]
|
||||
for k in INHERITABLE_BASE_KEYS:
|
||||
if k in base_entry:
|
||||
entry[k] = base_entry[k]
|
||||
|
||||
ctx = model.get("context_length")
|
||||
if ctx is not None:
|
||||
entry["max_input_tokens"] = int(ctx)
|
||||
max_out = model.get("max_completion_tokens")
|
||||
if max_out is not None:
|
||||
entry["max_output_tokens"] = int(max_out)
|
||||
entry["max_tokens"] = int(max_out)
|
||||
|
||||
pricing = _pricing(model.get("pricing", {}))
|
||||
entry.update(pricing)
|
||||
entry["supports_prompt_caching"] = "cache_read_input_token_cost" in pricing
|
||||
|
||||
reasoning = model.get("reasoning") is True
|
||||
entry["supports_reasoning"] = reasoning
|
||||
if reasoning:
|
||||
entry["reasoning_effort_levels"] = _reasoning_effort_levels(
|
||||
model.get("reasoning_options", [])
|
||||
)
|
||||
|
||||
func = model.get("functionality", {})
|
||||
entry["supports_function_calling"] = func.get("tool_call") is True
|
||||
entry["supports_parallel_function_calling"] = func.get("parallel_tool_call") is True
|
||||
is_struct = func.get("structured_output") is True
|
||||
entry["supports_response_schema"] = is_struct
|
||||
entry["supports_native_structured_output"] = is_struct
|
||||
entry["supports_system_messages"] = func.get("system_messages") is True
|
||||
entry["supports_tool_choice"] = func.get("tool_choice") is True
|
||||
|
||||
entry.update(_modality_flags(model.get("input_modalities", [])))
|
||||
|
||||
entry["mode"] = model.get("mode", "chat")
|
||||
|
||||
desc = model.get("description")
|
||||
if desc:
|
||||
entry["comment"] = desc
|
||||
|
||||
dep = model.get("deprecation_date")
|
||||
if dep:
|
||||
entry["deprecation_date"] = dep.split("T")[0]
|
||||
|
||||
entry["source"] = FRIENDLI_API_URL
|
||||
|
||||
transformed[f"{FRIENDLI_PROVIDER}/{model_id}"] = entry
|
||||
return transformed
|
||||
|
||||
# Synchronize local data with remote data
|
||||
def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
|
||||
# Update existing keys in local_data with values from remote_data
|
||||
# (replace_keys entries are swapped wholesale so a field the remote catalog
|
||||
# dropped, e.g. cache pricing, cannot survive as a stale value)
|
||||
for key in (set(local_data) & set(remote_data)):
|
||||
if key in replace_keys:
|
||||
local_data[key] = remote_data[key]
|
||||
else:
|
||||
local_data[key].update(remote_data[key])
|
||||
|
||||
# Add new keys from remote_data to local_data
|
||||
for key in (set(remote_data) - set(local_data)):
|
||||
local_data[key] = remote_data[key]
|
||||
|
||||
# Write data to the json file
|
||||
def write_to_file(file_path, data):
|
||||
try:
|
||||
# Open the file in write mode
|
||||
with open(file_path, "w") as file:
|
||||
# Dump the data as JSON into the file
|
||||
json.dump(data, file, indent=4)
|
||||
print("Values updated successfully.")
|
||||
except Exception as e:
|
||||
# Print an error message if writing to file fails
|
||||
print("Error updating JSON file:", e)
|
||||
|
||||
# Update the existing models and add the missing models for OpenRouter
|
||||
def transform_openrouter_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Add the fields 'max_tokens' and 'input_cost_per_token'
|
||||
obj = {
|
||||
"max_tokens": row["context_length"],
|
||||
"input_cost_per_token": float(row["pricing"]["prompt"]),
|
||||
}
|
||||
|
||||
# Add 'max_output_tokens' as a field if it is not None
|
||||
if "top_provider" in row and "max_completion_tokens" in row["top_provider"] and row["top_provider"]["max_completion_tokens"] is not None:
|
||||
obj['max_output_tokens'] = int(row["top_provider"]["max_completion_tokens"])
|
||||
|
||||
# Add the field 'output_cost_per_token'
|
||||
obj.update({
|
||||
"output_cost_per_token": float(row["pricing"]["completion"]),
|
||||
})
|
||||
|
||||
# Add field 'input_cost_per_image' if it exists and is non-zero
|
||||
if "pricing" in row and "image" in row["pricing"] and float(row["pricing"]["image"]) != 0.0:
|
||||
obj['input_cost_per_image'] = float(row["pricing"]["image"])
|
||||
|
||||
# Add the fields 'litellm_provider' and 'mode'
|
||||
obj.update({
|
||||
"litellm_provider": "openrouter",
|
||||
"mode": "chat"
|
||||
})
|
||||
|
||||
# Add the 'supports_vision' field if the modality is 'multimodal'
|
||||
if row.get('architecture', {}).get('modality') == 'multimodal':
|
||||
obj['supports_vision'] = True
|
||||
|
||||
# Use a composite key to store the transformed object
|
||||
transformed[f'openrouter/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
# Update the existing models and add the missing models for Vercel AI Gateway
|
||||
def transform_vercel_ai_gateway_data(data):
|
||||
transformed = {}
|
||||
if not data:
|
||||
return transformed
|
||||
for row in data:
|
||||
# Rows without token pricing or token limits (video/embedding models) previously KeyError'd the whole sync
|
||||
if any(row.get(k) is None for k in ("context_window", "max_tokens")) or any(
|
||||
row.get("pricing", {}).get(k) is None for k in ("input", "output")
|
||||
):
|
||||
continue
|
||||
obj = {
|
||||
"max_tokens": row["context_window"],
|
||||
"input_cost_per_token": float(row["pricing"]["input"]),
|
||||
"output_cost_per_token": float(row["pricing"]["output"]),
|
||||
'max_output_tokens': row['max_tokens'],
|
||||
'max_input_tokens': row["context_window"],
|
||||
}
|
||||
|
||||
# Handle cache pricing if available
|
||||
if "pricing" in row:
|
||||
if "input_cache_read" in row["pricing"] and row["pricing"]["input_cache_read"] is not None:
|
||||
obj['cache_read_input_token_cost'] = float(f"{float(row['pricing']['input_cache_read']):e}")
|
||||
|
||||
if "input_cache_write" in row["pricing"] and row["pricing"]["input_cache_write"] is not None:
|
||||
obj['cache_creation_input_token_cost'] = float(f"{float(row['pricing']['input_cache_write']):e}")
|
||||
|
||||
mode = "embedding" if "embedding" in row["id"].lower() else "chat"
|
||||
|
||||
obj.update({"litellm_provider": "vercel_ai_gateway", "mode": mode})
|
||||
|
||||
transformed[f'vercel_ai_gateway/{row["id"]}'] = obj
|
||||
|
||||
return transformed
|
||||
|
||||
|
||||
# Load local data from a specified file
|
||||
def load_local_data(file_path):
|
||||
try:
|
||||
# Open the file in read mode
|
||||
with open(file_path, "r") as file:
|
||||
# Load and return the JSON data
|
||||
return json.load(file)
|
||||
except FileNotFoundError:
|
||||
# Print an error message if the file is not found
|
||||
print("File not found:", file_path)
|
||||
return None
|
||||
except json.JSONDecodeError as e:
|
||||
# Print an error message if JSON decoding fails
|
||||
print("Error decoding JSON:", e)
|
||||
return None
|
||||
|
||||
def main():
|
||||
local_file_path = "model_prices_and_context_window.json" # Path to the local data file
|
||||
openrouter_url = "https://openrouter.ai/api/v1/models" # URL to fetch OpenRouter data
|
||||
vercel_ai_gateway_url = "https://ai-gateway.vercel.sh/v1/models" # URL to fetch Vercel AI Gateway data
|
||||
|
||||
# Load local data from file
|
||||
local_data = load_local_data(local_file_path)
|
||||
|
||||
# Fetch OpenRouter data
|
||||
openrouter_data = asyncio.run(fetch_data(openrouter_url))
|
||||
# Transform the fetched OpenRouter data
|
||||
openrouter_data = transform_openrouter_data(openrouter_data)
|
||||
|
||||
# Fetch Vercel AI Gateway data
|
||||
vercel_data = asyncio.run(fetch_data(vercel_ai_gateway_url))
|
||||
# Transform the fetched Vercel AI Gateway data
|
||||
vercel_data = transform_vercel_ai_gateway_data(vercel_data)
|
||||
|
||||
friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
|
||||
friendli_data = transform_friendli_data(friendli_data, local_data)
|
||||
|
||||
# Combine both datasets
|
||||
all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
|
||||
|
||||
# If both local and openrouter data are available, synchronize and save
|
||||
if local_data and all_remote_data:
|
||||
sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
|
||||
write_to_file(local_file_path, local_data)
|
||||
else:
|
||||
print("Failed to fetch model data from either local file or URL.")
|
||||
|
||||
# Entry point of the script
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
name: Updates model_prices_and_context_window.json and Create Pull Request
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "0 0 * * 0" # Run every Sundays at midnight
|
||||
#- cron: "0 0 * * *" # Run daily at midnight
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
auto_update_price_and_context_window:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Update JSON Data
|
||||
run: |
|
||||
uv run --frozen --with 'aiohttp==3.13.3' python ".github/scripts/auto_update_price_and_context_window_file.py"
|
||||
- name: Create Pull Request
|
||||
run: |
|
||||
git add model_prices_and_context_window.json
|
||||
git commit -m "Update model_prices_and_context_window.json file: $(date +'%Y-%m-%d')"
|
||||
gh pr create --title "Update model_prices_and_context_window.json file" \
|
||||
--body "Automated update for model_prices_and_context_window.json" \
|
||||
--head auto-update-price-and-context-window-$(date +'%Y-%m-%d') \
|
||||
--base main
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN }}
|
||||
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
91
.github/workflows/interactions-spec-canary.yml
vendored
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
name: Interactions spec canary
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "17 7 * * *"
|
||||
pull_request:
|
||||
paths:
|
||||
- .github/workflows/interactions-spec-canary.yml
|
||||
- tests/unit/interactions/**
|
||||
- litellm/types/interactions/**
|
||||
workflow_dispatch:
|
||||
|
||||
permissions: {}
|
||||
|
||||
concurrency: interactions-spec-canary
|
||||
|
||||
jobs:
|
||||
interactions-spec-canary:
|
||||
name: interactions-spec-canary
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
permissions:
|
||||
contents: read
|
||||
outputs:
|
||||
compliance: ${{ steps.compliance.outcome }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Python
|
||||
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
|
||||
- name: Install dependencies
|
||||
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
|
||||
|
||||
- name: Run the compliance tests against Google's live spec
|
||||
id: compliance
|
||||
run: |
|
||||
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
||||
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
||||
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
||||
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
||||
|
||||
tracking-issue:
|
||||
name: tracking-issue
|
||||
needs: interactions-spec-canary
|
||||
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
- name: File the tracking issue unless the compliance step passed, close it when it did
|
||||
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
||||
env:
|
||||
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
with:
|
||||
script: |
|
||||
const title = "Interactions spec canary is red";
|
||||
const outcome = process.env.COMPLIANCE_OUTCOME;
|
||||
const runUrl = process.env.RUN_URL;
|
||||
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
|
||||
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
|
||||
const open = found.items.find((issue) => issue.title === title);
|
||||
if (outcome === "success") {
|
||||
if (open === undefined) return;
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
|
||||
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
|
||||
return;
|
||||
}
|
||||
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
|
||||
if (open !== undefined) {
|
||||
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
|
||||
return;
|
||||
}
|
||||
await github.rest.issues.create({
|
||||
...context.repo,
|
||||
title,
|
||||
assignees: ["mateo-berri"],
|
||||
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
|
||||
});
|
||||
65
.github/workflows/sync-together-ai-models.yml
vendored
65
.github/workflows/sync-together-ai-models.yml
vendored
|
|
@ -1,65 +0,0 @@
|
|||
name: Sync Together AI model registry
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 6 * * *"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
jobs:
|
||||
sync_together_ai_models:
|
||||
if: github.repository == 'BerriAI/litellm'
|
||||
runs-on: ubuntu-latest
|
||||
env:
|
||||
BASE_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
ref: ${{ env.BASE_BRANCH }}
|
||||
persist-credentials: false
|
||||
- name: Set up uv
|
||||
uses: ./.github/actions/setup-uv-with-retries
|
||||
with:
|
||||
version: "0.10.9"
|
||||
- name: Look for an already-open sync PR
|
||||
id: existing
|
||||
run: |
|
||||
open_pr="$(gh pr list --repo "$GITHUB_REPOSITORY" --state open --limit 1000 --json headRefName \
|
||||
--jq '[.[].headRefName | select(startswith("litellm_together_registry_sync_"))] | first // empty')"
|
||||
echo "open_pr=$open_pr" >> "$GITHUB_OUTPUT"
|
||||
if [ -n "$open_pr" ]; then
|
||||
echo "An open sync PR already exists on branch $open_pr; skipping this run."
|
||||
fi
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
- name: Run the sync
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
uv run --frozen python scripts/sync_together_ai_models.py --write --pr-body-file "$RUNNER_TEMP/pr_body.md"
|
||||
env:
|
||||
TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
|
||||
- name: Create a pull request when the registry changed
|
||||
if: steps.existing.outputs.open_pr == ''
|
||||
run: |
|
||||
if git diff --quiet; then
|
||||
echo "Registry already in sync; no PR needed."
|
||||
exit 0
|
||||
fi
|
||||
branch="litellm_together_registry_sync_$(date +'%Y-%m-%d')"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$branch"
|
||||
git add model_prices_and_context_window.json \
|
||||
litellm/model_prices_and_context_window_backup.json
|
||||
git commit -m "feat(models): sync together_ai model registry $(date +'%Y-%m-%d')"
|
||||
gh auth setup-git
|
||||
git push origin "$branch"
|
||||
gh pr create --title "feat(models): sync together_ai model registry" \
|
||||
--body-file "$RUNNER_TEMP/pr_body.md" \
|
||||
--head "$branch" \
|
||||
--base "$BASE_BRANCH"
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GH_TOKEN || github.token }}
|
||||
|
|
@ -45,7 +45,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
|
|
@ -132,7 +132,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
steps:
|
||||
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
||||
with:
|
||||
|
|
|
|||
8
.github/workflows/test-merge-smoke.yml
vendored
8
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -33,7 +33,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
|
||||
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "3.15"]
|
||||
env:
|
||||
LITELLM_LOCAL_MODEL_COST_MAP: "True"
|
||||
steps:
|
||||
|
|
@ -76,6 +76,12 @@ jobs:
|
|||
- name: Import and CLI checks
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py cli
|
||||
|
||||
- name: Routing and UUID compatibility
|
||||
run: >-
|
||||
$IN_NETNS .venv/bin/python -m pytest
|
||||
tests/unit/router_strategy/test_complexity_router.py::TestCustomDimensions
|
||||
tests/unit/test_uuid_helper.py -q
|
||||
|
||||
- name: Proxy startup check
|
||||
run: $IN_NETNS .venv/bin/python .github/scripts/run_merge_smoke.py proxy-startup --diagnostics-dir "${RUNNER_TEMP}/smoke-diagnostics"
|
||||
|
||||
|
|
|
|||
3
.github/workflows/test-unit.yml
vendored
3
.github/workflows/test-unit.yml
vendored
|
|
@ -216,10 +216,12 @@ jobs:
|
|||
|
||||
- shard: llms-providers
|
||||
test-path: |-
|
||||
tests/unit/llms/test_api_base_trailing_slash_routes.py
|
||||
tests/unit/llms/test_cache_control_and_reasoning.py
|
||||
tests/unit/llms/test_custom_llm.py
|
||||
tests/unit/llms/test_file_content_block.py
|
||||
tests/unit/llms/test_file_search_responses.py
|
||||
tests/unit/llms/test_get_complete_url_trailing_slash.py
|
||||
tests/unit/llms/test_lifecycle_fix.py
|
||||
tests/unit/llms/test_oss_decision.py
|
||||
tests/unit/llms/test_polling_url_origin_match.py
|
||||
|
|
@ -423,6 +425,7 @@ jobs:
|
|||
tests/unit/realtime_api
|
||||
tests/unit/rerank_api
|
||||
tests/unit/search
|
||||
tests/unit/text_completions
|
||||
tests/unit/vector_stores
|
||||
tests/unit/videos
|
||||
python-version: "3.12"
|
||||
|
|
|
|||
11
Dockerfile
11
Dockerfile
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -61,6 +61,7 @@ RUN apk add --no-cache \
|
|||
python-3.13-dev \
|
||||
rust \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
|
|
@ -141,7 +142,7 @@ USER root
|
|||
RUN echo "https://packages.wolfi.dev/os" >> /etc/apk/repositories
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
|
|||
|
||||
### Verify Docker Image Signatures
|
||||
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
|
||||
|
||||
**Verify using the pinned commit hash (recommended):**
|
||||
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
|
||||
|
||||
```bash
|
||||
cosign verify \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
|
||||
ghcr.io/berriai/litellm:<release-tag>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ To qualify for a bounty, reports must include clear reproduction steps, a reprod
|
|||
- Audit Logs with retention policy
|
||||
- Control Allowed IP Addresses that can access your Cloud LiteLLM Instance
|
||||
|
||||
For security inquiries, please contact us at support@berri.ai
|
||||
For security inquiries, please contact us at security@berri.ai
|
||||
|
||||
#### Supported data regions for LiteLLM Cloud
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
|
||||
FROM $UV_IMAGE AS uvbin
|
||||
|
|
@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -109,7 +109,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -124,7 +124,12 @@ ENV HOME=/home/nonroot \
|
|||
PYTHONUNBUFFERED=1 \
|
||||
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
||||
|
||||
COPY --from=builder --chown=nonroot:nonroot /app /app
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/backend /app/backend
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
|
||||
RUN find /app/.venv -type f -path "*/tornado/test/*" -delete && \
|
||||
|
|
|
|||
|
|
@ -4,8 +4,7 @@ Every pull request whose diff against its merge base touches one of the two cost
|
|||
checks: the files parse and the backup copy matches the root file. A pull request that leaves both untouched
|
||||
skips them, since merging it keeps the base branch's copies and its head tree only carries whatever state the
|
||||
branch was cut from. Pull requests from the cost map sync bot (branches named litellm_cost_map_sync_*) always
|
||||
get the file checks and additionally may only touch those two files or the legacy schema
|
||||
path and may only add or update models.
|
||||
get the file checks and additionally may only touch those two files and may only add or update models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -21,7 +20,6 @@ from typing import Final
|
|||
|
||||
COST_MAP_PATH: Final = "model_prices_and_context_window.json"
|
||||
BACKUP_PATH: Final = "litellm/model_prices_and_context_window_backup.json"
|
||||
LEGACY_SCHEMA_PATH: Final = "model_prices_and_context_window.schema.json"
|
||||
GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
|
||||
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
|
||||
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
|
||||
|
|
@ -110,7 +108,7 @@ def _bot_failures(base: Snapshot, head_map: CostMap, changed_files: Sequence[str
|
|||
*(
|
||||
f"bot PRs may only change the cost map files, not {path}"
|
||||
for path in changed_files
|
||||
if path not in GUARDED_PATHS and path != LEGACY_SCHEMA_PATH
|
||||
if path not in GUARDED_PATHS
|
||||
),
|
||||
*(f"bot PRs may not remove models: {key}" for key in base_map if key not in head_map),
|
||||
*(f"bot PRs may not remove fields: {ref}" for ref in removed_fields),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,76 @@
|
|||
# Use your ChatGPT subscription through LiteLLM
|
||||
|
||||
Sign into ChatGPT in Codex or Pi and send model requests through your existing LiteLLM proxy. The client keeps your login and refreshes it. Each request uses that client's ChatGPT account, while a separate LiteLLM virtual key controls gateway access and usage attribution.
|
||||
|
||||
An administrator adds a `chatgpt/` model using the usual model configuration. No ChatGPT login needs to be added to the proxy. This uses the same separate-key convention as provider passthrough, including Claude Code subscription forwarding.
|
||||
|
||||
## Add a model to the proxy
|
||||
|
||||
Reuse your existing proxy. If you need one, follow the [proxy quickstart](https://docs.litellm.ai/docs/proxy/docker_quick_start). Merge this entry into its model list and restart the proxy:
|
||||
|
||||
```yaml
|
||||
model_list:
|
||||
- model_name: gpt-6-luna
|
||||
litellm_params:
|
||||
model: chatgpt/gpt-6-luna
|
||||
model_info:
|
||||
mode: responses
|
||||
```
|
||||
|
||||
Use a model available to the caller's subscription. The virtual key must allow the exposed model name. Leave saved ChatGPT logins and API credentials unset for a client-only deployment. Explicit cross-provider fallbacks can incur separate API charges; do not configure them for subscription-only traffic.
|
||||
|
||||
## Configure Codex
|
||||
|
||||
Use your existing LiteLLM URL and a virtual key assigned to your gateway user. Export that key as `LITELLM_API_KEY` in the environment that launches Codex. Keep the value out of committed configuration.
|
||||
|
||||
Add this provider to `~/.codex/config.toml`, preserving your other settings:
|
||||
|
||||
```toml
|
||||
model = "gpt-6-luna"
|
||||
model_provider = "litellm_subscription"
|
||||
|
||||
[model_providers.litellm_subscription]
|
||||
name = "LiteLLM subscription"
|
||||
base_url = "https://litellm.example.com/v1"
|
||||
wire_api = "responses"
|
||||
supports_websockets = false
|
||||
requires_openai_auth = true
|
||||
env_http_headers = { "x-litellm-api-key" = "LITELLM_API_KEY" }
|
||||
```
|
||||
|
||||
Replace `https://litellm.example.com` with your proxy URL. A local proxy normally uses `http://localhost:4000/v1`. Launch Codex, sign into ChatGPT if needed, and send a prompt with a model available to your subscription. The selected model must match the name exposed by the gateway.
|
||||
|
||||
For a managed workspace rollout, use OpenAI's [gateway configuration guide](https://learn.chatgpt.com/docs/enterprise/sign-in-with-chatgpt-through-a-gateway).
|
||||
|
||||
## Native compatibility and Pi
|
||||
|
||||
For a native Codex model catalog and request format, change Codex's `base_url` to `https://litellm.example.com/backend-api/codex`. This route uses the same credentials but does not need a model deployment or resolve model aliases. Restricted keys must allow the upstream model name.
|
||||
|
||||
### Configure Pi
|
||||
|
||||
Sign into the built-in `openai-codex` provider in Pi. Merge this override into `~/.pi/agent/models.json` and export your LiteLLM virtual key as `LITELLM_API_KEY` before launching Pi:
|
||||
|
||||
```json
|
||||
{
|
||||
"providers": {
|
||||
"openai-codex": {
|
||||
"baseUrl": "https://litellm.example.com/backend-api/codex",
|
||||
"headers": {
|
||||
"x-litellm-api-key": "LITELLM_API_KEY"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Keep Pi's subscription login as the provider credential. The header value names an environment variable, not a key to paste into the file.
|
||||
|
||||
## Verify account usage
|
||||
|
||||
Send a prompt and check the completed request in LiteLLM's Logs page. Its gateway user/key and token counts should match the caller. Give each person their own LiteLLM virtual key for separate attribution. The gateway credential and ChatGPT login are independently authenticated; LiteLLM does not establish that both identities belong to the same person.
|
||||
|
||||
The native routes are `GET /backend-api/codex/models` and `POST /backend-api/codex/responses`. Model requests use the upstream model name, so a restricted LiteLLM key must allow that name. OpenAI's model catalog is returned in its native format; seeing a model there does not override gateway permissions.
|
||||
|
||||
The proxy forwards the ChatGPT bearer and account header to the fixed OpenAI Codex endpoint. It removes gateway credentials and cookies from the upstream request and redacts credential headers in logging callbacks. The client remains responsible for refresh and reauthentication. Native passthrough returns authentication and quota errors without switching subscriptions. Standard model routes retain the caller login across ChatGPT retries; explicitly configured cross-provider fallbacks use their own credentials. An explicit workspace header is preserved for OpenAI to validate.
|
||||
|
||||
This flow does not connect accounts through the dashboard or pool subscription allowances. Use a Codex build that supports ChatGPT authentication with a custom model provider. The integration forwards HTTP and SSE requests; it does not add native WebSocket or other Codex backend routes.
|
||||
|
|
@ -1,4 +1,4 @@
|
|||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
|
||||
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
|
||||
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
|
||||
-----END PUBLIC KEY-----
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base image for building
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
|
||||
# Runtime image
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -59,6 +59,7 @@ RUN apk add --no-cache \
|
|||
python-3.13 \
|
||||
python-3.13-dev \
|
||||
openssl \
|
||||
libssl3 \
|
||||
openssl-dev \
|
||||
nodejs \
|
||||
npm \
|
||||
|
|
@ -122,7 +123,7 @@ ENV LITELLM_RELEASE_TAG=${LITELLM_RELEASE_TAG}
|
|||
USER root
|
||||
|
||||
# node (without npm) is required by the prisma CLI at runtime
|
||||
RUN apk add --no-cache bash openssl tzdata nodejs python-3.13 libsndfile libevent
|
||||
RUN apk add --no-cache bash openssl libssl3 tzdata nodejs python-3.13 libsndfile libevent
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
WORKDIR /app
|
||||
|
|
|
|||
|
|
@ -1,8 +1,8 @@
|
|||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# Base images
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Pinned by digest like the other base images; bump explicitly on Node upgrades.
|
||||
ARG UI_BUILD_IMAGE=node:24.19-alpine3.24@sha256:d32cdf619f63fe0471182d08996dd516c6275bb5fd31ae06e55a570bd9e1ad43
|
||||
|
|
@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
|
|||
coreutils \
|
||||
curl \
|
||||
openssl \
|
||||
libssl3 \
|
||||
libsndfile \
|
||||
nodejs \
|
||||
npm && break || sleep 5; \
|
||||
|
|
@ -134,7 +135,7 @@ RUN for i in 1 2 3; do \
|
|||
apk upgrade --no-cache && break || sleep 5; \
|
||||
done && \
|
||||
for i in 1 2 3; do \
|
||||
apk add --no-cache python-3.13 bash openssl tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
apk add --no-cache python-3.13 bash openssl libssl3 tzdata libsndfile nodejs libevent && break || sleep 5; \
|
||||
done
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ RUN uv venv --python python && \
|
|||
"google-cloud-aiplatform==1.133.0" \
|
||||
"google-genai==1.37.0" \
|
||||
"anthropic[vertex]==0.84.0" \
|
||||
"grpcio==1.78.0" \
|
||||
"grpcio==1.84.1" \
|
||||
"prometheus-client==0.20.0" \
|
||||
"langfuse==2.59.7" \
|
||||
"opentelemetry-api==1.28.0" \
|
||||
|
|
|
|||
|
|
@ -15,9 +15,10 @@ sys.path.insert(
|
|||
) # Adds the parent directory to the system path
|
||||
import json
|
||||
import sys
|
||||
from typing import Any, List, Literal, Optional
|
||||
from typing import Final, List, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -29,11 +30,13 @@ from litellm.llms.custom_httpx.http_handler import (
|
|||
get_async_httpx_client,
|
||||
httpxSpecialProvider,
|
||||
)
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_helpers import should_proceed_based_on_metadata
|
||||
from litellm.types.guardrails import GuardrailEventHooks
|
||||
|
||||
GUARDRAIL_NAME = "aporia"
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AporiaGuardrail(CustomGuardrail):
|
||||
|
|
@ -68,7 +71,7 @@ class AporiaGuardrail(CustomGuardrail):
|
|||
async def prepare_aporia_request(
|
||||
self, new_messages: List[dict], response_string: Optional[str] = None
|
||||
) -> dict:
|
||||
data: dict[str, Any] = {}
|
||||
data: dict[str, object] = {}
|
||||
if new_messages is not None:
|
||||
data["messages"] = new_messages
|
||||
if response_string is not None:
|
||||
|
|
@ -122,8 +125,10 @@ class AporiaGuardrail(CustomGuardrail):
|
|||
verbose_proxy_logger.debug("Aporia AI response: %s", response.text)
|
||||
if response.status_code == 200:
|
||||
# check if the response was flagged
|
||||
_json_response = response.json()
|
||||
action: str = _json_response.get(
|
||||
_json_response: Final = decoded_json_object(
|
||||
_DECODED_JSON.validate_python(response.json())
|
||||
)
|
||||
action: Final = _json_response.get(
|
||||
"action"
|
||||
) # possible values are modify, passthrough, block, rephrase
|
||||
if action == "block":
|
||||
|
|
|
|||
|
|
@ -7,9 +7,10 @@
|
|||
## This accepts a list of user id's for whom calls will be rejected
|
||||
|
||||
|
||||
from typing import Literal, Optional
|
||||
from typing import Final, Literal, Optional
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._internal_context import with_service_target
|
||||
|
|
@ -19,6 +20,9 @@ from litellm.integrations.custom_logger import CustomLogger
|
|||
from litellm.proxy._types import LiteLLM_EndUserTable, UserAPIKeyAuth
|
||||
from litellm.proxy.common_utils.user_api_key_cache import AUTH_OBJECTS_TARGET
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.table_repositories import EndUserRepository
|
||||
|
||||
_REQUEST_BODY_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ENTERPRISE_BlockedUserList(CustomLogger):
|
||||
|
|
@ -94,15 +98,14 @@ class ENTERPRISE_BlockedUserList(CustomLogger):
|
|||
)
|
||||
if end_user_cache_obj is None and self.prisma_client is not None:
|
||||
# check db
|
||||
end_user_obj = (
|
||||
await self.prisma_client.db.litellm_endusertable.find_unique(
|
||||
where={"user_id": user}
|
||||
)
|
||||
)
|
||||
end_user_id: Final = _REQUEST_BODY_VALUE.validate_python(user)
|
||||
end_user_obj = await EndUserRepository(
|
||||
self.prisma_client
|
||||
).table.find_unique(where={"user_id": end_user_id})
|
||||
if end_user_obj is None: # user not in db - assume not blocked
|
||||
end_user_obj = LiteLLM_EndUserTable(user_id=user, blocked=False)
|
||||
cache.set_cache(key=cache_key, value=end_user_obj, ttl=60)
|
||||
if end_user_obj is not None and end_user_obj.blocked is True:
|
||||
if end_user_obj.blocked is True:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
|
|
|
|||
|
|
@ -6,7 +6,10 @@
|
|||
# +-----------------------------------------------+
|
||||
# Thank you users! We ❤️ you! - Krrish & Ishaan
|
||||
|
||||
from typing import Final
|
||||
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -15,6 +18,8 @@ from litellm.proxy._types import UserAPIKeyAuth
|
|||
from litellm.proxy.guardrails._content_utils import iter_message_text
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_THRESHOLD_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
||||
user_api_key_cache = None
|
||||
|
|
@ -69,9 +74,11 @@ class ENTERPRISE_GoogleTextModeration(CustomLogger):
|
|||
f"{category}_confidence_threshold",
|
||||
default_confidence_threshold,
|
||||
)
|
||||
set_confidence_value = getattr(
|
||||
self,
|
||||
f"{category}_confidence_threshold",
|
||||
set_confidence_value = _THRESHOLD_VALUE.validate_python(
|
||||
getattr(
|
||||
self,
|
||||
f"{category}_confidence_threshold",
|
||||
)
|
||||
)
|
||||
verbose_proxy_logger.info(
|
||||
f"Google Text Moderation: {category}_confidence_threshold: {set_confidence_value}"
|
||||
|
|
|
|||
|
|
@ -1,15 +1,20 @@
|
|||
# this is an example endpoint to receive data from litellm
|
||||
from typing import Final
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Request
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
_LOGGED_EVENT: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
@app.post("/log-event")
|
||||
async def log_event(request: Request):
|
||||
try:
|
||||
print("Received /log-event request") # noqa
|
||||
# Assuming the incoming request has JSON data
|
||||
data = await request.json()
|
||||
data: Final = _LOGGED_EVENT.validate_python(await request.json())
|
||||
print("Received request data:") # noqa
|
||||
print(data) # noqa
|
||||
|
||||
|
|
|
|||
|
|
@ -8,25 +8,30 @@
|
|||
## This provides an LLM Guard Integration for content moderation on the proxy
|
||||
|
||||
import asyncio
|
||||
from typing import Final, Optional
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import aiohttp
|
||||
from fastapi import HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.utils import CallTypesLiteral
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class _ENTERPRISE_LLMGuard(CustomLogger):
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
self,
|
||||
mock_testing: bool = False,
|
||||
mock_redacted_text: Optional[dict] = None,
|
||||
mock_redacted_text: Mapping[str, object] | None = None,
|
||||
):
|
||||
self.mock_redacted_text = mock_redacted_text
|
||||
self.llm_guard_mode = litellm.llm_guard_mode
|
||||
|
|
@ -66,7 +71,9 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
|
|||
async with session.post(
|
||||
analyze_url, json={"prompt": text}
|
||||
) as response:
|
||||
redacted_text = await response.json()
|
||||
redacted_text = _DECODED_JSON.validate_python(
|
||||
await response.json()
|
||||
)
|
||||
verbose_proxy_logger.debug(
|
||||
f"LLM Guard: Received response - {redacted_text}"
|
||||
)
|
||||
|
|
@ -77,12 +84,13 @@ class _ENTERPRISE_LLMGuard(CustomLogger):
|
|||
"error": f"Invalid content moderation response: {redacted_text}"
|
||||
},
|
||||
)
|
||||
if redacted_text.get("is_valid", None) is False:
|
||||
redacted_response: Final = decoded_json_object(redacted_text)
|
||||
if redacted_response.get("is_valid", None) is False:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={"error": "Violated content safety policy"},
|
||||
)
|
||||
sanitized_prompt = redacted_text.get("sanitized_prompt")
|
||||
sanitized_prompt: Final = redacted_response.get("sanitized_prompt")
|
||||
return sanitized_prompt if isinstance(sanitized_prompt, str) else text
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.exception(
|
||||
|
|
|
|||
|
|
@ -2,6 +2,8 @@ import os
|
|||
|
||||
from fastapi import HTTPException, status
|
||||
|
||||
from litellm.proxy._types import MODEL_LIST_ROUTES
|
||||
|
||||
|
||||
class EnterpriseRouteChecks:
|
||||
@staticmethod
|
||||
|
|
@ -43,7 +45,7 @@ class EnterpriseRouteChecks:
|
|||
|
||||
# Routes that should remain accessible even when LLM API endpoints are disabled.
|
||||
# These are read-only model listing routes needed by the Admin UI.
|
||||
LLM_API_EXEMPT_ROUTES = ["/models", "/v1/models"]
|
||||
LLM_API_EXEMPT_ROUTES = MODEL_LIST_ROUTES
|
||||
|
||||
@staticmethod
|
||||
def should_call_route(route: str):
|
||||
|
|
|
|||
|
|
@ -1040,8 +1040,8 @@ class CheckBatchCost:
|
|||
from litellm.proxy.proxy_server import proxy_config
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
api_key: Final = getattr(job, "api_key", None)
|
||||
team_id: Final = getattr(job, "team_id", None)
|
||||
api_key: Final = job.api_key if hasattr(job, "api_key") else None
|
||||
team_id: Final = job.team_id if hasattr(job, "team_id") else None
|
||||
has_key_row: Final = api_key is not None and is_valid_sha256_hash(api_key)
|
||||
try:
|
||||
key_row: Final = (
|
||||
|
|
|
|||
|
|
@ -44,6 +44,7 @@ from litellm.constants import (
|
|||
)
|
||||
from litellm.files.types import FileRetrieveCallOptions, FileRetrieveProvider
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.litellm_core_utils.core_helpers import require_attributes
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR
|
||||
from litellm.litellm_core_utils.internal_call_metadata import requests_background
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -133,7 +134,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.proxy.utils import PrismaClient as _PrismaClient
|
||||
|
||||
Span = Union[_Span, Any]
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
PrismaClient = _PrismaClient
|
||||
else:
|
||||
|
|
@ -490,6 +491,14 @@ _PROVIDER_FILE_ID_KEYS: Final = tuple(dict.fromkeys(chain.from_iterable(_PROVIDE
|
|||
_PROVIDER_FILE_RETRIEVE_RETRY_DELAYS_SECONDS: Final = (0.5, 1.0, 2.0)
|
||||
|
||||
|
||||
class _SupportsKeyedGet(Protocol):
|
||||
def get(self, key: str, default: object, /) -> object: ...
|
||||
|
||||
|
||||
def _keyed_get(source: object, key: str, default: object = None) -> object:
|
||||
return require_attributes(source, _SupportsKeyedGet, "get").get(key, default)
|
||||
|
||||
|
||||
class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
||||
# Class variables or attributes
|
||||
def __init__(
|
||||
|
|
@ -1611,7 +1620,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
return holders if holders or "input_file_id" in request else healthy_deployments
|
||||
|
||||
async def async_pre_call_deployment_hook(
|
||||
self, kwargs: Dict[str, Any], call_type: Optional[CallTypes]
|
||||
self, kwargs: dict[str, object], call_type: Optional[CallTypes]
|
||||
) -> Optional[dict]:
|
||||
"""
|
||||
Allow modifying the request just before it's sent to the deployment.
|
||||
|
|
@ -1623,11 +1632,11 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
model_file_id_mapping = cast(Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping"))
|
||||
# model_info may be at top-level or nested under litellm_metadata
|
||||
# (batch/file operations use litellm_metadata)
|
||||
model_id = cast(Optional[str], kwargs.get("model_info", {}).get("id", None))
|
||||
model_id = cast(Optional[str], _keyed_get(kwargs.get("model_info", {}), "id"))
|
||||
if model_id is None:
|
||||
model_id = cast(
|
||||
Optional[str],
|
||||
kwargs.get("litellm_metadata", {}).get("model_info", {}).get("id", None),
|
||||
_keyed_get(_keyed_get(kwargs.get("litellm_metadata", {}), "model_info", {}), "id"),
|
||||
)
|
||||
for accessor_key in accessor_keys:
|
||||
input_file_id = cast(Optional[str], kwargs.get(accessor_key))
|
||||
|
|
@ -1824,7 +1833,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
),
|
||||
)
|
||||
|
||||
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span) -> dict:
|
||||
async def get_model_file_id_mapping(self, file_ids: Sequence[str], litellm_parent_otel_span: Span | None) -> dict:
|
||||
"""
|
||||
Get model-specific file IDs for a list of proxy file IDs.
|
||||
Returns a dictionary mapping litellm_proxy/ file_id -> model_id -> model_file_id
|
||||
|
|
@ -1866,7 +1875,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
llm_router: Optional[Router],
|
||||
_create_file_request: CreateFileRequest,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
) -> List[OpenAIFileObject]:
|
||||
if llm_router is None:
|
||||
raise Exception("LLM Router not initialized. Ensure models added to proxy.")
|
||||
|
|
@ -1882,7 +1891,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
create_file_request: CreateFileRequest,
|
||||
llm_router: Router,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> OpenAIFileObject:
|
||||
responses = await self.create_file_for_each_model(
|
||||
|
|
@ -1949,7 +1958,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
file_objects: List[OpenAIFileObject],
|
||||
create_file_request: CreateFileRequest,
|
||||
internal_usage_cache: InternalUsageCache,
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
target_model_names_list: List[str],
|
||||
) -> OpenAIFileObject:
|
||||
## GET THE FILE TYPE FROM THE CREATE FILE REQUEST
|
||||
|
|
@ -2652,7 +2661,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints):
|
|||
# Continue with other files even if one fails
|
||||
continue
|
||||
|
||||
def _get_content_type_from_file_object(self, file_object: Optional[Any]) -> str:
|
||||
def _get_content_type_from_file_object(self, file_object: object) -> str:
|
||||
"""
|
||||
Determine content type from file object.
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import InternalUsageCache as _InternalUsageCache
|
||||
from litellm.proxy.utils import PrismaClient as _PrismaClient
|
||||
|
||||
Span = Union[_Span, Any]
|
||||
Span = _Span
|
||||
InternalUsageCache = _InternalUsageCache
|
||||
PrismaClient = _PrismaClient
|
||||
else:
|
||||
|
|
@ -107,7 +107,7 @@ class _PROXY_LiteLLMManagedVectorStores(
|
|||
llm_router: Router,
|
||||
model: str,
|
||||
request_data: dict[str, object] | VectorStoreCreateOptionalRequestParams,
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
) -> VectorStoreCreateResponse:
|
||||
"""
|
||||
Create a vector store for a specific model.
|
||||
|
|
@ -134,7 +134,7 @@ class _PROXY_LiteLLMManagedVectorStores(
|
|||
create_request: VectorStoreCreateOptionalRequestParams,
|
||||
llm_router: Router,
|
||||
target_model_names_list: List[str],
|
||||
litellm_parent_otel_span: Span,
|
||||
litellm_parent_otel_span: Span | None,
|
||||
user_api_key_dict: UserAPIKeyAuth,
|
||||
) -> VectorStoreCreateResponse:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from litellm.repositories.budget_repository import BudgetRepository
|
|||
from litellm.repositories.object_permission_repository import ObjectPermissionRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.project_repository import ProjectRepository
|
||||
from litellm.repositories.table_repositories import PrismaTableRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
|
@ -73,6 +74,16 @@ def _verification_token_table(
|
|||
return VerificationTokenRepository(prisma_client).table
|
||||
|
||||
|
||||
class _WriterVerificationTokenRepository(PrismaTableRepository["prisma_models.LiteLLM_VerificationToken"]):
|
||||
table_name = "litellm_verificationtoken"
|
||||
|
||||
|
||||
def _writer_verification_token_table(
|
||||
prisma_client: PrismaClient,
|
||||
) -> TableActions["prisma_models.LiteLLM_VerificationToken"]:
|
||||
return _WriterVerificationTokenRepository(prisma_client, use_writer=True).table
|
||||
|
||||
|
||||
def _budget_table(prisma_client: PrismaClient) -> TableActions["prisma_models.LiteLLM_BudgetTable"]:
|
||||
return BudgetRepository(prisma_client).table
|
||||
|
||||
|
|
@ -849,7 +860,7 @@ async def update_project(
|
|||
else None
|
||||
)
|
||||
if current_project is not None and data.team_id != current_project.team_id:
|
||||
mismatched_key_count: Final = await prisma_client.writer_db.litellm_verificationtoken.count(
|
||||
mismatched_key_count: Final = await _writer_verification_token_table(prisma_client).count(
|
||||
where={
|
||||
"project_id": data.project_id,
|
||||
"OR": [{"team_id": {"not": data.team_id}}, {"team_id": None}],
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from collections.abc import Mapping
|
|||
from typing import TYPE_CHECKING, Final, List, Optional, Protocol
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
|
|
@ -37,6 +38,7 @@ if TYPE_CHECKING:
|
|||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
router = APIRouter()
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class ManagedVectorStoreRow(Protocol):
|
||||
|
|
@ -348,11 +350,11 @@ async def update_vector_store(
|
|||
raise HTTPException(status_code=500, detail="Database not connected")
|
||||
|
||||
try:
|
||||
update_data = data.model_dump(exclude_unset=True)
|
||||
vector_store_id: Final[str] = update_data.pop("vector_store_id")
|
||||
update_data = data.model_dump(exclude_unset=True, exclude={"vector_store_id"})
|
||||
vector_store_id: Final = data.vector_store_id
|
||||
if update_data.get("vector_store_metadata") is not None:
|
||||
update_data["vector_store_metadata"] = safe_dumps(
|
||||
update_data["vector_store_metadata"]
|
||||
_DECODED_JSON.validate_python(update_data["vector_store_metadata"])
|
||||
)
|
||||
|
||||
updated = await managed_vector_store_table(prisma_client).update(
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:1d95114038f76513a9ace6fca107d5582b08c65981f81f61cb56bf7fd2ef216d
|
||||
ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base@sha256:360896efb0753a5701c48f860e17130c205eae5a264797aa3e7ea126ee2923a7
|
||||
ARG UV_IMAGE=ghcr.io/astral-sh/uv:0.11.7@sha256:240fb85ab0f263ef12f492d8476aa3a2e4e1e333f7d67fbdd923d00a506a516a
|
||||
# Checksum from https://www.pgbouncer.org/downloads/ (the Wolfi repo only carries 1.24.x)
|
||||
ARG PGBOUNCER_VERSION=1.26.0
|
||||
|
|
@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
|
|||
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
|
||||
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
|
||||
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -125,7 +125,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime
|
|||
USER root
|
||||
|
||||
RUN for i in 1 2 3; do \
|
||||
apk add --no-cache bash openssl tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
apk add --no-cache bash openssl libssl3 tzdata python-3.13 libsndfile libatomic libevent && break; \
|
||||
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
|
||||
sleep 5; \
|
||||
done
|
||||
|
|
@ -140,7 +140,12 @@ ENV HOME=/home/nonroot \
|
|||
PYTHONUNBUFFERED=1 \
|
||||
PRISMA_BINARY_CACHE_DIR=/opt/prisma/binaries
|
||||
|
||||
COPY --from=builder --chown=nonroot:nonroot /app /app
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/.venv /app/.venv
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/litellm /app/litellm
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/enterprise /app/enterprise
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/gateway /app/gateway
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/docker/component_entrypoint.sh /app/docker/component_entrypoint.sh
|
||||
COPY --from=builder --chown=nonroot:nonroot /app/schema.prisma /app/schema.prisma
|
||||
COPY --from=builder /opt/prisma /opt/prisma
|
||||
COPY --from=pgbouncer-builder /usr/local/bin/pgbouncer /usr/local/bin/pgbouncer
|
||||
|
||||
|
|
|
|||
8
litellm-rust/Cargo.lock
generated
8
litellm-rust/Cargo.lock
generated
|
|
@ -8202,18 +8202,18 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "webpki-root-certs"
|
||||
version = "1.0.9"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
|
||||
checksum = "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "webpki-roots"
|
||||
version = "1.0.9"
|
||||
version = "1.0.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7dcd9d09a39985f5344844e66b0c530a33843579125f23e21e9f0f220850f22a"
|
||||
checksum = "f5f280bbb2b287bc8e7de7e3852945a0ff50d1bbcb5ed42b6c7a568c546df171"
|
||||
dependencies = [
|
||||
"rustls-pki-types",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -124,7 +124,7 @@ moka = { version = "0.12.16", features = ["future"] }
|
|||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
url = "2.5.8"
|
||||
percent-encoding = "2.3"
|
||||
webpki-roots = "1"
|
||||
webpki-roots = "1.0.10"
|
||||
time = { version = "0.3.53", features = ["parsing"] }
|
||||
criterion = "0.8.2"
|
||||
fancy-regex = "0.19.2"
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ mod rules;
|
|||
mod status;
|
||||
mod vertex_ai;
|
||||
|
||||
pub use original::{ExceptionFamily, OriginalException};
|
||||
pub use original::{ExceptionFamily, OriginalException, TimeoutPhase};
|
||||
pub use public::{MappedFailure, PublicError, UpstreamResponse};
|
||||
|
||||
use rules::{Rule, contains_any, first_match};
|
||||
|
|
@ -67,7 +67,6 @@ const TIMEOUT_MARKERS: &[&str] = &[
|
|||
pub struct ExceptionContext {
|
||||
pub model: String,
|
||||
pub custom_llm_provider: String,
|
||||
pub asynchronous: bool,
|
||||
pub vertex_project: Option<String>,
|
||||
pub vertex_location: Option<String>,
|
||||
pub model_group: Option<String>,
|
||||
|
|
@ -105,11 +104,12 @@ pub fn exception_type(
|
|||
(None, message.clone(), None)
|
||||
}
|
||||
OriginalException::Timeout {
|
||||
phase,
|
||||
timeout_seconds,
|
||||
elapsed_seconds,
|
||||
} => (
|
||||
None,
|
||||
timeout_message(context.asynchronous, *timeout_seconds, *elapsed_seconds),
|
||||
timeout_message(*phase, *timeout_seconds, *elapsed_seconds),
|
||||
None,
|
||||
),
|
||||
};
|
||||
|
|
@ -161,21 +161,16 @@ fn family_rules(family: ExceptionFamily) -> &'static [Rule] {
|
|||
}
|
||||
}
|
||||
|
||||
/// The text the Python HTTP handler's timeout carries: the sync and async handlers word it
|
||||
/// differently.
|
||||
fn timeout_message(
|
||||
asynchronous: bool,
|
||||
/// The text the Python HTTP handler's timeout carries: the phase that expired and the limit
|
||||
/// httpx enforced for it, both rendered as Python would print them.
|
||||
pub fn timeout_message(
|
||||
phase: TimeoutPhase,
|
||||
timeout_seconds: Option<f64>,
|
||||
elapsed_seconds: Option<f64>,
|
||||
) -> String {
|
||||
let timeout = python_float(timeout_seconds);
|
||||
if asynchronous {
|
||||
let elapsed =
|
||||
python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
|
||||
format!("Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
|
||||
} else {
|
||||
format!("Connection timed out after {timeout} seconds.")
|
||||
}
|
||||
let elapsed = python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0));
|
||||
format!("{phase} timed out. Timeout passed={timeout}, time taken={elapsed} seconds")
|
||||
}
|
||||
|
||||
fn python_float(value: Option<f64>) -> String {
|
||||
|
|
@ -356,6 +351,7 @@ mod tests {
|
|||
#[test]
|
||||
fn a_handler_timeout_is_a_408_without_a_response() {
|
||||
let original = OriginalException::Timeout {
|
||||
phase: TimeoutPhase::Read,
|
||||
timeout_seconds: Some(0.5),
|
||||
elapsed_seconds: Some(0.5031),
|
||||
};
|
||||
|
|
@ -363,7 +359,7 @@ mod tests {
|
|||
mapped("mistral", &original),
|
||||
MappedFailure {
|
||||
error: PublicError::Timeout { status: 408 },
|
||||
message: "MistralException - Connection timed out after 0.5 seconds.".into(),
|
||||
message: "MistralException - Read timed out. Timeout passed=0.5, time taken=0.503 seconds".into(),
|
||||
upstream: None,
|
||||
debug_info: DEBUG.into(),
|
||||
}
|
||||
|
|
@ -440,39 +436,44 @@ mod tests {
|
|||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::sync(
|
||||
false,
|
||||
#[case::connect(TimeoutPhase::Connect, "Connection timed out.")]
|
||||
#[case::read(TimeoutPhase::Read, "Read timed out.")]
|
||||
#[case::write(TimeoutPhase::Write, "Write timed out.")]
|
||||
#[case::pool(TimeoutPhase::Pool, "Connection pool timed out.")]
|
||||
fn timeout_text_names_the_phase_that_expired(
|
||||
#[case] phase: TimeoutPhase,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
timeout_message(phase, Some(0.5), Some(0.5031)),
|
||||
format!("{expected} Timeout passed=0.5, time taken=0.503 seconds")
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::rounds_the_elapsed_time(
|
||||
Some(0.5),
|
||||
Some(0.5031),
|
||||
"Connection timed out after 0.5 seconds."
|
||||
)]
|
||||
#[case::async_rounds_the_elapsed_time(
|
||||
true,
|
||||
Some(0.5),
|
||||
Some(0.5031),
|
||||
"Connection timed out. Timeout passed=0.5, time taken=0.503 seconds"
|
||||
"Timeout passed=0.5, time taken=0.503 seconds"
|
||||
)]
|
||||
#[case::whole_seconds_keep_a_decimal(
|
||||
true,
|
||||
Some(600.0),
|
||||
Some(2.0),
|
||||
"Connection timed out. Timeout passed=600.0, time taken=2.0 seconds"
|
||||
"Timeout passed=600.0, time taken=2.0 seconds"
|
||||
)]
|
||||
#[case::unknown_values_render_as_none(
|
||||
true,
|
||||
None,
|
||||
None,
|
||||
"Connection timed out. Timeout passed=None, time taken=None seconds"
|
||||
"Timeout passed=None, time taken=None seconds"
|
||||
)]
|
||||
fn timeout_text_follows_the_delivery_mode(
|
||||
#[case] asynchronous: bool,
|
||||
fn timeout_text_renders_durations_as_python_floats(
|
||||
#[case] timeout_seconds: Option<f64>,
|
||||
#[case] elapsed_seconds: Option<f64>,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
timeout_message(asynchronous, timeout_seconds, elapsed_seconds),
|
||||
expected
|
||||
timeout_message(TimeoutPhase::Read, timeout_seconds, elapsed_seconds),
|
||||
format!("Read timed out. {expected}")
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ pub enum OriginalException {
|
|||
message: String,
|
||||
},
|
||||
Timeout {
|
||||
phase: TimeoutPhase,
|
||||
timeout_seconds: Option<f64>,
|
||||
elapsed_seconds: Option<f64>,
|
||||
},
|
||||
|
|
@ -20,6 +21,19 @@ pub enum OriginalException {
|
|||
},
|
||||
}
|
||||
|
||||
/// The httpx timeout that expired, spelled the way the Python HTTP handler names it.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, strum::Display, strum::VariantArray)]
|
||||
pub enum TimeoutPhase {
|
||||
#[strum(serialize = "Connection")]
|
||||
Connect,
|
||||
#[strum(serialize = "Read")]
|
||||
Read,
|
||||
#[strum(serialize = "Write")]
|
||||
Write,
|
||||
#[strum(serialize = "Connection pool")]
|
||||
Pool,
|
||||
}
|
||||
|
||||
/// Which provider-specific text rules apply before the shared status table.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ExceptionFamily {
|
||||
|
|
|
|||
|
|
@ -72,26 +72,26 @@ fn duration(seconds: f64) -> Result<Duration, Error> {
|
|||
#[derive(Clone, Default)]
|
||||
pub(crate) struct CacheHeaders(std::sync::Arc<std::sync::OnceLock<String>>);
|
||||
|
||||
impl litellm_host::interceptors::Interceptors<litellm_inference::RouteError> for CacheHeaders {
|
||||
impl litellm_host::interceptors::Interceptors<litellm_inference::Error> for CacheHeaders {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
wire: litellm_host::interceptors::WireRequest,
|
||||
_: litellm_host::interceptors::RequestContext,
|
||||
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::RouteError> {
|
||||
) -> Result<litellm_host::interceptors::WireRequest, litellm_inference::Error> {
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(
|
||||
&self,
|
||||
_: litellm_host::interceptors::RawResponse,
|
||||
) -> Result<(), litellm_inference::RouteError> {
|
||||
) -> Result<(), litellm_inference::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(
|
||||
&self,
|
||||
facts: litellm_host::interceptors::ExecutionFacts,
|
||||
) -> Result<(), litellm_inference::RouteError> {
|
||||
) -> Result<(), litellm_inference::Error> {
|
||||
if let litellm_host::interceptors::ResultSource::Cache { key } = facts.source {
|
||||
let _ = self.0.set(key);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ use axum::{
|
|||
response::{IntoResponse, Response},
|
||||
};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error as InferenceError;
|
||||
use litellm_llms::base_llm::ocr::error::Error as OcrError;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
|
|
@ -19,7 +19,7 @@ pub enum Error {
|
|||
)]
|
||||
UnknownModel(String),
|
||||
#[error(transparent)]
|
||||
Route(#[from] RouteError),
|
||||
Route(#[from] InferenceError),
|
||||
#[error(transparent)]
|
||||
Ocr(#[from] OcrError),
|
||||
#[error("{0} is not implemented by the Rust gateway")]
|
||||
|
|
@ -36,8 +36,8 @@ impl IntoResponse for Error {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<litellm_host_http::Error<RouteError>> for Error {
|
||||
fn from(error: litellm_host_http::Error<RouteError>) -> Self {
|
||||
impl From<litellm_host_http::Error<InferenceError>> for Error {
|
||||
fn from(error: litellm_host_http::Error<InferenceError>) -> Self {
|
||||
match error {
|
||||
litellm_host_http::Error::Call(error) => Self::Route(error),
|
||||
litellm_host_http::Error::Protocol => Self::Internal(error.to_string()),
|
||||
|
|
@ -50,7 +50,7 @@ impl Error {
|
|||
match self {
|
||||
Self::Auth(error) => error.status(),
|
||||
Self::Unsupported(_)
|
||||
| Self::Route(RouteError::Unsupported(_))
|
||||
| Self::Route(InferenceError::Unsupported(_))
|
||||
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
|
||||
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
|
||||
Self::Ocr(
|
||||
|
|
@ -64,10 +64,10 @@ impl Error {
|
|||
.and_then(|status| StatusCode::from_u16(status).ok())
|
||||
.unwrap_or(StatusCode::INTERNAL_SERVER_ERROR),
|
||||
Self::InvalidBody(_) | Self::UnknownModel(_) => StatusCode::BAD_REQUEST,
|
||||
Self::Route(RouteError::Transport(TransportError::Http { status, .. })) => {
|
||||
Self::Route(InferenceError::Transport(TransportError::Http { status, .. })) => {
|
||||
StatusCode::from_u16(*status).unwrap_or(StatusCode::BAD_GATEWAY)
|
||||
}
|
||||
Self::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
|
||||
Self::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey { .. })) => {
|
||||
StatusCode::UNAUTHORIZED
|
||||
}
|
||||
Self::Route(error) if error.is_request() => StatusCode::BAD_REQUEST,
|
||||
|
|
@ -97,7 +97,9 @@ impl Error {
|
|||
/// body already in that shape passes through, any other has its message extracted.
|
||||
pub fn body(&self, request_id: Option<&str>) -> Value {
|
||||
let raw = match self {
|
||||
Self::Route(RouteError::Transport(TransportError::Http { body, .. })) => body.clone(),
|
||||
Self::Route(InferenceError::Transport(TransportError::Http { body, .. })) => {
|
||||
body.clone()
|
||||
}
|
||||
other => other.to_string(),
|
||||
};
|
||||
let parsed = serde_json::from_str::<Value>(&raw).ok();
|
||||
|
|
@ -175,7 +177,7 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn upstream(status: u16, body: &str) -> Error {
|
||||
Error::Route(RouteError::Transport(TransportError::Http {
|
||||
Error::Route(InferenceError::Transport(TransportError::Http {
|
||||
status,
|
||||
body: body.into(),
|
||||
}))
|
||||
|
|
@ -225,16 +227,16 @@ mod tests {
|
|||
|
||||
#[rstest]
|
||||
#[case::upstream_status(upstream(429, ""), StatusCode::TOO_MANY_REQUESTS)]
|
||||
#[case::rejected_request(Error::Route(RouteError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
|
||||
#[case::rejected_request(Error::Route(InferenceError::InvalidRequest("top_k".into())), StatusCode::BAD_REQUEST)]
|
||||
#[case::missing_key(
|
||||
Error::Route(RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
Error::Route(InferenceError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
})),
|
||||
StatusCode::UNAUTHORIZED,
|
||||
)]
|
||||
#[case::lost_connection(
|
||||
Error::Route(RouteError::Transport(TransportError::Network("reset".into()))),
|
||||
Error::Route(InferenceError::Transport(TransportError::Network("reset".into()))),
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
)]
|
||||
fn status_follows_who_is_at_fault(#[case] error: Error, #[case] status: StatusCode) {
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
pub mod route;
|
||||
pub mod types;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
mod common_utils;
|
||||
pub mod constants;
|
||||
pub(crate) mod handler;
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ use litellm_host::{
|
|||
lifecycle::{CallEvent, ExecutionEvent},
|
||||
observation::observation_channel,
|
||||
};
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use rstest::{fixture, rstest};
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -166,12 +166,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -179,11 +179,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ use std::sync::Arc;
|
|||
|
||||
use crate::common_utils::messages_provider;
|
||||
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
pub use types::{
|
||||
MessagesCall, MessagesCallResponse, MessagesSettings, MessagesShaping, litellm_params,
|
||||
messages_body,
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ use litellm_host::interceptors::{
|
|||
ResultSource, WireRequest,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{CacheRequest, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -133,12 +133,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -146,11 +146,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,4 +3,4 @@
|
|||
- The route returns its completed response directly
|
||||
- Route closures supply OCR-specific host capabilities, such as the caller's Azure AD token provider (`src/route.rs`)
|
||||
- Provider code reaches the caller's hooks mid-call only through `litellm_llms::base_llm::ocr::handler::CallHooks`, which this crate implements over its host until it folds into `litellm_host::interceptors::Interceptors`
|
||||
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::RouteError` until OCR folds into `litellm_llms::Error`
|
||||
- OCR failures use `base_llm/ocr/error.rs`, the recorded exception to `litellm_inference::Error` until OCR folds into `litellm_llms::Error`
|
||||
|
|
|
|||
|
|
@ -249,7 +249,7 @@ pub(crate) fn resolve_provider_config<'a>(
|
|||
provider: LlmProviders::Mistral,
|
||||
}
|
||||
}
|
||||
Err(litellm_inference::RouteError::InvalidProvider(provider)) => {
|
||||
Err(litellm_inference::Error::InvalidProvider(provider)) => {
|
||||
return Err(Error::InvalidProvider(provider));
|
||||
}
|
||||
Err(error) => return Err(Error::InvalidProvider(error.to_string())),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_host::observation::ObservationSender;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
|
||||
pub mod ids;
|
||||
pub mod route;
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ use litellm_host::interceptors::{
|
|||
WireRequest,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{CacheRequest, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -124,12 +124,12 @@ struct ChangingHooks {
|
|||
facts: std::sync::Mutex<Vec<ExecutionFacts>>,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for ChangingHooks {
|
||||
impl Interceptors<Error> for ChangingHooks {
|
||||
async fn before_provider_request(
|
||||
&self,
|
||||
mut wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
if self.rewrite {
|
||||
wire.body["temperature"] = json!(if call < 2 { 0.1 } else { 0.8 });
|
||||
|
|
@ -137,11 +137,11 @@ impl Interceptors<RouteError> for ChangingHooks {
|
|||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.facts.lock().unwrap().push(facts);
|
||||
Ok(())
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
pub mod types;
|
||||
pub use litellm_inference::RouteError as Error;
|
||||
pub use litellm_inference::Error;
|
||||
mod constants;
|
||||
mod handler;
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
- `gateway-inference` (serving) -> `inference-<fmt>` (one API's call) -> `inference` (shared base)
|
||||
- This crate is the shared base every format crate builds on
|
||||
- `RouteError` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
|
||||
- `Error` (`src/error.rs`), `CallOptions` (`src/lib.rs`), `CallContext` (`src/context.rs`)
|
||||
- diagnostic spans (`src/diagnostic.rs`), outbound send and signing (`src/outbound.rs`), provider resolution (`src/provider.rs`), `CoreResources` (`src/resources.rs`)
|
||||
- response caching (`src/caching.rs`): `Cachable`, `StreamCachable`, `CacheRequest`, `CallCache`, `execute_unary`, `execute_streaming`, stream capture
|
||||
- Shared test helpers belong in litellm-inference-testing, used only as a dev dependency
|
||||
|
|
@ -49,14 +49,14 @@
|
|||
- `litellm-inference-<fmt>` mirrors the route packages (`litellm/ocr/`, `litellm/messages/`, ...)
|
||||
- Provider code never imports from `inference` or `inference-*`
|
||||
- Handlers belong in `inference-*` or `llms`, never in a host crate
|
||||
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::RouteError as Error;`, which keeps the pre-split `<fmt>::Error` name
|
||||
- Import every item from its canonical path. Never re-export another crate's items or give an item a second public path. The only allowed re-exports are a private submodule surfacing its item at its module root (`mod error; pub use error::Error;`) and each format crate's `pub use litellm_inference::Error;`, which keeps the pre-split `<fmt>::Error` name
|
||||
|
||||
## Error placement
|
||||
|
||||
- The workspace `Error definitions` rules shape each crate's error. This section decides which crate and module a failure belongs to
|
||||
- A failure is declared once, by the lowest crate that raises it
|
||||
- every crate above nests it unchanged (`#[error(transparent)] Auth(#[from] litellm_auth::Error)`) or maps it once at its boundary, as `src/error.rs` does for `litellm_llms::Error`
|
||||
- `RouteError` collects route failures for every format crate and never re-declares a variant a lower crate raises
|
||||
- `Error` collects route failures for every format crate and never re-declares a variant a lower crate raises
|
||||
- Scope follows the concept, not the first caller
|
||||
- an error type under `litellm-llms`'s `<provider>/` is private to that provider: no other provider and nothing in `base_llm` may import it
|
||||
- a failure two providers or two routes can hit (wire framing, stream event decoding, a malformed provider response) belongs to the crate that owns the concept: `litellm-framer` for framing, `litellm_llms::Error` for the transformation layer
|
||||
|
|
|
|||
|
|
@ -24,9 +24,9 @@ use serde::{Deserialize, Serialize, de::DeserializeOwned};
|
|||
use serde_json::Value;
|
||||
use tokio_util::codec::Decoder;
|
||||
|
||||
use crate::RouteError;
|
||||
use crate::Error;
|
||||
|
||||
pub trait Cachable: Protocol<Error = RouteError> {
|
||||
pub trait Cachable: Protocol<Error = Error> {
|
||||
const SURFACE: &'static str;
|
||||
|
||||
fn reusable(_response: &Self::Response) -> bool {
|
||||
|
|
@ -145,15 +145,15 @@ pub async fn execute_unary<P, F, Fut>(
|
|||
request: CacheRequest,
|
||||
cache: Option<Arc<dyn ResponseCacheService>>,
|
||||
options: Option<CacheOptions>,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
provider: F,
|
||||
) -> Result<P::Response, RouteError>
|
||||
) -> Result<P::Response, Error>
|
||||
where
|
||||
P: Cachable,
|
||||
P::Response: Serialize + DeserializeOwned,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<P::Response, RouteError>>,
|
||||
Fut: Future<Output = Result<P::Response, Error>>,
|
||||
{
|
||||
let identity = request.identity.clone();
|
||||
crate::diagnostic::provider(&identity.model, &identity.provider);
|
||||
|
|
@ -191,15 +191,15 @@ pub async fn execute_streaming<P, F, Fut>(
|
|||
request: CacheRequest,
|
||||
cache: Option<Arc<dyn ResponseCacheService>>,
|
||||
options: Option<CacheOptions>,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
provider: F,
|
||||
) -> Result<OutputOf<P>, RouteError>
|
||||
) -> Result<OutputOf<P>, Error>
|
||||
where
|
||||
P: StreamCachable,
|
||||
P::Response: Serialize + DeserializeOwned,
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: Future<Output = Result<OutputOf<P>, RouteError>>,
|
||||
Fut: Future<Output = Result<OutputOf<P>, Error>>,
|
||||
{
|
||||
let identity = request.identity.clone();
|
||||
crate::diagnostic::provider(&identity.model, &identity.provider);
|
||||
|
|
@ -299,9 +299,9 @@ impl<P: StreamCachable> CallCache<P> {
|
|||
}
|
||||
|
||||
fn capture_stream<P: StreamCachable>(
|
||||
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>>,
|
||||
chunks: futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>>,
|
||||
session: CacheSession,
|
||||
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, RouteError>> {
|
||||
) -> futures_util::stream::BoxStream<'static, Result<P::Chunk, Error>> {
|
||||
stream::try_unfold(
|
||||
(chunks, Some(Vec::<u8>::new()), session),
|
||||
|(mut chunks, captured, session)| async move {
|
||||
|
|
@ -330,7 +330,7 @@ fn capture_stream<P: StreamCachable>(
|
|||
{
|
||||
session.store(entry).await;
|
||||
}
|
||||
Ok::<_, RouteError>(None)
|
||||
Ok::<_, Error>(None)
|
||||
}
|
||||
}
|
||||
},
|
||||
|
|
@ -369,9 +369,9 @@ fn successful_stream(text: &str, terminal: &str) -> bool {
|
|||
|
||||
async fn publish(
|
||||
facts: ExecutionFacts,
|
||||
interceptors: &impl Interceptors<RouteError>,
|
||||
interceptors: &impl Interceptors<Error>,
|
||||
observers: Option<&ObservationSender>,
|
||||
) -> Result<(), RouteError> {
|
||||
) -> Result<(), Error> {
|
||||
if let Some(observers) = observers {
|
||||
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
|
||||
facts: facts.clone(),
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ use litellm_host::{
|
|||
observation::ObservationSender,
|
||||
};
|
||||
|
||||
use crate::{CallOptions, RouteError};
|
||||
use crate::{CallOptions, Error};
|
||||
|
||||
pub struct CallContext<'a, I> {
|
||||
pub interceptors: &'a I,
|
||||
|
|
@ -13,7 +13,7 @@ pub struct CallContext<'a, I> {
|
|||
pub cache: CachePolicy,
|
||||
}
|
||||
|
||||
impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
||||
impl<'a, I: Interceptors<Error>> CallContext<'a, I> {
|
||||
pub fn new(interceptors: &'a I, options: CallOptions) -> Self {
|
||||
Self {
|
||||
interceptors,
|
||||
|
|
@ -22,7 +22,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
}
|
||||
}
|
||||
|
||||
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
pub async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
if let Some(observers) = &self.observers {
|
||||
observers.emit(CallEvent::Execution(ExecutionEvent::ResultReady {
|
||||
facts: facts.clone(),
|
||||
|
|
@ -31,7 +31,7 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
self.interceptors.result_ready(facts).await
|
||||
}
|
||||
|
||||
pub async fn response_received(&self, body: &str) -> Result<(), RouteError> {
|
||||
pub async fn response_received(&self, body: &str) -> Result<(), Error> {
|
||||
let raw = RawResponse {
|
||||
body: body.to_owned(),
|
||||
};
|
||||
|
|
@ -43,6 +43,6 @@ impl<'a, I: Interceptors<RouteError>> CallContext<'a, I> {
|
|||
self.interceptors
|
||||
.after_provider_response(raw)
|
||||
.await
|
||||
.map_err(RouteError::post_call)
|
||||
.map_err(Error::post_call)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ use litellm_http::transport::Error as TransportError;
|
|||
use litellm_llms::Error as LlmError;
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum RouteError {
|
||||
pub enum Error {
|
||||
#[error("expected {expected}, got {actual}")]
|
||||
InvalidType {
|
||||
expected: &'static str,
|
||||
|
|
@ -38,10 +38,10 @@ pub enum RouteError {
|
|||
#[error(transparent)]
|
||||
Secret(#[from] SecretError),
|
||||
#[error("post-call hook failed: {0}")]
|
||||
PostCallHook(#[source] Arc<RouteError>),
|
||||
PostCallHook(#[source] Arc<Error>),
|
||||
}
|
||||
|
||||
impl From<litellm_host::machine::MachineFault> for RouteError {
|
||||
impl From<litellm_host::machine::MachineFault> for Error {
|
||||
fn from(fault: litellm_host::machine::MachineFault) -> Self {
|
||||
use litellm_host::machine::MachineFault;
|
||||
Self::InvalidRequest(match fault {
|
||||
|
|
@ -51,7 +51,7 @@ impl From<litellm_host::machine::MachineFault> for RouteError {
|
|||
}
|
||||
}
|
||||
|
||||
impl RouteError {
|
||||
impl Error {
|
||||
pub fn post_call(error: Self) -> Self {
|
||||
Self::PostCallHook(Arc::new(error))
|
||||
}
|
||||
|
|
@ -76,7 +76,7 @@ impl RouteError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<LlmError> for RouteError {
|
||||
impl From<LlmError> for Error {
|
||||
fn from(error: LlmError) -> Self {
|
||||
match error {
|
||||
LlmError::InvalidType { expected, actual } => Self::InvalidType { expected, actual },
|
||||
|
|
@ -99,7 +99,7 @@ impl SecretError {
|
|||
}
|
||||
}
|
||||
|
||||
impl From<litellm_secrets::Error> for RouteError {
|
||||
impl From<litellm_secrets::Error> for Error {
|
||||
fn from(error: litellm_secrets::Error) -> Self {
|
||||
Self::Secret(SecretError(Arc::new(error)))
|
||||
}
|
||||
|
|
@ -115,22 +115,22 @@ impl Eq for SecretError {}
|
|||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::RouteError;
|
||||
use super::Error;
|
||||
use litellm_llms::{Error as LlmError, ErrorDetail};
|
||||
use rstest::rstest;
|
||||
|
||||
#[test]
|
||||
fn a_missing_api_key_is_the_environment_not_the_request() {
|
||||
assert!(
|
||||
!RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
!Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
})
|
||||
.is_request()
|
||||
);
|
||||
assert!(RouteError::Auth(litellm_auth::Error::InvalidHeader).is_request());
|
||||
assert!(RouteError::InvalidRequest("top_k".into()).is_request());
|
||||
assert!(!RouteError::InvalidResponse("bad json".into()).is_request());
|
||||
assert!(Error::Auth(litellm_auth::Error::InvalidHeader).is_request());
|
||||
assert!(Error::InvalidRequest("top_k".into()).is_request());
|
||||
assert!(!Error::InvalidResponse("bad json".into()).is_request());
|
||||
}
|
||||
#[rstest]
|
||||
#[case::request(true)]
|
||||
|
|
@ -139,7 +139,7 @@ mod tests {
|
|||
let source = serde_json::from_str::<serde_json::Value>("{").unwrap_err();
|
||||
let source_message = source.to_string();
|
||||
let detail = ErrorDetail::invalid("test payload", source);
|
||||
let error = RouteError::from(if request {
|
||||
let error = Error::from(if request {
|
||||
LlmError::InvalidRequest(detail)
|
||||
} else {
|
||||
LlmError::InvalidResponse(detail)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ pub mod outbound;
|
|||
pub mod provider;
|
||||
pub mod resources;
|
||||
|
||||
pub use error::RouteError;
|
||||
pub use error::Error;
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CallOptions {
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ use litellm_core_utils::get_llm_provider_logic::{
|
|||
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
|
||||
};
|
||||
|
||||
use crate::error::RouteError as Error;
|
||||
use crate::error::Error;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ResolvedProvider<'a> {
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ use litellm_host::{
|
|||
protocol::Protocol,
|
||||
};
|
||||
use litellm_inference::{
|
||||
RouteError,
|
||||
Error,
|
||||
caching::{Cachable, CacheRequest, StreamCachable, execute_streaming, execute_unary},
|
||||
};
|
||||
use rstest::{fixture, rstest};
|
||||
|
|
@ -36,7 +36,7 @@ struct TestRoute;
|
|||
impl Protocol for TestRoute {
|
||||
type Request = Value;
|
||||
type Response = Value;
|
||||
type Error = RouteError;
|
||||
type Error = Error;
|
||||
type HostCall = Infallible;
|
||||
type Chunk = Bytes;
|
||||
type StreamHead = ();
|
||||
|
|
@ -204,7 +204,7 @@ async fn streamed(
|
|||
.chunks(3)
|
||||
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
|
||||
.collect::<Vec<_>>();
|
||||
let ending = fail.then_some(Err(RouteError::Unsupported("test transport failure")));
|
||||
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
|
||||
Ok(CallOutput::Stream {
|
||||
head: (),
|
||||
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
|
||||
|
|
@ -215,7 +215,7 @@ async fn streamed(
|
|||
.unwrap()
|
||||
}
|
||||
|
||||
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, RouteError> {
|
||||
async fn consume(output: OutputOf<TestRoute>) -> Result<Vec<u8>, Error> {
|
||||
let CallOutput::Stream { chunks, .. } = output else {
|
||||
panic!("expected a stream");
|
||||
};
|
||||
|
|
@ -315,7 +315,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
|
|||
None,
|
||||
|| async {
|
||||
calls.fetch_add(1, Ordering::SeqCst);
|
||||
Err(RouteError::Unsupported("test provider failure"))
|
||||
Err(Error::Unsupported("test provider failure"))
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
|
@ -467,8 +467,8 @@ struct CacheHitAccounting {
|
|||
reject: bool,
|
||||
}
|
||||
|
||||
impl Interceptors<RouteError> for CacheHitAccounting {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), RouteError> {
|
||||
impl Interceptors<Error> for CacheHitAccounting {
|
||||
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
|
||||
self.calls.fetch_add(1, Ordering::SeqCst);
|
||||
let ResultSource::Cache { key } = facts.source else {
|
||||
panic!("expected cache source")
|
||||
|
|
@ -482,7 +482,7 @@ impl Interceptors<RouteError> for CacheHitAccounting {
|
|||
);
|
||||
self.key.set(key).unwrap();
|
||||
if self.reject {
|
||||
return Err(RouteError::Unsupported("cache accounting rejected"));
|
||||
return Err(Error::Unsupported("cache accounting rejected"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -491,11 +491,11 @@ impl Interceptors<RouteError> for CacheHitAccounting {
|
|||
&self,
|
||||
wire: WireRequest,
|
||||
_: RequestContext,
|
||||
) -> Result<WireRequest, RouteError> {
|
||||
) -> Result<WireRequest, Error> {
|
||||
Ok(wire)
|
||||
}
|
||||
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), RouteError> {
|
||||
async fn after_provider_response(&self, _: RawResponse) -> Result<(), Error> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
|
@ -571,7 +571,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
|
|||
if reject {
|
||||
assert!(matches!(
|
||||
result,
|
||||
Err(RouteError::Unsupported("cache accounting rejected"))
|
||||
Err(Error::Unsupported("cache accounting rejected"))
|
||||
));
|
||||
} else {
|
||||
assert_eq!(result.unwrap(), expected);
|
||||
|
|
@ -590,7 +590,7 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
|
|||
impl Protocol for UnaryTestRoute {
|
||||
type Request = Value;
|
||||
type Response = Value;
|
||||
type Error = RouteError;
|
||||
type Error = Error;
|
||||
type HostCall = Infallible;
|
||||
type Chunk = Infallible;
|
||||
type StreamHead = Infallible;
|
||||
|
|
|
|||
|
|
@ -235,7 +235,10 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
|
||||
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
|
||||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -272,7 +275,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use pyo3::{
|
||||
exceptions::{PyRuntimeError, PyValueError},
|
||||
prelude::*,
|
||||
|
|
@ -19,9 +19,9 @@ pyo3::create_exception!(
|
|||
"The provider call was already issued and failed. Args are (status, message); status is 0 when there was no HTTP response."
|
||||
);
|
||||
|
||||
pub(crate) fn route_error_to_pyerr(error: RouteError) -> PyErr {
|
||||
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
|
||||
match error {
|
||||
RouteError::Transport(TransportError::Http { status, body }) => {
|
||||
Error::Transport(TransportError::Http { status, body }) => {
|
||||
RustUpstreamError::new_err((status, body))
|
||||
}
|
||||
other => by_fault(other.is_request(), other.to_string()),
|
||||
|
|
@ -42,13 +42,13 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::unsupported(RouteError::Unsupported("test capability"), true)]
|
||||
#[case::invalid_provider(RouteError::InvalidProvider("unknown".into()), true)]
|
||||
#[case::invalid_request(RouteError::InvalidRequest("empty messages".into()), true)]
|
||||
#[case::unsupported(Error::Unsupported("test capability"), true)]
|
||||
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
|
||||
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
|
||||
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
|
||||
#[case::network(TransportError::Network("timed out".into()).into(), false)]
|
||||
#[case::invalid_response(RouteError::InvalidResponse("missing usage".into()), false)]
|
||||
fn route_failures_are_terminal(#[case] error: RouteError, #[case] is_request: bool) {
|
||||
#[case::invalid_response(Error::InvalidResponse("missing usage".into()), false)]
|
||||
fn route_failures_are_terminal(#[case] error: Error, #[case] is_request: bool) {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let failure = route_error_to_pyerr(error);
|
||||
|
|
@ -86,14 +86,12 @@ mod tests {
|
|||
fn missing_api_key_stays_a_runtime_error_while_other_auth_failures_are_value_errors() {
|
||||
Python::initialize();
|
||||
Python::attach(|py| {
|
||||
let missing =
|
||||
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
}));
|
||||
let missing = route_error_to_pyerr(Error::Auth(litellm_auth::Error::MissingApiKey {
|
||||
provider: "Anthropic",
|
||||
environment_variable: "ANTHROPIC_API_KEY",
|
||||
}));
|
||||
assert!(missing.is_instance_of::<PyRuntimeError>(py));
|
||||
let invalid =
|
||||
route_error_to_pyerr(RouteError::Auth(litellm_auth::Error::InvalidHeader));
|
||||
let invalid = route_error_to_pyerr(Error::Auth(litellm_auth::Error::InvalidHeader));
|
||||
assert!(invalid.is_instance_of::<PyValueError>(py));
|
||||
});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
|
|||
use litellm_host::{call::HostedMachine, protocol::Protocol};
|
||||
use litellm_host_python::{PythonBinding, PythonHostCalls, from_py};
|
||||
use litellm_http::transport::Error as TransportError;
|
||||
use litellm_inference::RouteError;
|
||||
use litellm_inference::Error;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
|
|
@ -119,14 +119,14 @@ impl InferenceHost {
|
|||
public_response(py, self.module, response)
|
||||
}
|
||||
|
||||
pub fn error(&self, py: Python<'_>, error: RouteError) -> PyResult<PyErr> {
|
||||
if let RouteError::Secret(source) = &error
|
||||
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
|
||||
if let Error::Secret(source) = &error
|
||||
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
|
||||
{
|
||||
return Ok(original);
|
||||
}
|
||||
let native = match error {
|
||||
RouteError::Transport(TransportError::Http { status, body }) => {
|
||||
Error::Transport(TransportError::Http { status, body }) => {
|
||||
let error = RustUpstreamError::new_err((status, body));
|
||||
error
|
||||
.value(py)
|
||||
|
|
@ -144,7 +144,7 @@ impl InferenceHost {
|
|||
}
|
||||
|
||||
pub(super) trait InferenceRoute: Sized + 'static {
|
||||
type Protocol: Protocol<Error = RouteError>;
|
||||
type Protocol: Protocol<Error = Error>;
|
||||
const OPERATION: LoggingOperation;
|
||||
const SYNC_CALL_TYPE: &'static str;
|
||||
const ASYNC_CALL_TYPE: &'static str;
|
||||
|
|
|
|||
|
|
@ -88,20 +88,23 @@ fn aws_context(
|
|||
}
|
||||
|
||||
fn read_timeout(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<Duration>> {
|
||||
let Some(value) = value.filter(|value| !value.is_none()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
let seconds = match value.extract::<f64>() {
|
||||
Ok(value) => Some(value),
|
||||
Err(_) => value.getattr("read")?.extract::<Option<f64>>()?,
|
||||
};
|
||||
seconds
|
||||
read_timeout_seconds(value)?
|
||||
.map(|value| {
|
||||
Duration::try_from_secs_f64(value).map_err(|_| PyValueError::new_err("invalid timeout"))
|
||||
})
|
||||
.transpose()
|
||||
}
|
||||
|
||||
pub(super) fn read_timeout_seconds(value: Option<&Bound<'_, PyAny>>) -> PyResult<Option<f64>> {
|
||||
let Some(value) = value.filter(|value| !value.is_none()) else {
|
||||
return Ok(None);
|
||||
};
|
||||
match value.extract::<f64>() {
|
||||
Ok(value) => Ok(Some(value)),
|
||||
Err(_) => value.getattr("read")?.extract::<Option<f64>>(),
|
||||
}
|
||||
}
|
||||
|
||||
fn vault_context(params: Option<&Bound<'_, PyAny>>) -> PyResult<HashicorpOperationContext> {
|
||||
let params = params.and_then(|value| value.cast::<PyDict>().ok());
|
||||
let nested = params
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ mod operation;
|
|||
|
||||
pub(super) use operation::{Failure, FailureKind, FailureStage, delete, rotate, write};
|
||||
|
||||
use litellm_core_utils::exception_mapping_utils::{TimeoutPhase, timeout_message};
|
||||
use litellm_secrets::hashicorp::{Error, RawOperationError};
|
||||
use pyo3::prelude::*;
|
||||
|
||||
use super::mutation::{error_value, http_message, json_value};
|
||||
use super::provider::read_timeout_seconds;
|
||||
|
||||
const HTTP_TRANSPORT_MODULE: &str = "litellm.rust_bridge.host.http_transport";
|
||||
|
||||
|
|
@ -69,21 +71,17 @@ fn cause_message(py: Python<'_>, kind: FailureKind, context: &ErrorContext) -> P
|
|||
}
|
||||
FailureKind::Native(RawOperationError::Timeout { method, elapsed }) => {
|
||||
if method == "POST" {
|
||||
let elapsed = py
|
||||
.import("builtins")?
|
||||
.call_method1("round", (elapsed.as_secs_f64(), 3))?;
|
||||
let kwargs = pyo3::types::PyDict::new(py);
|
||||
kwargs.set_item(
|
||||
"message",
|
||||
format!(
|
||||
"Connection timed out. Timeout passed={}, time taken={} seconds",
|
||||
context.timeout.as_deref().unwrap_or("None"),
|
||||
elapsed.str()?
|
||||
timeout_message(
|
||||
TimeoutPhase::Read,
|
||||
context.timeout,
|
||||
Some(elapsed.as_secs_f64()),
|
||||
),
|
||||
)?;
|
||||
kwargs.set_item("model", "default-model-name")?;
|
||||
kwargs.set_item("llm_provider", "litellm-httpx-handler")?;
|
||||
kwargs.set_item("headers", pyo3::types::PyDict::new(py))?;
|
||||
py.import("litellm.exceptions")?
|
||||
.getattr("Timeout")?
|
||||
.call((), Some(&kwargs))?
|
||||
|
|
@ -159,7 +157,7 @@ fn response_text(py: Python<'_>, body: &[u8]) -> PyResult<String> {
|
|||
|
||||
#[derive(Default)]
|
||||
pub(super) struct ErrorContext {
|
||||
timeout: Option<String>,
|
||||
timeout: Option<f64>,
|
||||
aiohttp: bool,
|
||||
}
|
||||
|
||||
|
|
@ -173,7 +171,7 @@ impl ErrorContext {
|
|||
return Ok(Self::default());
|
||||
}
|
||||
Ok(Self {
|
||||
timeout: timeout.map(|value| value.str()?.extract()).transpose()?,
|
||||
timeout: read_timeout_seconds(timeout)?,
|
||||
aiohttp: py
|
||||
.import(HTTP_TRANSPORT_MODULE)?
|
||||
.getattr("uses_aiohttp_transport")?
|
||||
|
|
|
|||
|
|
@ -175,7 +175,7 @@ argilla_batch_size: Optional[int] = None
|
|||
datadog_use_v1: Optional[bool] = False # if you want to use v1 datadog logged payload.
|
||||
gcs_pub_sub_use_v1: Optional[bool] = False # if you want to use v1 gcs pubsub logged payload
|
||||
generic_api_use_v1: Optional[bool] = False # if you want to use v1 generic api logged payload
|
||||
argilla_transformation_object: Optional[Dict[str, Any]] = None
|
||||
argilla_transformation_object: Mapping[str, str] | None = None
|
||||
_async_input_callback: List[Union[str, Callable[..., object], "CustomLogger"]] = ( # CustomLogger is lazy-loaded
|
||||
[]
|
||||
) # internal variable - async custom callbacks are routed here.
|
||||
|
|
@ -333,7 +333,7 @@ in_memory_llm_clients_cache: "LLMClientCache"
|
|||
safe_memory_mode: bool = False
|
||||
enable_azure_ad_token_refresh: Optional[bool] = False
|
||||
# Proxy Authentication - auto-obtain/refresh OAuth2/JWT tokens for LiteLLM Proxy
|
||||
proxy_auth: Optional[Any] = None
|
||||
proxy_auth: _ProxyAuthHandlerType | None = None
|
||||
### DEFAULT AZURE API VERSION ###
|
||||
AZURE_DEFAULT_API_VERSION = "2025-02-01-preview" # this is updated to the latest
|
||||
### DEFAULT WATSONX API VERSION ###
|
||||
|
|
@ -503,7 +503,7 @@ skill_search_embedding_model: Optional[str] = None
|
|||
# Supports both old format (Dict[str, str]) and new format (Dict[str, Dict[str, Any]])
|
||||
# New format: { "displayName": { "url": "...", "index": 0 } }
|
||||
# Old format: { "displayName": "url" } (for backward compatibility)
|
||||
public_model_groups_links: Dict[str, Union[str, Dict[str, Any]]] = {}
|
||||
public_model_groups_links: dict[str, str | dict[str, object]] = {}
|
||||
#### REQUEST PRIORITIZATION #######
|
||||
priority_reservation: Optional[Dict[str, Union[float, "PriorityReservationDict"]]] = None
|
||||
# priority_reservation_settings is lazy-loaded via __getattr__
|
||||
|
|
@ -682,6 +682,7 @@ darkbloom_models: Set = set()
|
|||
openinfer_models: Final[Set[str]] = set()
|
||||
v0_models: Set = set()
|
||||
morph_models: Set = set()
|
||||
opencode_go_models: Final[set[str]] = set() # mutable-ok: populated when the shared model catalog loads
|
||||
lambda_ai_models: Set = set()
|
||||
inception_models: Set = set()
|
||||
hyperbolic_models: Set = set()
|
||||
|
|
@ -970,6 +971,8 @@ def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, obje
|
|||
v0_models.add(key)
|
||||
elif value.get("litellm_provider") == "morph":
|
||||
morph_models.add(key)
|
||||
elif value.get("litellm_provider") == "opencode_go":
|
||||
opencode_go_models.add(key)
|
||||
elif value.get("litellm_provider") == "lambda_ai":
|
||||
lambda_ai_models.add(key)
|
||||
elif value.get("litellm_provider") == "inception":
|
||||
|
|
@ -1137,6 +1140,7 @@ model_list = list(
|
|||
| openinfer_models
|
||||
| v0_models
|
||||
| morph_models
|
||||
| opencode_go_models
|
||||
| lambda_ai_models
|
||||
| inception_models
|
||||
| black_forest_labs_models
|
||||
|
|
@ -1254,6 +1258,7 @@ def _build_models_by_provider() -> dict:
|
|||
"openinfer": openinfer_models,
|
||||
"v0": v0_models,
|
||||
"morph": morph_models,
|
||||
"opencode_go": opencode_go_models,
|
||||
"lambda_ai": lambda_ai_models,
|
||||
"inception": inception_models,
|
||||
"hyperbolic": hyperbolic_models,
|
||||
|
|
@ -2289,6 +2294,10 @@ if TYPE_CHECKING:
|
|||
LiteLLM_UpperboundKeyGenerateParams,
|
||||
)
|
||||
from litellm.utils import ModelResponseListIterator as _ModelResponseListIterator
|
||||
from collections.abc import Awaitable
|
||||
from litellm.types.utils import ModelResponse as _ModelResponseType
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper as _CustomStreamWrapperType
|
||||
from litellm.proxy_auth import ProxyAuthHandler as _ProxyAuthHandlerType
|
||||
|
||||
# Cost calculator functions
|
||||
cost_per_token: Callable[..., Tuple[float, float]]
|
||||
|
|
@ -2312,7 +2321,7 @@ if TYPE_CHECKING:
|
|||
supports_audio_output: Callable[..., bool]
|
||||
supports_system_messages: Callable[..., bool]
|
||||
supports_reasoning: Callable[..., bool]
|
||||
acreate: Callable[..., Any]
|
||||
acreate: Callable[..., Awaitable[_ModelResponseType | _CustomStreamWrapperType]]
|
||||
get_max_tokens: Callable[..., int]
|
||||
get_model_info: Callable[..., _ModelInfoType]
|
||||
register_prompt_template: Callable[..., None]
|
||||
|
|
@ -2610,8 +2619,10 @@ def __getattr__(name: str) -> Any:
|
|||
cancel_run,
|
||||
delete_run,
|
||||
)
|
||||
from types import MappingProxyType
|
||||
|
||||
return locals()[name]
|
||||
evals_exports: Final[Mapping[str, object]] = MappingProxyType(locals())
|
||||
return evals_exports[name]
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ import importlib
|
|||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
|
|
@ -456,7 +456,7 @@ def _lazy_import_http_handlers(name: str) -> object:
|
|||
params: Final = {"timeout": async_timeout, "client_alias": "module level aclient"}
|
||||
|
||||
# Create the client instance
|
||||
provider_id: Final = cast(Any, "litellm_module_level_client")
|
||||
provider_id: Final = "litellm_module_level_client"
|
||||
async_client: Final = get_async_httpx_client(
|
||||
llm_provider=provider_id,
|
||||
params=params,
|
||||
|
|
|
|||
|
|
@ -1,10 +1,14 @@
|
|||
"""
|
||||
Internal unified UUID helper.
|
||||
|
||||
Always uses fastuuid for performance.
|
||||
Uses fastuuid where supported and the standard library on Python 3.15+.
|
||||
"""
|
||||
|
||||
import fastuuid as _uuid
|
||||
import sys
|
||||
import uuid as _uuid
|
||||
|
||||
if sys.version_info < (3, 15):
|
||||
import fastuuid as _uuid
|
||||
|
||||
# Expose a module-like alias so callers can use: uuid.uuid4()
|
||||
uuid = _uuid
|
||||
|
|
|
|||
|
|
@ -5,7 +5,10 @@ Supports dynamic cost parameters that allow platform owners
|
|||
to define custom costs per agent query or per token.
|
||||
"""
|
||||
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final, SupportsFloat, SupportsIndex
|
||||
|
||||
from typing_extensions import Buffer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -15,6 +18,15 @@ else:
|
|||
LitellmLoggingObject = Any
|
||||
|
||||
|
||||
def _custom_response_cost(model_call_details: Mapping[str, object]) -> float | None:
|
||||
response_cost: Final = model_call_details.get("response_cost", None)
|
||||
if response_cost is None:
|
||||
return None
|
||||
if isinstance(response_cost, (str, Buffer, SupportsFloat, SupportsIndex)):
|
||||
return float(response_cost)
|
||||
raise TypeError(f"float() argument must be a string or a real number, not {type(response_cost).__name__!r}")
|
||||
|
||||
|
||||
class A2ACostCalculator:
|
||||
@staticmethod
|
||||
def calculate_a2a_cost(
|
||||
|
|
@ -45,9 +57,9 @@ class A2ACostCalculator:
|
|||
model_call_details: Final = litellm_logging_obj.model_call_details
|
||||
|
||||
# Check if user set a custom response cost (backward compatibility)
|
||||
response_cost: Final = model_call_details.get("response_cost", None)
|
||||
if response_cost is not None:
|
||||
return float(response_cost)
|
||||
custom_response_cost: Final = _custom_response_cost(model_call_details)
|
||||
if custom_response_cost is not None:
|
||||
return custom_response_cost
|
||||
|
||||
# Get litellm_params for cost parameters
|
||||
litellm_params: Final = model_call_details.get("litellm_params", {}) or {}
|
||||
|
|
|
|||
|
|
@ -84,14 +84,23 @@ def _filter_reserved_headers(
|
|||
return filtered or None
|
||||
|
||||
|
||||
def _calling_key_hash(litellm_params: Mapping[str, object]) -> str | None:
|
||||
principal: Final = litellm_params.get(A2A_USER_API_KEY_HASH_PARAM)
|
||||
if principal is None or isinstance(principal, str):
|
||||
return principal
|
||||
if not principal:
|
||||
return None
|
||||
raise AttributeError(f"{type(principal).__name__!r} object has no attribute 'encode'", name="encode", obj=principal)
|
||||
|
||||
|
||||
def _request_scoped_runtime_session_id(
|
||||
params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, Any],
|
||||
litellm_params: Mapping[str, object],
|
||||
) -> str | None:
|
||||
context_id: Final = get_session_id_from_a2a_params(params)
|
||||
if not isinstance(context_id, str) or not context_id:
|
||||
return None
|
||||
return scope_session_to_principal(context_id, litellm_params.get(A2A_USER_API_KEY_HASH_PARAM))
|
||||
return scope_session_to_principal(context_id, _calling_key_hash(litellm_params))
|
||||
|
||||
|
||||
def _validate_runtime_session_id(session_id: str, model: str) -> str:
|
||||
|
|
@ -109,6 +118,13 @@ def _validate_runtime_session_id(session_id: str, model: str) -> str:
|
|||
)
|
||||
|
||||
|
||||
def _agentcore_model_name(litellm_params: Mapping[str, object]) -> str:
|
||||
model: Final = litellm_params.get("model", "")
|
||||
if isinstance(model, str):
|
||||
return model
|
||||
raise AttributeError(f"{type(model).__name__!r} object has no attribute 'startswith'", name="startswith", obj=model)
|
||||
|
||||
|
||||
class BedrockAgentCoreA2ATransformation:
|
||||
"""
|
||||
Request/response transformation for Bedrock AgentCore A2A agents.
|
||||
|
|
@ -152,11 +168,8 @@ class BedrockAgentCoreA2ATransformation:
|
|||
"""
|
||||
# Extract model and strip the "bedrock/" prefix
|
||||
# "bedrock/agentcore/arn:aws:..." → "agentcore/arn:aws:..."
|
||||
model: Final = litellm_params.get("model", "")
|
||||
if model.startswith("bedrock/"):
|
||||
agentcore_model = model[len("bedrock/") :]
|
||||
else:
|
||||
agentcore_model = model
|
||||
model: Final = _agentcore_model_name(litellm_params)
|
||||
agentcore_model: Final = model.removeprefix("bedrock/")
|
||||
|
||||
# Build optional_params from litellm_params (everything except model and custom_llm_provider)
|
||||
optional_params: Final = {k: v for k, v in litellm_params.items() if k not in ("model", "custom_llm_provider")}
|
||||
|
|
|
|||
|
|
@ -8,11 +8,31 @@ WXO uses a REST API (not A2A/JSON-RPC) with an async-poll execution model:
|
|||
"""
|
||||
|
||||
import asyncio
|
||||
from collections.abc import AsyncIterator, Mapping
|
||||
from typing import Any, Final
|
||||
from collections.abc import AsyncIterator, Iterable, Mapping, Sequence
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, iter_items
|
||||
from litellm.llms.openai.common_utils import decoded_json_object
|
||||
|
||||
|
||||
def _a2a_text_part(part: object) -> Sequence[object]:
|
||||
part_fields: Final = as_str_mapping(part)
|
||||
if not isinstance(part_fields, dict):
|
||||
return ()
|
||||
if part_fields.get("kind") in (None, "", "text") and part_fields.get("text"):
|
||||
return (part_fields["text"],)
|
||||
return ()
|
||||
|
||||
|
||||
def _join_with_spaces(texts: Sequence[object]) -> str:
|
||||
strings: Final = [text for text in texts if isinstance(text, str)]
|
||||
if len(strings) == len(texts):
|
||||
return " ".join(strings)
|
||||
first_bad: Final = next(position for position, text in enumerate(texts) if not isinstance(text, str))
|
||||
raise TypeError(f"sequence item {first_bad}: expected str instance, {type(texts[first_bad]).__name__} found")
|
||||
|
||||
|
||||
class WatsonxOrchestrateTransformation:
|
||||
|
|
@ -29,22 +49,16 @@ class WatsonxOrchestrateTransformation:
|
|||
return f"{cp4d_host.rstrip('/')}/orchestrate/cpd/instances/{instance_id}"
|
||||
|
||||
@staticmethod
|
||||
def extract_text_from_a2a_params(params: dict[str, Any]) -> str:
|
||||
def extract_text_from_a2a_params(params: Mapping[str, object]) -> str:
|
||||
"""
|
||||
Extract user message text from A2A MessageSendParams.
|
||||
|
||||
A2A format: params.message.parts[*] where part.kind == "text"
|
||||
"""
|
||||
message: Final = params.get("message", {})
|
||||
parts: Final = message.get("parts", [])
|
||||
texts: Final = []
|
||||
for part in parts:
|
||||
if not isinstance(part, dict):
|
||||
continue
|
||||
kind = part.get("kind")
|
||||
if kind in (None, "", "text") and part.get("text"):
|
||||
texts.append(part["text"])
|
||||
return " ".join(texts) or ""
|
||||
message: Final = decoded_json_object(params.get("message", {}))
|
||||
parts: Final[Iterable[object]] = iter_items(message.get("parts", []))
|
||||
texts: Final = list(chain.from_iterable(map(_a2a_text_part, parts)))
|
||||
return _join_with_spaces(texts) or ""
|
||||
|
||||
@staticmethod
|
||||
def build_wxo_run_body(
|
||||
|
|
|
|||
|
|
@ -4,15 +4,20 @@ Utility functions for A2A protocol.
|
|||
|
||||
import hashlib
|
||||
from collections.abc import Mapping
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.core_helpers import has_attributes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from a2a.types import SendMessageRequest, SendStreamingMessageRequest
|
||||
|
||||
|
||||
class _SupportsModelDump(Protocol):
|
||||
def model_dump(self, *, mode: str) -> object: ...
|
||||
|
||||
|
||||
class A2ARequestUtils:
|
||||
"""Utility class for A2A request/response processing."""
|
||||
|
||||
|
|
@ -74,7 +79,7 @@ class A2ARequestUtils:
|
|||
@staticmethod
|
||||
def get_input_message_from_request(
|
||||
request: "SendMessageRequest | SendStreamingMessageRequest",
|
||||
) -> Any:
|
||||
) -> object:
|
||||
"""
|
||||
Extract the input message from an A2A request.
|
||||
|
||||
|
|
@ -84,7 +89,7 @@ class A2ARequestUtils:
|
|||
Returns:
|
||||
The message object/dict or None
|
||||
"""
|
||||
params: Final = getattr(request, "params", None)
|
||||
params: Final[object] = getattr(request, "params", None)
|
||||
if params is None:
|
||||
return None
|
||||
return getattr(params, "message", None)
|
||||
|
|
@ -127,10 +132,12 @@ class A2ARequestUtils:
|
|||
# the dict branch — request-side parts are a2a-sdk Part RootModels whose
|
||||
# kind/text live on part.root, which the object branch cannot read. This
|
||||
# mirrors how the response side already works (it operates on model_dump).
|
||||
input_message = A2ARequestUtils.get_input_message_from_request(request)
|
||||
if input_message is not None and hasattr(input_message, "model_dump"):
|
||||
input_message = input_message.model_dump(mode="json")
|
||||
input_text: Final = A2ARequestUtils.extract_text_from_message(input_message)
|
||||
input_message: Final = A2ARequestUtils.get_input_message_from_request(request)
|
||||
input_text: Final = A2ARequestUtils.extract_text_from_message(
|
||||
input_message.model_dump(mode="json")
|
||||
if has_attributes(input_message, _SupportsModelDump, "model_dump")
|
||||
else input_message
|
||||
)
|
||||
prompt_tokens: Final = A2ARequestUtils.count_tokens(input_text)
|
||||
|
||||
# Count output tokens
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
# Cache for the loaded configuration
|
||||
_BETA_HEADERS_CONFIG: dict | None = None
|
||||
|
|
@ -44,6 +45,7 @@ _BETA_HEADERS_CONFIG_LOADER: Final = ThreadPoolExecutor(max_workers=1, thread_na
|
|||
|
||||
BetaHeadersConfig: TypeAlias = Mapping[str, str | Mapping[str, str | None]]
|
||||
_BETA_HEADERS_CONFIG_ADAPTER: Final = TypeAdapter(BetaHeadersConfig)
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class GetAnthropicBetaHeadersConfig:
|
||||
|
|
@ -75,7 +77,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
}
|
||||
|
||||
@staticmethod
|
||||
def _check_is_valid_dict(fetched_config: dict) -> bool:
|
||||
def _check_is_valid_dict(fetched_config: object) -> bool:
|
||||
"""Check if fetched config is a non-empty dict with expected structure."""
|
||||
if not isinstance(fetched_config, dict):
|
||||
verbose_logger.warning(
|
||||
|
|
@ -109,7 +111,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
return True
|
||||
|
||||
@classmethod
|
||||
def validate_beta_headers_config(cls, fetched_config: dict) -> bool:
|
||||
def validate_beta_headers_config(cls, fetched_config: object) -> bool:
|
||||
"""
|
||||
Validate the integrity of a fetched beta headers config.
|
||||
|
||||
|
|
@ -118,7 +120,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
return cls._check_is_valid_dict(fetched_config)
|
||||
|
||||
@staticmethod
|
||||
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> dict:
|
||||
def fetch_remote_beta_headers_config(url: str, timeout: int = 5) -> object:
|
||||
"""
|
||||
Fetch the beta headers config from a remote URL.
|
||||
|
||||
|
|
@ -127,7 +129,7 @@ class GetAnthropicBetaHeadersConfig:
|
|||
"""
|
||||
response: Final = httpx.get(url, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
return response.json()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
|
||||
|
||||
def get_beta_headers_config(url: str) -> dict:
|
||||
|
|
@ -145,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
Dict containing the beta headers configuration
|
||||
"""
|
||||
# Check if local-only mode is enabled
|
||||
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
|
||||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
|
|
@ -160,7 +162,9 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
# Validate the fetched config
|
||||
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content):
|
||||
if not GetAnthropicBetaHeadersConfig.validate_beta_headers_config(fetched_config=content) or not isinstance(
|
||||
content, dict
|
||||
):
|
||||
verbose_logger.warning(
|
||||
"LiteLLM: Fetched beta headers config failed integrity check. Using local backup instead. url=%s",
|
||||
url,
|
||||
|
|
|
|||
|
|
@ -7,10 +7,14 @@ Similar to litellm/litellm_core_utils/exception_mapping_utils.py but for Anthrop
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
|
||||
|
||||
from .exceptions import AnthropicErrorDetail, AnthropicErrorResponse, AnthropicErrorType
|
||||
|
||||
_PARSED_ERROR_BODY: Final = TypeAdapter(object)
|
||||
|
||||
# HTTP status code -> Anthropic error type
|
||||
# Source: https://docs.anthropic.com/en/api/errors
|
||||
ANTHROPIC_ERROR_TYPE_MAP: Final[dict[int, AnthropicErrorType]] = {
|
||||
|
|
@ -79,7 +83,7 @@ class AnthropicExceptionMapping:
|
|||
- Generic: {"message": "..."}
|
||||
- Plain strings
|
||||
"""
|
||||
parsed: Final = safe_json_loads(raw_message)
|
||||
parsed: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
if isinstance(parsed, dict):
|
||||
# Bedrock format
|
||||
if "detail" in parsed and isinstance(parsed["detail"], dict):
|
||||
|
|
@ -145,9 +149,8 @@ class AnthropicExceptionMapping:
|
|||
AnthropicErrorResponse dict
|
||||
"""
|
||||
# Try to parse as JSON once
|
||||
parsed: dict | None = safe_json_loads(raw_message)
|
||||
if not isinstance(parsed, dict):
|
||||
parsed = None
|
||||
decoded: Final = _PARSED_ERROR_BODY.validate_python(safe_json_loads(raw_message))
|
||||
parsed: Final[dict[str, object] | None] = decoded if isinstance(decoded, dict) else None
|
||||
|
||||
# If parsed and already in Anthropic format - passthrough
|
||||
if parsed is not None and AnthropicExceptionMapping._is_anthropic_error_dict(parsed):
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ def get_assistants(
|
|||
api_base: str | None = None,
|
||||
api_version: str | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[Assistant]:
|
||||
) -> SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]]:
|
||||
aget_assistants: Final[bool | None] = kwargs.pop("aget_assistants", None)
|
||||
if aget_assistants is not None and not isinstance(aget_assistants, bool):
|
||||
raise Exception("Invalid value passed in for aget_assistants. Only bool or None allowed")
|
||||
|
|
@ -100,7 +100,7 @@ def get_assistants(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[Assistant] | None = None
|
||||
response: SyncCursorPage[Assistant] | Coroutine[None, None, AsyncCursorPage[Assistant]] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -564,7 +564,7 @@ def create_thread(
|
|||
tool_resources: OpenAICreateThreadParamsToolResources | None = None,
|
||||
client: OpenAI | None = None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""
|
||||
- get the llm provider
|
||||
- if openai - route it there
|
||||
|
|
@ -610,7 +610,7 @@ def create_thread(
|
|||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -735,7 +735,7 @@ def get_thread(
|
|||
thread_id: str,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> Thread:
|
||||
) -> Thread | Coroutine[None, None, Thread]:
|
||||
"""Get the thread object, given a thread_id"""
|
||||
aget_thread: Final = kwargs.pop("aget_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -757,7 +757,7 @@ def get_thread(
|
|||
timeout = 600.0
|
||||
api_base: str | None = None
|
||||
api_key: str | None = None
|
||||
response: Thread | None = None
|
||||
response: Thread | Coroutine[None, None, Thread] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -903,7 +903,7 @@ def add_message(
|
|||
metadata: dict | None = None,
|
||||
client=None,
|
||||
**kwargs,
|
||||
) -> OpenAIMessage:
|
||||
) -> OpenAIMessage | Coroutine[None, None, OpenAIMessage]:
|
||||
### COMMON OBJECTS ###
|
||||
a_add_message: Final = kwargs.pop("a_add_message", None)
|
||||
_message_data: Final = MessageData(role=role, content=content, attachments=attachments, metadata=metadata)
|
||||
|
|
@ -935,7 +935,7 @@ def add_message(
|
|||
timeout = 600.0
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
response: OpenAIMessage | None = None
|
||||
response: OpenAIMessage | Coroutine[None, None, OpenAIMessage] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
@ -1065,7 +1065,7 @@ def get_messages(
|
|||
thread_id: str,
|
||||
client: Any | None = None,
|
||||
**kwargs,
|
||||
) -> SyncCursorPage[OpenAIMessage]:
|
||||
) -> SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]]:
|
||||
aget_messages: Final = kwargs.pop("aget_messages", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
litellm_params_dict: Final = get_litellm_params(**kwargs)
|
||||
|
|
@ -1086,7 +1086,7 @@ def get_messages(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: SyncCursorPage[OpenAIMessage] | None = None
|
||||
response: SyncCursorPage[OpenAIMessage] | Coroutine[None, None, AsyncCursorPage[OpenAIMessage]] | None = None
|
||||
api_key: str | None = None
|
||||
api_base: str | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
|
|
@ -1256,7 +1256,7 @@ def run_thread(
|
|||
client: object | None = None,
|
||||
event_handler: AssistantEventHandler | None = None, # for stream=True calls
|
||||
**kwargs,
|
||||
) -> Run:
|
||||
) -> Run | Coroutine[None, None, Run]:
|
||||
"""Run a given thread + assistant."""
|
||||
arun_thread: Final = kwargs.pop("arun_thread", None)
|
||||
optional_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
|
|
@ -1278,7 +1278,7 @@ def run_thread(
|
|||
elif timeout is None:
|
||||
timeout = 600.0
|
||||
|
||||
response: Run | None = None
|
||||
response: Run | Coroutine[None, None, Run] | None = None
|
||||
if custom_llm_provider == "openai":
|
||||
api_base = (
|
||||
optional_params.api_base # for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ This module is dependency-injected: callers pass the proxy ``llm_router`` and
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
import litellm
|
||||
from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
|
||||
|
|
@ -25,12 +25,12 @@ if TYPE_CHECKING:
|
|||
def resolve_embedding_router(
|
||||
embedding_model: str,
|
||||
llm_router: Router | None,
|
||||
llm_model_list: list[dict[str, Any]] | None,
|
||||
llm_model_list: Sequence[Mapping[str, object]] | None,
|
||||
) -> Router | None:
|
||||
"""Return ``llm_router`` iff it serves ``embedding_model`` as a deployment."""
|
||||
if llm_router is None:
|
||||
return None
|
||||
router_model_names: Final[list[str]] = (
|
||||
router_model_names: Final[Sequence[object]] = (
|
||||
[m["model_name"] for m in llm_model_list if "model_name" in m] if llm_model_list is not None else []
|
||||
)
|
||||
if embedding_model in router_model_names:
|
||||
|
|
|
|||
|
|
@ -24,6 +24,16 @@ if current == false then
|
|||
end
|
||||
if ARGV[3] then
|
||||
local decoded, stored = pcall(cjson.decode, current)
|
||||
if ARGV[4] then
|
||||
local legacy = current
|
||||
if decoded and type(stored) == 'string' then legacy = stored end
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
if eligible['model_id'] == legacy then
|
||||
redis.call('EXPIRE', KEYS[1], ARGV[2])
|
||||
return current
|
||||
end
|
||||
end
|
||||
end
|
||||
if decoded and type(stored) == 'table' then
|
||||
for _, eligible in ipairs(cjson.decode(ARGV[3])) do
|
||||
local matches = true
|
||||
|
|
@ -68,6 +78,7 @@ def claim_affinity_pin_in_memory(
|
|||
ttl_seconds: int,
|
||||
*,
|
||||
eligible_values: tuple[Mapping[str, str], ...] | None = None,
|
||||
preserve_legacy_model_ids: bool = False,
|
||||
) -> object:
|
||||
"""No await between read and write, so same-loop claims agree during a Redis outage."""
|
||||
existing: Final[object] = cache.in_memory_cache.get_cache(cache_key)
|
||||
|
|
@ -75,7 +86,12 @@ def claim_affinity_pin_in_memory(
|
|||
if _legacy_pin_matches(existing, pin_value):
|
||||
set_local_affinity_pin(cache, cache_key, pin_value, ttl_seconds)
|
||||
return existing
|
||||
winner: Final = existing if existing is not None and existing in (eligible_values or ()) else pin_value
|
||||
eligible_existing: Final = existing in (eligible_values or ()) or (
|
||||
preserve_legacy_model_ids
|
||||
and isinstance(existing, str)
|
||||
and any("model_id" in value and _legacy_pin_matches(existing, value) for value in (eligible_values or ()))
|
||||
)
|
||||
winner: Final = existing if existing is not None and eligible_existing else pin_value
|
||||
set_local_affinity_pin(cache, cache_key, winner, ttl_seconds)
|
||||
return winner
|
||||
|
||||
|
|
@ -94,6 +110,7 @@ async def claim_affinity_pin(
|
|||
ttl_seconds: int,
|
||||
*,
|
||||
eligible_values: tuple[Mapping[str, str], ...] | None = None,
|
||||
preserve_legacy_model_ids: bool = False,
|
||||
) -> object:
|
||||
"""Return the authoritative first writer, replacing it only when it becomes ineligible.
|
||||
|
||||
|
|
@ -112,6 +129,7 @@ async def claim_affinity_pin(
|
|||
if eligible_values is not None
|
||||
else ()
|
||||
),
|
||||
*(("1",) if preserve_legacy_model_ids and eligible_values is not None else ()),
|
||||
)
|
||||
raw: Final = cast( # cast-ok: Redis scripts return heterogeneous values; only object is asserted here
|
||||
object, await claim_script(keys=(cache_key,), args=args)
|
||||
|
|
@ -124,4 +142,11 @@ async def claim_affinity_pin(
|
|||
return winner
|
||||
except Exception as error: # noqa: BLE001 # Redis/Lua faults retain same-pod affinity through local claims
|
||||
verbose_router_logger.debug("Affinity cache: Redis claim failed, using pod-local claim. error=%s", error)
|
||||
return claim_affinity_pin_in_memory(cache, cache_key, pin_value, ttl_seconds, eligible_values=eligible_values)
|
||||
return claim_affinity_pin_in_memory(
|
||||
cache,
|
||||
cache_key,
|
||||
pin_value,
|
||||
ttl_seconds,
|
||||
eligible_values=eligible_values,
|
||||
preserve_legacy_model_ids=preserve_legacy_model_ids,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -13,10 +13,14 @@ import json
|
|||
from contextlib import suppress
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import print_verbose, verbose_logger
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
_DECODED_BLOB: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class AzureBlobCache(BaseCache):
|
||||
def __init__(self, account_url, container) -> None:
|
||||
|
|
@ -65,7 +69,7 @@ class AzureBlobCache(BaseCache):
|
|||
print_verbose(f"Get Azure Blob Cache: key: {key}")
|
||||
as_bytes: Final = self.container_client.download_blob(key).readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
|
|
@ -86,7 +90,7 @@ class AzureBlobCache(BaseCache):
|
|||
blob: Final = await self.async_container_client.download_blob(key)
|
||||
as_bytes: Final = await blob.readall()
|
||||
as_str: Final = as_bytes.decode("utf-8")
|
||||
cached_response: Final[object] = json.loads(as_str)
|
||||
cached_response: Final = _DECODED_BLOB.validate_python(json.loads(as_str))
|
||||
verbose_logger.debug(
|
||||
"Got Azure Blob Cache: key: %s, cached_response %s. Type Response %s",
|
||||
key,
|
||||
|
|
|
|||
|
|
@ -9,14 +9,7 @@ Has 4 methods:
|
|||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span | Any
|
||||
else:
|
||||
Span = Any
|
||||
from typing import Final
|
||||
|
||||
|
||||
class BaseCache(ABC):
|
||||
|
|
|
|||
|
|
@ -1,15 +1,19 @@
|
|||
import json
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from .base_cache import BaseCache
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
Span = _Span | Any
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
||||
_DECODED_DISK_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
class DiskCache(BaseCache):
|
||||
def __init__(self, disk_cache_dir: str | None = None):
|
||||
|
|
@ -44,7 +48,7 @@ class DiskCache(BaseCache):
|
|||
original_cached_response: Final = self.disk_cache.get(key)
|
||||
if original_cached_response:
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
cached_response = _DECODED_DISK_VALUE.validate_python(json.loads(original_cached_response))
|
||||
except Exception:
|
||||
cached_response = original_cached_response
|
||||
return cached_response
|
||||
|
|
|
|||
|
|
@ -95,7 +95,10 @@ class InMemoryCache(BaseCache):
|
|||
"""
|
||||
Check if a specific key is expired
|
||||
"""
|
||||
return key in self.ttl_dict and self._clock() > self.ttl_dict[key]
|
||||
try:
|
||||
return self._clock() > self.ttl_dict[key]
|
||||
except KeyError:
|
||||
return False
|
||||
|
||||
def _cache_keys(self) -> tuple[str, ...]:
|
||||
return tuple(self.cache_dict) # pyright: ignore[reportUnknownVariableType] # cache_dict is untyped legacy state
|
||||
|
|
@ -227,7 +230,10 @@ class InMemoryCache(BaseCache):
|
|||
if key in self.cache_dict:
|
||||
if self.evict_element_if_expired(key):
|
||||
return None
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
try:
|
||||
original_cached_response: Final = self.cache_dict[key]
|
||||
except KeyError:
|
||||
return None
|
||||
try:
|
||||
cached_response = json.loads(original_cached_response)
|
||||
except Exception:
|
||||
|
|
|
|||
|
|
@ -1738,7 +1738,9 @@ class RedisCache(BaseCache):
|
|||
return key_value_dict
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
async def async_get_cache(self, key, parent_otel_span: Span | None = None, **kwargs):
|
||||
async def async_get_cache(
|
||||
self, key, parent_otel_span: Span | None = None, *, raise_on_error: bool = False, **kwargs
|
||||
):
|
||||
from redis.asyncio import Redis
|
||||
|
||||
_redis_client: Final[Redis] = self.init_async_client()
|
||||
|
|
@ -1781,6 +1783,8 @@ class RedisCache(BaseCache):
|
|||
)
|
||||
)
|
||||
print_verbose(f"litellm.caching.caching: async get() - Got exception from REDIS: {e}")
|
||||
if raise_on_error:
|
||||
raise
|
||||
_record_swallowed_redis_failure(self._circuit_breaker, e)
|
||||
|
||||
@_redis_circuit_breaker_guard
|
||||
|
|
|
|||
|
|
@ -156,6 +156,9 @@ DEFAULT_REPLICATE_POLLING_RETRIES: Final = get_env_int("DEFAULT_REPLICATE_POLLIN
|
|||
DEFAULT_REPLICATE_POLLING_DELAY_SECONDS: Final = get_env_int("DEFAULT_REPLICATE_POLLING_DELAY_SECONDS", 1)
|
||||
DEFAULT_IMAGE_TOKEN_COUNT: Final = get_env_int("DEFAULT_IMAGE_TOKEN_COUNT", 250)
|
||||
HF_CONFIG_FETCH_TIMEOUT_SECONDS: Final = 10.0
|
||||
HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS: Final = get_env_int_in_range(
|
||||
"HUGGINGFACE_TOKENIZER_LOAD_RETRY_SECONDS", 300, 1, 86400
|
||||
)
|
||||
|
||||
# Maximum wall-clock seconds a streaming response is allowed to run.
|
||||
# Streams exceeding this duration are terminated with a Timeout error.
|
||||
|
|
@ -477,6 +480,7 @@ REDIS_DAILY_END_USER_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_end_user_sp
|
|||
REDIS_DAILY_AGENT_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_agent_spend_update_buffer"
|
||||
REDIS_DAILY_TAG_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_daily_tag_spend_update_buffer"
|
||||
REDIS_WINDOW_SPEND_UPDATE_BUFFER_KEY: Final = "litellm_window_spend_update_buffer"
|
||||
REDIS_TOOL_DISCOVERY_BUFFER_KEY: Final = "litellm_tool_discovery_buffer"
|
||||
MAX_REDIS_BUFFER_DEQUEUE_COUNT: Final = get_env_int("MAX_REDIS_BUFFER_DEQUEUE_COUNT", 100)
|
||||
REDIS_SPEND_LOGS_BUFFER_KEY: Final = "litellm_spend_logs_buffer"
|
||||
REDIS_SPEND_LOGS_BUFFER_MAX_ROWS: Final = 100000
|
||||
|
|
@ -608,6 +612,7 @@ NON_LLM_CONNECTION_TIMEOUT: Final = get_env_int(
|
|||
) # timeout for adjacent services (e.g. jwt auth)
|
||||
MAX_EXCEPTION_MESSAGE_LENGTH: Final = get_env_int("MAX_EXCEPTION_MESSAGE_LENGTH", 2000)
|
||||
MAX_STRING_LENGTH_PROMPT_IN_DB: Final = get_env_int("MAX_STRING_LENGTH_PROMPT_IN_DB", 2048)
|
||||
MAX_STRING_LENGTH_COMPACTION_SUMMARY_IN_DB: Final = 64 * 1024
|
||||
BEDROCK_MAX_POLICY_SIZE: Final = get_env_int("BEDROCK_MAX_POLICY_SIZE", 75)
|
||||
# One entry per distinct AWS credential-argument set. Per-user cost attribution passes the attributed
|
||||
# identity as aws_session_name, so this bounds how many attributed identities keep a cached STS session.
|
||||
|
|
@ -857,6 +862,7 @@ LITELLM_CHAT_PROVIDERS: Final = [
|
|||
"heroku",
|
||||
"oci",
|
||||
"morph",
|
||||
"opencode_go",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"vercel_ai_gateway",
|
||||
|
|
@ -1112,6 +1118,7 @@ openai_compatible_providers: Final[list[str]] = [
|
|||
"v0",
|
||||
"helicone",
|
||||
"morph",
|
||||
"opencode_go",
|
||||
"lambda_ai",
|
||||
"inception",
|
||||
"hyperbolic",
|
||||
|
|
@ -1838,6 +1845,7 @@ LITELLM_UI_SESSION_DURATION: Final = os.getenv("LITELLM_UI_SESSION_DURATION", "2
|
|||
|
||||
########################### DB CRON JOB NAMES ###########################
|
||||
DB_SPEND_UPDATE_JOB_NAME: Final = "db_spend_update_job"
|
||||
TOOL_DISCOVERY_COMMIT_JOB_NAME: Final = "tool_discovery_commit_job"
|
||||
DB_DAILY_TAG_SPEND_UPDATE_JOB_NAME: Final = "db_daily_tag_spend_update_job"
|
||||
PROMETHEUS_EMIT_BUDGET_METRICS_JOB_NAME: Final = "prometheus_emit_budget_metrics"
|
||||
CLOUDZERO_EXPORT_USAGE_DATA_JOB_NAME: Final = "cloudzero_export_usage_data"
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.litellm_core_utils.llm_cost_calc.utils import (
|
|||
_SERVICE_TIER_TO_COST_KEY_SUFFIX,
|
||||
BilledTokenRates,
|
||||
CostCalculatorUtils,
|
||||
calculate_cache_writing_cost,
|
||||
calculate_cost_component,
|
||||
generic_cost_per_character,
|
||||
generic_cost_per_token,
|
||||
|
|
@ -1200,12 +1201,16 @@ def get_usage_object(
|
|||
elif isinstance(usage_obj, dict):
|
||||
return Usage(**usage_obj)
|
||||
elif isinstance(usage_obj, BaseModel):
|
||||
return Usage(**usage_obj.model_dump())
|
||||
return _usage_from_model(usage_obj)
|
||||
else:
|
||||
verbose_logger.debug("Unknown usage object type: %s, usage_obj: %s", type(usage_obj), usage_obj)
|
||||
return None
|
||||
|
||||
|
||||
def _usage_from_model(usage_model: BaseModel) -> Usage:
|
||||
return Usage(**usage_model.model_dump())
|
||||
|
||||
|
||||
def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper:
|
||||
match usage:
|
||||
case DecisionsUsage():
|
||||
|
|
@ -1546,6 +1551,8 @@ def completion_cost(
|
|||
"""
|
||||
try:
|
||||
call_type = infer_call_type(call_type, completion_response) or "completion"
|
||||
if isinstance(completion_response, DecisionsResponse) and completion_response.usage is None:
|
||||
raise ValueError("Cannot calculate Decisions cost: provider did not report usage")
|
||||
if pricing_provider is not None:
|
||||
custom_llm_provider = pricing_provider # rebind-ok: model_info.pricing_provider overrides the routing provider for cost-map lookups
|
||||
|
||||
|
|
@ -1673,8 +1680,10 @@ def completion_cost(
|
|||
verbose_logger.debug("selected model name for cost calculation: %s", model)
|
||||
|
||||
if completion_response is not None and (
|
||||
isinstance(completion_response, BaseModel) or isinstance(completion_response, dict)
|
||||
): # tts returns a custom class
|
||||
isinstance(completion_response, (BaseModel, dict))
|
||||
or isinstance(completion_response, HttpxBinaryResponseContent)
|
||||
and completion_response.usage is not None
|
||||
):
|
||||
if isinstance(completion_response, dict):
|
||||
usage_obj: dict | Usage | None = completion_response.get("usage", {})
|
||||
else:
|
||||
|
|
@ -1684,7 +1693,7 @@ def completion_cost(
|
|||
setattr(
|
||||
completion_response,
|
||||
"usage",
|
||||
litellm.Usage(**_usage_for_dump.model_dump()),
|
||||
_usage_from_model(_usage_for_dump),
|
||||
)
|
||||
if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)):
|
||||
_usage = _decisions_usage(usage_obj).model_dump()
|
||||
|
|
@ -2262,6 +2271,8 @@ def response_cost_calculator(
|
|||
### VERTEX LOCATION ###
|
||||
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
|
||||
region_name: str | None = None,
|
||||
custom_cost_per_token: CostPerToken | None = None,
|
||||
custom_cost_per_second: float | None = None,
|
||||
) -> float:
|
||||
"""
|
||||
Returns
|
||||
|
|
@ -2290,6 +2301,8 @@ def response_cost_calculator(
|
|||
optional_params=optional_params,
|
||||
custom_pricing=custom_pricing,
|
||||
base_model=base_model,
|
||||
custom_cost_per_token=custom_cost_per_token,
|
||||
custom_cost_per_second=custom_cost_per_second,
|
||||
prompt=prompt,
|
||||
standard_built_in_tools_params=standard_built_in_tools_params,
|
||||
litellm_model_name=litellm_model_name,
|
||||
|
|
@ -2873,6 +2886,7 @@ def _batch_rate(
|
|||
model_info: ModelInfo,
|
||||
key: Literal[
|
||||
"input_cost_per_audio_token_batches",
|
||||
"cache_read_input_audio_token_cost_batches",
|
||||
"input_cost_per_image_token_batches",
|
||||
"input_cost_per_video_token_batches",
|
||||
],
|
||||
|
|
@ -2958,7 +2972,12 @@ def batch_cost_calculator(
|
|||
total_completion_cost = 0.0
|
||||
if batch_rates.input is not None:
|
||||
total_prompt_cost = _batch_prompt_cost(
|
||||
usage, model_info, batch_rates.input, batch_rates.cache_read, batch_rates.cache_creation
|
||||
usage,
|
||||
model_info,
|
||||
batch_rates.input,
|
||||
batch_rates.cache_read,
|
||||
batch_rates.cache_creation,
|
||||
batch_rates.cache_creation_rates_by_ttl,
|
||||
)
|
||||
elif input_cost_per_token:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
|
|
@ -3003,10 +3022,13 @@ def _batch_prompt_cost(
|
|||
input_rate: float,
|
||||
cache_read_rate: float | None,
|
||||
cache_creation_rate: float | None,
|
||||
cache_creation_rates_by_ttl: Mapping[str, float],
|
||||
) -> float:
|
||||
details: Final = parse_prompt_tokens_details(usage)
|
||||
cached_tokens: Final = details["cache_hit_tokens"] if cache_read_rate is not None else 0
|
||||
written_tokens: Final = details["cache_creation_tokens"] if cache_creation_rate is not None else 0
|
||||
cached_audio_tokens: Final = details["cache_hit_audio_tokens"] if cache_read_rate is not None else 0
|
||||
bills_cache_writes: Final = cache_creation_rate is not None or bool(cache_creation_rates_by_ttl)
|
||||
written_tokens: Final = details["cache_creation_tokens"] if bills_cache_writes else 0
|
||||
audio_tokens, image_tokens, video_tokens = (
|
||||
details["audio_tokens"],
|
||||
details["image_tokens"],
|
||||
|
|
@ -3020,13 +3042,27 @@ def _batch_prompt_cost(
|
|||
+ audio_tokens * _batch_rate(model_info, "input_cost_per_audio_token_batches", input_rate)
|
||||
+ image_tokens * _batch_rate(model_info, "input_cost_per_image_token_batches", input_rate)
|
||||
+ video_tokens * _batch_rate(model_info, "input_cost_per_video_token_batches", input_rate)
|
||||
+ cached_tokens * (cache_read_rate or 0.0)
|
||||
+ written_tokens * (cache_creation_rate or 0.0)
|
||||
+ (cached_tokens - cached_audio_tokens) * (cache_read_rate or 0.0)
|
||||
+ cached_audio_tokens
|
||||
* _batch_rate(model_info, "cache_read_input_audio_token_cost_batches", cache_read_rate or 0.0)
|
||||
+ (
|
||||
calculate_cache_writing_cost(
|
||||
written_tokens,
|
||||
details["cache_creation_token_details"],
|
||||
input_rate if cache_creation_rate is None else cache_creation_rate,
|
||||
cache_creation_rates_by_ttl,
|
||||
)
|
||||
if bills_cache_writes
|
||||
else 0.0
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
_ATTRIBUTE_VALUE: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
def _attribute_value(obj: object, name: str) -> object:
|
||||
return getattr(obj, name)
|
||||
return _ATTRIBUTE_VALUE.validate_python(getattr(obj, name))
|
||||
|
||||
|
||||
def _summable_prompt_token_fields(prompt_tokens_details: BaseModel) -> list[str]:
|
||||
|
|
@ -3661,7 +3697,26 @@ def _get_model_info_or_none(model: str, custom_llm_provider: str) -> ModelInfo |
|
|||
return None
|
||||
|
||||
|
||||
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...]) -> float | None:
|
||||
_TranscriptionRateKey: TypeAlias = Literal[
|
||||
"input_cost_per_second", "input_cost_per_audio_token", "input_cost_per_token", "output_cost_per_token"
|
||||
]
|
||||
|
||||
|
||||
def _model_info_rate(info: ModelInfo, key: _TranscriptionRateKey) -> float | None:
|
||||
match key:
|
||||
case "input_cost_per_second":
|
||||
return info.get("input_cost_per_second")
|
||||
case "input_cost_per_audio_token":
|
||||
return info.get("input_cost_per_audio_token")
|
||||
case "input_cost_per_token":
|
||||
return info.get("input_cost_per_token")
|
||||
case "output_cost_per_token":
|
||||
return info.get("output_cost_per_token")
|
||||
case _:
|
||||
assert_never(key)
|
||||
|
||||
|
||||
def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[_TranscriptionRateKey, ...]) -> float | None:
|
||||
"""First of ``keys`` this entry prices, read off the raw ``litellm.model_cost`` entry
|
||||
because ``get_model_info`` synthesizes zero token rates for entries that omit them."""
|
||||
if info is None:
|
||||
|
|
@ -3670,12 +3725,18 @@ def _declared_transcription_rate(info: ModelInfo | None, keys: tuple[str, ...])
|
|||
if declared is None:
|
||||
return None
|
||||
return next(
|
||||
(float(value) for key in keys if declared.get(key) is not None and (value := info.get(key)) is not None),
|
||||
(
|
||||
float(value)
|
||||
for key in keys
|
||||
if declared.get(key) is not None and (value := _model_info_rate(info, key)) is not None
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _transcription_rate(keys: tuple[str, ...], override: ModelInfo | None, base: ModelInfo | None) -> float:
|
||||
def _transcription_rate(
|
||||
keys: tuple[_TranscriptionRateKey, ...], override: ModelInfo | None, base: ModelInfo | None
|
||||
) -> float:
|
||||
rates: Final = (_declared_transcription_rate(info, keys) for info in (override, base))
|
||||
return next((rate for rate in rates if rate is not None), 0.0)
|
||||
|
||||
|
|
|
|||
|
|
@ -122,6 +122,7 @@ class SpeechToCompletionBridgeHandler:
|
|||
if isinstance(result, ModelResponse):
|
||||
return self.transformation_handler.transform_response(
|
||||
model_response=result,
|
||||
model=model,
|
||||
response_format=requested_response_format if isinstance(requested_response_format, str) else None,
|
||||
)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Final, cast
|
|||
from typing_extensions import NotRequired, ReadOnly, TypedDict
|
||||
|
||||
from litellm.constants import OPENAI_CHAT_COMPLETION_PARAMS
|
||||
from litellm.litellm_core_utils.core_helpers import drops_unsupported_params
|
||||
from litellm.litellm_core_utils.hidden_params import get_hidden_params, served_from_cache, set_hidden_param
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -38,6 +39,11 @@ class ChatAudioParam(TypedDict):
|
|||
format: ReadOnly[NotRequired[str]]
|
||||
|
||||
|
||||
class ChatSpeechConfigAudioParam(TypedDict):
|
||||
speech_config: ReadOnly[Mapping[str, object]]
|
||||
format: ReadOnly[str]
|
||||
|
||||
|
||||
class SpeechToCompletionBridgeTransformationHandler:
|
||||
def _validate_response_format(
|
||||
self, model: str, custom_llm_provider: str, optional_params: Mapping[str, object]
|
||||
|
|
@ -49,6 +55,20 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return
|
||||
from litellm.exceptions import BadRequestError
|
||||
|
||||
if custom_llm_provider.startswith("vertex_ai"):
|
||||
from litellm.llms.vertex_ai.text_to_speech.transformation import VertexAITextToSpeechConfig
|
||||
|
||||
vertex_supported: Final = ", ".join(
|
||||
sorted({*GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS, *VertexAITextToSpeechConfig.GEMINI_FORMAT_MAPPINGS})
|
||||
)
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
f"Vertex AI Gemini TTS does not support response_format='{response_format}'."
|
||||
f" Supported response formats: {vertex_supported}."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
supported: Final = ", ".join(sorted(GEMINI_TTS_SUPPORTED_RESPONSE_FORMATS))
|
||||
raise BadRequestError(
|
||||
message=(
|
||||
|
|
@ -75,8 +95,27 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return response_format if isinstance(response_format, str) else None
|
||||
|
||||
def _chat_audio_param(
|
||||
self, model: str, voice: str | Mapping[str, object] | None, optional_params: Mapping[str, object]
|
||||
) -> ChatAudioParam | None:
|
||||
self,
|
||||
model: str,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
drop_params: bool,
|
||||
custom_llm_provider: str,
|
||||
) -> ChatAudioParam | ChatSpeechConfigAudioParam | None:
|
||||
import litellm
|
||||
|
||||
if isinstance(voice, Mapping) and self._is_gemini_tts_model(litellm.model_alias_map.get(model, model)):
|
||||
from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import (
|
||||
normalize_gemini_speech_config,
|
||||
reject_unknown_gemini_tts_voice_keys,
|
||||
)
|
||||
|
||||
reject_unknown_gemini_tts_voice_keys(model, voice, drop_params, custom_llm_provider)
|
||||
speech_audio: Final[ChatSpeechConfigAudioParam] = {
|
||||
"speech_config": normalize_gemini_speech_config(voice, drops_unsupported_params(drop_params)),
|
||||
"format": GEMINI_TTS_CHAT_AUDIO_FORMAT,
|
||||
}
|
||||
return speech_audio
|
||||
if not isinstance(voice, str):
|
||||
return None
|
||||
audio_format: Final = self._chat_audio_format(model, optional_params)
|
||||
|
|
@ -90,9 +129,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
self,
|
||||
model: str,
|
||||
input: str,
|
||||
voice: str | dict | None,
|
||||
optional_params: dict,
|
||||
litellm_params: dict,
|
||||
voice: str | Mapping[str, object] | None,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object],
|
||||
headers: dict,
|
||||
litellm_logging_obj: "LiteLLMLoggingObj",
|
||||
custom_llm_provider: str,
|
||||
|
|
@ -104,7 +143,13 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
"messages": [user_message],
|
||||
"modalities": ["audio"],
|
||||
**self._chat_completion_params(optional_params),
|
||||
"audio": self._chat_audio_param(model, voice, optional_params),
|
||||
"audio": self._chat_audio_param(
|
||||
model,
|
||||
voice,
|
||||
optional_params,
|
||||
drops_unsupported_params(litellm_params.get("drop_params")),
|
||||
custom_llm_provider,
|
||||
),
|
||||
**litellm_params,
|
||||
"headers": headers,
|
||||
"litellm_logging_obj": litellm_logging_obj,
|
||||
|
|
@ -153,8 +198,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return wav_header + pcm_data
|
||||
|
||||
def _is_gemini_tts_model(self, model: str) -> bool:
|
||||
"""Check if the model is a Gemini TTS model that returns PCM16 data."""
|
||||
return "gemini" in model.lower() and ("tts" in model.lower() or "preview-tts" in model.lower())
|
||||
from litellm.utils import is_gemini_tts_model
|
||||
|
||||
return is_gemini_tts_model(model)
|
||||
|
||||
def _gemini_tts_response_body(self, decoded_audio: bytes, response_format: str | None) -> tuple[bytes, str]:
|
||||
if response_format == GEMINI_TTS_RAW_RESPONSE_FORMAT:
|
||||
|
|
@ -162,7 +208,7 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
return self._convert_pcm16_to_wav(decoded_audio), "audio/wav"
|
||||
|
||||
def transform_response(
|
||||
self, model_response: "ModelResponse", response_format: str | None
|
||||
self, model_response: "ModelResponse", model: str, response_format: str | None
|
||||
) -> "HttpxBinaryResponseContent":
|
||||
import base64
|
||||
|
||||
|
|
@ -176,10 +222,9 @@ class SpeechToCompletionBridgeTransformationHandler:
|
|||
raise ValueError("No audio part found in the response")
|
||||
decoded_audio: Final = base64.b64decode(audio_part.data)
|
||||
|
||||
model: Final = getattr(model_response, "model", "")
|
||||
content, content_type = (
|
||||
self._gemini_tts_response_body(decoded_audio, response_format)
|
||||
if self._is_gemini_tts_model(model)
|
||||
if self._is_gemini_tts_model(model) or self._is_gemini_tts_model(model_response.model or "")
|
||||
else (decoded_audio, "audio/mpeg")
|
||||
)
|
||||
response: Final = httpx.Response(
|
||||
|
|
|
|||
|
|
@ -1273,7 +1273,7 @@ class MidStreamFallbackError(ServiceUnavailableError):
|
|||
self.message = _saved_message
|
||||
self.args = (_saved_message,)
|
||||
|
||||
original_headers: Final = getattr(original_exception, "litellm_response_headers", None)
|
||||
original_headers: Final[object] = getattr(original_exception, "litellm_response_headers", None)
|
||||
if original_headers is not None:
|
||||
self.litellm_response_headers = original_headers
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from mcp.types import Tool as MCPTool
|
|||
from openai.types.chat import ChatCompletionToolParam
|
||||
from openai.types.responses.function_tool_param import FunctionToolParam
|
||||
from openai.types.shared_params.function_definition import FunctionDefinition
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
|
|
@ -20,6 +21,8 @@ from litellm.constants import (
|
|||
from litellm.types.llms.anthropic import AnthropicMessagesTool
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
########################################################
|
||||
# List MCP Tool functions
|
||||
|
|
@ -200,15 +203,18 @@ async def call_mcp_tool(
|
|||
return tool_result
|
||||
|
||||
|
||||
def _decode_json_arguments(arguments: str) -> object:
|
||||
try:
|
||||
return _DECODED_JSON.validate_python(json.loads(arguments))
|
||||
except json.JSONDecodeError:
|
||||
return {}
|
||||
|
||||
|
||||
def _get_function_arguments(function: FunctionDefinition) -> dict:
|
||||
"""Helper to safely get and parse function arguments."""
|
||||
arguments = function.get("arguments", {})
|
||||
if isinstance(arguments, str):
|
||||
try:
|
||||
arguments = json.loads(arguments)
|
||||
except json.JSONDecodeError:
|
||||
arguments = {}
|
||||
return arguments if isinstance(arguments, dict) else {}
|
||||
arguments: Final = function.get("arguments", {})
|
||||
decoded: Final = _decode_json_arguments(arguments) if isinstance(arguments, str) else arguments
|
||||
return decoded if isinstance(decoded, dict) else {}
|
||||
|
||||
|
||||
def transform_openai_tool_call_request_to_mcp_tool_call_request(
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import asyncio
|
|||
import contextvars
|
||||
import time
|
||||
import uuid as uuid_module
|
||||
from collections.abc import Coroutine
|
||||
from collections.abc import Awaitable, Coroutine
|
||||
from functools import partial
|
||||
from typing import Final, Literal, cast
|
||||
|
||||
|
|
@ -293,6 +293,14 @@ def create_file(
|
|||
raise e
|
||||
|
||||
|
||||
async def _await_retrieved_file(pending: Awaitable[FileObject | OpenAIFileObject]) -> FileObject | OpenAIFileObject:
|
||||
return await pending
|
||||
|
||||
|
||||
async def _await_deleted_file(pending: Awaitable[object]) -> object:
|
||||
return await pending
|
||||
|
||||
|
||||
@client
|
||||
async def afile_retrieve(
|
||||
file_id: str,
|
||||
|
|
@ -325,7 +333,7 @@ async def afile_retrieve(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_retrieved_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
@ -505,7 +513,7 @@ async def afile_delete(
|
|||
func_with_context: Final = partial(ctx.run, func)
|
||||
init_response: Final = await loop.run_in_executor(None, func_with_context)
|
||||
if asyncio.iscoroutine(init_response):
|
||||
response = await init_response
|
||||
response = await _await_deleted_file(init_response)
|
||||
else:
|
||||
response = init_response
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import datetime
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Iterator
|
||||
from typing import TYPE_CHECKING, Any, Final, Optional, cast
|
||||
from typing import TYPE_CHECKING, Final, Optional, cast
|
||||
|
||||
import anyio
|
||||
|
||||
|
|
@ -163,7 +163,7 @@ class FileContentStreamingResponse:
|
|||
merged_hidden_params: Final = cast(
|
||||
"StandardLoggingHiddenParams",
|
||||
{
|
||||
**cast(dict[str, Any], payload.get("hidden_params") or {}),
|
||||
**(payload.get("hidden_params") or {}),
|
||||
**self._hidden_params,
|
||||
},
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,6 +37,7 @@ from litellm.harness.context import GatewayTarget
|
|||
from litellm.harness.errors import HarnessError, HarnessInstallFailed
|
||||
from litellm.harness.types import Harness, Usage
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.litellm_params import STREAM_USAGE_INJECTED_FIELD
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -690,18 +691,21 @@ class ModelEndpoint:
|
|||
async def _invoke_sdk(
|
||||
self,
|
||||
route: str,
|
||||
kwargs: dict[str, Any], # mutable-ok: injects stream_options into the SDK kwargs
|
||||
kwargs: dict[str, Any], # mutable-ok: SDK call kwargs built by _sdk_kwargs
|
||||
) -> object:
|
||||
if route == ROUTE_MESSAGES:
|
||||
return await litellm.anthropic.messages.acreate(**kwargs)
|
||||
if route == ROUTE_CHAT:
|
||||
if kwargs.get("stream"):
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
kwargs["stream_options"] = { # mutable-ok: JSON field sent to litellm.acompletion
|
||||
"include_usage": True,
|
||||
**stream_options,
|
||||
if not kwargs.get("stream"):
|
||||
return await litellm.acompletion(**kwargs)
|
||||
stream_options = kwargs.get("stream_options") or {} # mutable-ok: empty default for a JSON field
|
||||
return await litellm.acompletion(
|
||||
**{
|
||||
**kwargs,
|
||||
"stream_options": {"include_usage": True, **stream_options},
|
||||
STREAM_USAGE_INJECTED_FIELD: "include_usage" not in stream_options,
|
||||
}
|
||||
return await litellm.acompletion(**kwargs)
|
||||
)
|
||||
return await litellm.aresponses(**kwargs)
|
||||
|
||||
async def _call_sdk(self, route: str, body: Mapping[str, object]) -> Response:
|
||||
|
|
|
|||
|
|
@ -4,7 +4,6 @@ from __future__ import annotations
|
|||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
from litellm.harness.context import SessionContext
|
||||
from litellm.harness.errors import CapabilityUnsupported
|
||||
|
|
@ -38,5 +37,5 @@ class BaseHarnessHandler(ABC):
|
|||
async def resume(self, ctx: SessionContext, native_session_id: str) -> None:
|
||||
"""Continue the runtime's own session on the next turn."""
|
||||
|
||||
async def history(self, ctx: SessionContext) -> list[dict[str, Any]]: # mutable-ok: public history() API shape
|
||||
async def history(self, ctx: SessionContext) -> list[dict[str, object]]: # mutable-ok: public history() API shape
|
||||
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
|
||||
|
|
|
|||
|
|
@ -70,10 +70,10 @@ async def send_stdin(proc: Process, data: str) -> None:
|
|||
|
||||
|
||||
async def private_dir_for(sandbox: Sandbox) -> str:
|
||||
tempdir = getattr(sandbox, "tempdir", None)
|
||||
tempdir: Final = sandbox.tempdir if hasattr(sandbox, "tempdir") else None
|
||||
if tempdir is None:
|
||||
raise SandboxError(f"{type(sandbox).__name__} has no tempdir(); CLI harnesses need a private config dir")
|
||||
path: str = await tempdir()
|
||||
path: Final = await tempdir()
|
||||
return path
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -161,7 +161,7 @@ class DeepAgentsHandler(BaseHarnessHandler):
|
|||
|
||||
async def history(
|
||||
self, ctx: SessionContext
|
||||
) -> list[dict[str, Any]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
|
||||
) -> list[dict[str, object]]: # mutable-ok: BaseHarnessHandler.history API returns OpenAI message dicts
|
||||
agent, deps = self._require_agent()
|
||||
snapshot = await agent.aget_state(self._run_config(ctx, None))
|
||||
messages = (snapshot.values or MappingProxyType({})).get("messages") or ()
|
||||
|
|
|
|||
|
|
@ -12,11 +12,10 @@ import inspect
|
|||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import AsyncIterator, Callable, Coroutine, Generator, Mapping, Sequence
|
||||
from collections.abc import AsyncIterator, Awaitable, Callable, Coroutine, Generator, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
from typing import (
|
||||
Any,
|
||||
Final,
|
||||
get_args,
|
||||
)
|
||||
|
|
@ -331,11 +330,15 @@ async def _put_end(queue: asyncio.Queue[Event | _End], end: _End) -> None:
|
|||
queue.put_nowait(end)
|
||||
|
||||
|
||||
async def _await_decision(pending: Awaitable[object]) -> object:
|
||||
return await pending
|
||||
|
||||
|
||||
async def call_approval_handler(handler: ApprovalHandler, approval: Approval) -> None:
|
||||
"""Run on_approval (sync in a worker thread, or async) and resolve approval."""
|
||||
try:
|
||||
if inspect.iscoroutinefunction(handler):
|
||||
decision: object = await handler(approval)
|
||||
decision: object = await _await_decision(handler(approval))
|
||||
else:
|
||||
decision = await asyncio.to_thread(handler, approval)
|
||||
if inspect.isawaitable(decision):
|
||||
|
|
@ -749,7 +752,7 @@ class AsyncSession:
|
|||
|
||||
async def history(
|
||||
self,
|
||||
) -> list[dict[str, Any]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
|
||||
) -> list[dict[str, object]]: # mutable-ok: public API returns OpenAI-format message dicts from the handler
|
||||
if not self.harness_config.capabilities.history:
|
||||
raise CapabilityUnsupported(f"Harness.{self.config.harness.name} does not expose history")
|
||||
await self._ensure_ready()
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import contextvars
|
|||
import importlib
|
||||
from collections.abc import Coroutine
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Final, Literal, Optional, cast, overload
|
||||
from typing import TYPE_CHECKING, Final, Literal, cast, overload
|
||||
|
||||
from pydantic import TypeAdapter, ValidationError
|
||||
|
||||
|
|
@ -70,16 +70,17 @@ from litellm.utils import (
|
|||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
|
||||
# Cache for ImageEditRequestUtils to avoid repeated __getattr__ calls
|
||||
_ImageEditRequestUtils_cache: Optional["ImageEditRequestUtils"] = None
|
||||
_ImageEditRequestUtils_cache: type["ImageEditRequestUtils"] | None = None
|
||||
|
||||
|
||||
def _get_ImageEditRequestUtils() -> "ImageEditRequestUtils":
|
||||
def _get_ImageEditRequestUtils() -> type["ImageEditRequestUtils"]:
|
||||
"""Get ImageEditRequestUtils, loading it lazily if needed."""
|
||||
global _ImageEditRequestUtils_cache
|
||||
if _ImageEditRequestUtils_cache is None:
|
||||
# Access via module to trigger __getattr__ if not cached
|
||||
module: Final = importlib.import_module(__name__)
|
||||
_ImageEditRequestUtils_cache = module.ImageEditRequestUtils
|
||||
from litellm.images import main as images_main
|
||||
|
||||
_ImageEditRequestUtils_cache = images_main.ImageEditRequestUtils
|
||||
assert _ImageEditRequestUtils_cache is not None # Type narrowing for type checker
|
||||
return _ImageEditRequestUtils_cache
|
||||
|
||||
|
|
|
|||
|
|
@ -115,14 +115,16 @@ class AlertingHangingRequestCheck:
|
|||
if hanging_request_data.alerted:
|
||||
continue
|
||||
|
||||
request_status = await proxy_logging_obj.internal_usage_cache.async_get_cache(
|
||||
key=f"request_status:{hanging_request_data.request_id}",
|
||||
litellm_parent_otel_span=None,
|
||||
local_only=True,
|
||||
)
|
||||
# this means the request status was either success or fail
|
||||
# and is not hanging
|
||||
if request_status is not None:
|
||||
if (
|
||||
await proxy_logging_obj.internal_usage_cache.async_get_cache(
|
||||
key=f"request_status:{hanging_request_data.request_id}",
|
||||
litellm_parent_otel_span=None,
|
||||
local_only=True,
|
||||
)
|
||||
is not None
|
||||
):
|
||||
# clear this request from hanging request cache since the request was either success or failed
|
||||
self.hanging_request_cache._remove_key(
|
||||
key=request_id,
|
||||
|
|
|
|||
|
|
@ -56,8 +56,9 @@ async def fetch_user_spend_rows(
|
|||
today_str: Final = today.strftime("%Y-%m-%d")
|
||||
month_start_str: Final = today.replace(day=1).strftime("%Y-%m-%d")
|
||||
baseline_start_str: Final = (today - datetime.timedelta(days=max(baseline_days, 1))).strftime("%Y-%m-%d")
|
||||
raw: Final = await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
|
||||
return USER_SPEND_ROWS_ADAPTER.validate_python(raw)
|
||||
return USER_SPEND_ROWS_ADAPTER.validate_python(
|
||||
await prisma_client.db.query_raw(USER_SPEND_QUERY, today_str, month_start_str, baseline_start_str)
|
||||
)
|
||||
|
||||
|
||||
def _daily_threshold_event(row: UserSpendRow, args: SlackAlertingArgs, today_str: str) -> UserSpendAlertEvent | None:
|
||||
|
|
|
|||
|
|
@ -4,11 +4,16 @@ AgentOps integration for LiteLLM - Provides OpenTelemetry tracing for LLM calls
|
|||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
|
||||
from litellm.litellm_core_utils.core_helpers import as_str_mapping
|
||||
from litellm.llms.custom_httpx.http_handler import get_httpx_client
|
||||
|
||||
_DECODED_JSON: Final = TypeAdapter(object)
|
||||
|
||||
|
||||
@dataclass
|
||||
class AgentOpsConfig:
|
||||
|
|
@ -58,9 +63,10 @@ class AgentOps(OpenTelemetry):
|
|||
project_id = None
|
||||
if config.api_key:
|
||||
try:
|
||||
response: Final = self._fetch_auth_token(config.api_key, config.auth_endpoint)
|
||||
jwt_token = response.get("token")
|
||||
project_id = response.get("project_id")
|
||||
response: Final = as_str_mapping(self._fetch_auth_token(config.api_key, config.auth_endpoint))
|
||||
if response is not None:
|
||||
jwt_token = response.get("token")
|
||||
project_id = response.get("project_id")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
|
@ -78,12 +84,9 @@ class AgentOps(OpenTelemetry):
|
|||
"telemetry.sdk.name": "agentops",
|
||||
}
|
||||
|
||||
if project_id:
|
||||
resource_attrs["project.id"] = project_id
|
||||
self.resource_attributes = {**resource_attrs, "project.id": project_id} if project_id else resource_attrs
|
||||
|
||||
self.resource_attributes = resource_attrs
|
||||
|
||||
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> dict[str, Any]:
|
||||
def _fetch_auth_token(self, api_key: str, auth_endpoint: str) -> object:
|
||||
"""
|
||||
Fetch JWT authentication token from AgentOps API
|
||||
|
||||
|
|
@ -111,6 +114,6 @@ class AgentOps(OpenTelemetry):
|
|||
if response.status_code != 200:
|
||||
raise Exception(f"Failed to fetch auth token: {response.text}")
|
||||
|
||||
return response.json()
|
||||
return _DECODED_JSON.validate_python(response.json())
|
||||
finally:
|
||||
client.close()
|
||||
|
|
|
|||
|
|
@ -166,7 +166,7 @@ class ArizeLogger(OpenTelemetry):
|
|||
primary ``otel`` callback which handles proxy-level parent spans.
|
||||
"""
|
||||
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: Any | None):
|
||||
def set_attributes(self, span: Span, kwargs, response_obj: object):
|
||||
ArizeLogger.set_arize_attributes(span, kwargs, response_obj)
|
||||
|
||||
@staticmethod
|
||||
|
|
|
|||
|
|
@ -3,9 +3,10 @@
|
|||
|
||||
import os
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
from typing import Final, Protocol
|
||||
|
||||
import httpx
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm import verbose_logger
|
||||
|
|
@ -24,6 +25,18 @@ from litellm.utils import print_verbose
|
|||
API_BASE: Final = "https://api.braintrustdata.com/v1"
|
||||
|
||||
|
||||
class _BraintrustProjectBody(TypedDict):
|
||||
id: ReadOnly[str]
|
||||
|
||||
|
||||
class _BraintrustProjectResponse(Protocol):
|
||||
def json(self) -> _BraintrustProjectBody: ...
|
||||
|
||||
|
||||
def _project_id(response: _BraintrustProjectResponse) -> str:
|
||||
return response.json()["id"]
|
||||
|
||||
|
||||
def get_utc_datetime():
|
||||
import datetime as dt
|
||||
|
||||
|
|
@ -80,8 +93,7 @@ class BraintrustLogger(CustomLogger):
|
|||
headers=self.headers,
|
||||
json={"name": project_name},
|
||||
)
|
||||
project_dict: Final = response.json()
|
||||
project_id: Final = project_dict["id"]
|
||||
project_id: Final = _project_id(response)
|
||||
self._project_id_cache[project_name] = project_id
|
||||
return project_id
|
||||
except httpx.HTTPStatusError as e:
|
||||
|
|
@ -100,8 +112,7 @@ class BraintrustLogger(CustomLogger):
|
|||
headers=self.headers,
|
||||
json={"name": project_name},
|
||||
)
|
||||
project_dict: Final = response.json()
|
||||
project_id: Final = project_dict["id"]
|
||||
project_id: Final = _project_id(response)
|
||||
self._project_id_cache[project_name] = project_id
|
||||
return project_id
|
||||
except httpx.HTTPStatusError as e:
|
||||
|
|
@ -112,18 +123,14 @@ class BraintrustLogger(CustomLogger):
|
|||
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
|
||||
)
|
||||
|
||||
project_dict: Final = project.json()
|
||||
|
||||
self.default_project_id = project_dict["id"]
|
||||
self.default_project_id = _project_id(project)
|
||||
|
||||
def create_sync_default_project_and_experiment(self):
|
||||
project: Final = self.global_braintrust_sync_http_handler.post(
|
||||
f"{self.api_base}/project", headers=self.headers, json={"name": "litellm"}
|
||||
)
|
||||
|
||||
project_dict: Final = project.json()
|
||||
|
||||
self.default_project_id = project_dict["id"]
|
||||
self.default_project_id = _project_id(project)
|
||||
|
||||
def log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
verbose_logger.debug("REACHES BRAINTRUST SUCCESS")
|
||||
|
|
|
|||
|
|
@ -19,8 +19,9 @@
|
|||
"""Output modules for writing CBF data to various destinations."""
|
||||
|
||||
import zoneinfo
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Final
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import polars as pl
|
||||
|
|
@ -188,7 +189,7 @@ class CloudZeroStreamer:
|
|||
)
|
||||
raise
|
||||
|
||||
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> dict[str, Any]:
|
||||
def _prepare_batch_payload(self, batch_date: str, batch_data: pl.DataFrame, operation: str) -> Mapping[str, object]:
|
||||
"""Prepare batch payload according to CloudZero AnyCost API format."""
|
||||
# Convert batch_date to month for the API (YYYY-MM format)
|
||||
try:
|
||||
|
|
@ -205,7 +206,7 @@ class CloudZeroStreamer:
|
|||
if record:
|
||||
data_records.append(record)
|
||||
|
||||
payload: Final = {"month": month_str, "operation": operation, "data": data_records}
|
||||
payload: Final[Mapping[str, object]] = {"month": month_str, "operation": operation, "data": data_records}
|
||||
|
||||
return payload
|
||||
|
||||
|
|
|
|||
|
|
@ -352,7 +352,7 @@ class CustomGuardrail(CustomLogger):
|
|||
self,
|
||||
violation_message: str,
|
||||
request_data: dict[str, Any],
|
||||
detection_info: dict[str, Any] | None = None,
|
||||
detection_info: Mapping[str, object] | None = None,
|
||||
original_response: object = None,
|
||||
) -> None:
|
||||
"""
|
||||
|
|
@ -1368,7 +1368,10 @@ class CustomGuardrail(CustomLogger):
|
|||
if event_type is not None:
|
||||
guardrail_mode = event_type
|
||||
elif isinstance(self.event_hook, Mode):
|
||||
guardrail_mode = GuardrailMode(**dict(self.event_hook.model_dump()))
|
||||
guardrail_mode = GuardrailMode(
|
||||
tags=copy.deepcopy(self.event_hook.tags),
|
||||
default=copy.deepcopy(self.event_hook.default),
|
||||
)
|
||||
else:
|
||||
guardrail_mode = self.event_hook
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,6 @@ Usage:
|
|||
"""
|
||||
|
||||
from abc import abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
|
|
@ -154,7 +153,7 @@ class CustomSecretManager(BaseSecretManager):
|
|||
optional_params: dict | None = None,
|
||||
timeout: float | httpx.Timeout | None = None,
|
||||
tags: dict | list | None = None,
|
||||
) -> dict[str, Any]:
|
||||
) -> dict[str, object]:
|
||||
"""
|
||||
Asynchronously write a secret to your custom secret manager.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Duplicate -> https://github.com/confident-ai/deepeval/blob/main/deepeval/tracing/api.py
|
||||
from enum import Enum
|
||||
from typing import Any, ClassVar, Literal
|
||||
from typing import ClassVar, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field
|
||||
|
||||
|
|
@ -55,7 +55,7 @@ class TraceApi(LiteLLMBaseModel):
|
|||
tool_spans: list[BaseApiSpan] = Field(alias="toolSpans")
|
||||
start_time: str = Field(alias="startTime")
|
||||
end_time: str = Field(alias="endTime")
|
||||
metadata: dict[str, Any] | None = Field(None)
|
||||
metadata: dict[str, object] | None = Field(None)
|
||||
tags: list[str] | None = Field(None)
|
||||
environment: str | None = Field(None)
|
||||
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue