scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-05 08:07:05 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 137

Merge branch 'BerriAI:main' into main

Browse source
This commit is contained in:
abhijitjavelin 2025-09-28 07:53:06 +05:30 • committed by GitHub
commit aedcedb2b6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
155 changed files with 1646 additions and 1240 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-linting.yml vendored
View file

@ -61,7 +61,7 @@ jobs:
- name: Run MyPy type checking
run: |
cd litellm
poetry run mypy . --ignore-missing-imports --disable-error-code=var-annotated
poetry run mypy .
cd ..
- name: Check for circular imports

28
ci_cd/security_scans.sh
View file

@ -66,18 +66,34 @@ run_grype_scans() {
echo "Scanning locally built LiteLLM image for high-severity vulnerabilities..."
echo "Using locally built image: litellm:latest"
# Run grype scan and check for vulnerabilities with CVSS >= 4.0
# Allowlist of CVEs to be ignored in failure threshold/reporting
# - CVE-2025-8869: Not applicable on Python >=3.13 (PEP 706 implemented); pip fallback unused; no OS-level fix
ALLOWED_CVES=(
"CVE-2025-8869"
)
# Build JSON array of allowlisted CVE IDs for jq
ALLOWED_IDS_JSON=$(printf '%s\n' "${ALLOWED_CVES[@]}" | jq -R . | jq -s .)
echo "Checking for vulnerabilities with CVSS score >= 4.0..."
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq -r '.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) | .vulnerability.id' | wc -l)
echo "Allowlisted CVEs (ignored in threshold): ${ALLOWED_CVES[*]}"
HIGH_SEVERITY_COUNT=$(grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| .vulnerability.id' | wc -l)
if [ "$HIGH_SEVERITY_COUNT" -gt 0 ]; then
echo "ERROR: Found $HIGH_SEVERITY_COUNT vulnerabilities with CVSS score >= 4.0 in litellm:latest"
echo "Detailed vulnerability report:"
grype litellm:latest -o json | jq -r '
grype litellm:latest -o json | jq --argjson allow "$ALLOWED_IDS_JSON" -r '
["Package", "Version", "Vulnerability ID", "CVSS Score", "Severity", "Fix Version", "Description"],
(.matches[] | select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0) |
[.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description]) |
@tsv' | column -t -s $'\t'
(.matches[]
| select(.vulnerability.cvss[]?.metrics.baseScore >= 4.0)
| select((.vulnerability.id as $id | $allow | index($id) | not))
| [.artifact.name, .artifact.version, .vulnerability.id, .vulnerability.cvss[0].metrics.baseScore, .vulnerability.severity, (.vulnerability.fix.versions[0] // "No fix available"), .vulnerability.description])
| @tsv' | column -t -s $'\t'
exit 1
else
echo "No high-severity vulnerabilities (CVSS >= 4.0) found in litellm:latest"

110
cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md
View file

@ -26,7 +26,7 @@ git diff <previous_commit_hash> HEAD -- model_prices_and_context_window.json
### 2. Release Notes Structure
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable):
Follow this exact structure based on recent stable releases (v1.76.3-stable, v1.77.2-stable, v1.77.5-stable):
```markdown
---
@ -41,7 +41,7 @@ hide_table_of_contents: false
[Docker and pip installation tabs]
## Key Highlights
[3-5 bullet points of major features]
[3-5 bullet points of major features - prioritize MCP OAuth 2.0, scheduled key rotations, and major model updates]
## New Models / Updated Models
#### New Model Support
@ -65,26 +65,32 @@ hide_table_of_contents: false
## Management Endpoints / UI
#### Features
[UI and management features]
[UI and management features - group by functionality like Proxy CLI Auth, Virtual Keys, Models + Endpoints]
#### Bugs
[Management-related bug fixes]
## Logging / Guardrail Integrations
## Logging / Guardrail / Prompt Management Integrations
#### Features
[Organized by integration provider with proper doc links]
#### Guardrails
[Guardrail-specific features and fixes]
#### New Integration
[Major new integrations]
#### Prompt Management
[Prompt management integrations like BitBucket]
## Spend Tracking, Budgets and Rate Limiting
[Cost tracking, service tier pricing, rate limiting improvements]
## MCP Gateway
[MCP-specific features, OAuth 2.0, configuration improvements]
## Performance / Loadbalancing / Reliability improvements
[Infrastructure improvements]
[Infrastructure improvements, memory fixes, performance optimizations]
## General Proxy Improvements
[Other proxy-related changes]
## Documentation Updates
[Documentation improvements, guides, corrections - separate section for visibility]
## New Contributors
[List of first-time contributors]
@ -101,6 +107,11 @@ hide_table_of_contents: false
- CPU usage optimizations
- Timeout controls
- Worker configuration
- Memory leak fixes
- Cache performance improvements
- Database connection management
- Dependency management (fastuuid, etc.)
- Configuration management
**New Models/Updated Models:**
- Extract from model_prices_and_context_window.json diff
@ -132,20 +143,32 @@ hide_table_of_contents: false
- Dashboard improvements
- Team management
- Key management
- Proxy CLI authentication and improvements
- Virtual key management and scheduled rotations
- SSO configuration fixes
- Admin settings updates
- Management routes and endpoints
**Logging / Guardrail Integrations:**
**Logging / Guardrail / Prompt Management Integrations:**
- **Structure:**
- `#### Features` - organized by integration provider with proper doc links
- `#### Guardrails` - guardrail-specific features and fixes
- `#### Prompt Management` - prompt management integrations
- `#### New Integration` - major new integrations
- **Integration Categories:**
- **[DataDog](../../docs/proxy/logging#datadog)** - group all DataDog-related changes
- **[Langfuse](../../docs/proxy/logging#langfuse)** - Langfuse-specific features
- **[Prometheus](../../docs/proxy/logging#prometheus)** - monitoring improvements
- **[PostHog](../../docs/observability/posthog)** - observability integration
- **[SQS](../../docs/proxy/logging#sqs)** - SQS logging features
- **[Opik](../../docs/proxy/logging#opik)** - Opik integration improvements
- Other logging providers with proper doc links
- **Guardrail Categories:**
- LakeraAI, Presidio, Noma, and other guardrail providers
- **Prompt Management:**
- BitBucket, GitHub, and other prompt management integrations
- Use bullet points under each provider for multiple features
- Separate logging features from guardrails clearly
- Separate logging features from guardrails and prompt management clearly
### 4. Documentation Linking Strategy
@ -189,15 +212,26 @@ From git diff analysis, create tables like:
- `[Perf]`, `Performance`, `RPS` → Performance Improvements
- `[Bug]`, `[Bug Fix]`, `Fix` → Bug Fixes section
- `[Feat]`, `[Feature]`, `Add support` → Features section
- `[Docs]` → Documentation (usually exclude from main sections)
- `[Docs]` → Documentation Updates section
- Provider names (Gemini, OpenAI, etc.) → Group under provider
- `MCP`, `oauth`, `Model Context Protocol` → MCP Gateway
- `service_tier`, `priority`, `cost tracking` → Spend Tracking, Budgets and Rate Limiting
**By PR Content Analysis:**
- New model additions → New Models section
- UI changes → Management Endpoints/UI
- Logging/observability → Logging/Guardrail Integrations
- Rate limiting/budgets → Performance/Reliability
- Authentication → Management Endpoints
- Logging/observability → Logging/Guardrail/Prompt Management Integrations
- Rate limiting/budgets → Spend Tracking, Budgets and Rate Limiting
- Authentication → Management Endpoints/UI
- MCP-related changes → MCP Gateway
- Documentation updates → Documentation Updates
- Performance/memory fixes → Performance/Loadbalancing/Reliability improvements
**Special Categorization Rules:**
- **Service tier pricing** (OpenAI priority/flex) → Spend Tracking section (NOT provider features)
- **Cost breakdown in logging** → Spend Tracking section
- **MCP configuration/OAuth** → MCP Gateway (NOT General Proxy Improvements)
- **All documentation PRs** → Documentation Updates section for visibility
### 7. Writing Style Guidelines
@ -226,6 +260,18 @@ From git diff analysis, create tables like:
- Ensure model pricing is accurate
- Confirm provider names are consistent
- Review for typos and formatting issues
- **Count PRs by section** - Provide final count like:
```
## MM/DD/YYYY
* New Models / Updated Models: XX
* LLM API Endpoints: XX
* Management Endpoints / UI: XX
* Logging / Guardrail / Prompt Management Integrations: XX
* Spend Tracking, Budgets and Rate Limiting: XX
* MCP Gateway: XX
* Performance / Loadbalancing / Reliability improvements: XX
* Documentation Updates: XX
```
### 9. Common Patterns to Follow
@ -295,6 +341,40 @@ This release has a known issue...
- Complex configuration options
- Migration requirements
### 11. New Sections and Categories (Added in v1.77.5)
**MCP Gateway Section:**
- All MCP-related changes go here (not in General Proxy Improvements)
- OAuth 2.0 flow improvements
- MCP configuration and tools
- Server management features
**Spend Tracking, Budgets and Rate Limiting Section:**
- Service tier pricing (OpenAI priority/flex pricing)
- Cost tracking and breakdown features
- Rate limiting improvements (Parallel Request Limiter v3)
- Priority reservation fixes
- Metadata handling for rate limiting
**Documentation Updates Section:**
- Create separate section for all documentation improvements
- Include provider documentation fixes
- Model reference updates
- New guides and tutorials
- Documentation corrections and clarifications
- This gives documentation changes proper visibility
**Management Endpoints / UI Grouping:**
- Group related features under sub-categories:
- **Proxy CLI Auth** - CLI authentication improvements
- **Virtual Keys** - Key rotation and management
- **Models + Endpoints** - Provider and endpoint management
**Logging Section Expansion:**
- Rename to "Logging / Guardrail / Prompt Management Integrations"
- Add **Prompt Management** subsection for BitBucket, GitHub integrations
- Keep guardrails separate from logging features
## Example Command Workflow
```bash

BIN
docs/my-website/img/release_notes/perf_imp.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.2 MiB

BIN
docs/my-website/img/release_notes/quota.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 276 KiB

22
docs/my-website/release_notes/v1.77.3-stable/index.md
View file

@ -1,5 +1,5 @@
---
title: "[Preview] v1.77.3-stable - Priority Based Rate Limiting"
title: "v1.77.3-stable - Priority Based Rate Limiting"
slug: "v1-77-3"
date: 2025-09-21T10:00:00
authors:
@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem';
docker run \
-e STORE_MODEL_IN_DB=True \
-p 4000:4000 \
ghcr.io/berriai/litellm:main-v1.77.3.rc.1
ghcr.io/berriai/litellm:v1.77.3-stable
```
</TabItem>
@ -51,11 +51,27 @@ pip install litellm==1.77.3
## Priority Quota Reservation
This release adds support for priority quota reservation. This allows Proxy Admins to reserve specific percentages of model capacity for different use cases.
This is great for use cases where you want to ensure your realtime use cases must always get priority responses and background development jobs can take longer.
<Image img={require('../../img/release_notes/quota.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release adds support for priority quota reservation. This allows **Proxy Admins** to reserve TPM/RPM capacity for keys based on metadata priority levels, ensuring critical production workloads get guaranteed access regardless of development traffic volume.
Get started [here](../../docs/proxy/dynamic_rate_limit#priority-quota-reservation)
<iframe width="700" height="500" src="https://www.loom.com/embed/1b54b93139ee415d959402cc0629f3f7" frameborder="0" webkitallowfullscreen mozallowfullscreen allowfullscreen></iframe>
## +550 RPS Performance Improvements
<Image img={require('../../img/release_notes/perf_imp.png')} style={{ width: '800px', height: 'auto' }} />
<br/>
This release delivers significant RPS improvements through targeted optimizations.
We've achieved a +500 RPS boost by fixing cache type inconsistencies that were causing frequent cache misses, plus an additional +50 RPS by removing unnecessary coroutine checks from the hot path.
## New Models / Updated Models

285
docs/my-website/release_notes/v1.77.5-stable/index.md Normal file
View file

@ -0,0 +1,285 @@
---
title: "[Preview] v1.77.5-stable - MCP OAuth 2.0 Support"
slug: "v1-77-5"
date: 2025-09-29T10:00:00
authors:
- name: Krrish Dholakia
title: CEO, LiteLLM
url: https://www.linkedin.com/in/krish-d/
image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg
- name: Ishaan Jaff
title: CTO, LiteLLM
url: https://www.linkedin.com/in/reffajnaahsi/
image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg
hide_table_of_contents: false
---
import Image from '@theme/IdealImage';
import Tabs from '@theme/Tabs';
import TabItem from '@theme/TabItem';
## Deploy this version
<Tabs>
<TabItem value="docker" label="Docker">
``` showLineNumbers title="docker run litellm"
```
</TabItem>
<TabItem value="pip" label="Pip">
``` showLineNumbers title="pip install litellm"
```
</TabItem>
</Tabs>
---
## Key Highlights
- **MCP OAuth 2.0 Support** - Enhanced authentication for Model Context Protocol integrations
- **Scheduled Key Rotations** - Automated key rotation capabilities for enhanced security
- **New Gemini 2.5 Flash & Flash-lite Models** - Latest September 2025 preview models with improved pricing and features
- **Performance Improvements** - Critical InMemoryCache unbounded growth resolution
## New Models / Updated Models
#### New Model Support
| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features |
| -------- | ----- | -------------- | ------------------- | -------------------- | -------- |
| Gemini | `gemini-2.5-flash-preview-09-2025` | 1M | $0.30 | $2.50 | Chat, reasoning, vision, audio |
| Gemini | `gemini-2.5-flash-lite-preview-09-2025` | 1M | $0.10 | $0.40 | Chat, reasoning, vision, audio |
| Gemini | `gemini-flash-latest` | 1M | $0.30 | $2.50 | Chat, reasoning, vision, audio |
| Gemini | `gemini-flash-lite-latest` | 1M | $0.10 | $0.40 | Chat, reasoning, vision, audio |
| DeepSeek | `deepseek-chat` | 131K | $0.60 | $1.70 | Chat, function calling, caching |
| DeepSeek | `deepseek-reasoner` | 131K | $0.60 | $1.70 | Chat, reasoning |
| Bedrock | `deepseek.v3-v1:0` | 164K | $0.58 | $1.68 | Chat, reasoning, function calling |
| Azure | `azure/gpt-5-codex` | 272K | $1.25 | $10.00 | Responses API, reasoning, vision |
| OpenAI | `gpt-5-codex` | 272K | $1.25 | $10.00 | Responses API, reasoning, vision |
| SambaNova | `sambanova/DeepSeek-V3.1` | 33K | $3.00 | $4.50 | Chat, reasoning, function calling |
| SambaNova | `sambanova/gpt-oss-120b` | 131K | $3.00 | $4.50 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-coder-480b-a35b-v1:0` | 262K | $0.22 | $1.80 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-235b-a22b-2507-v1:0` | 262K | $0.22 | $0.88 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-coder-30b-a3b-v1:0` | 262K | $0.15 | $0.60 | Chat, reasoning, function calling |
| Bedrock | `qwen.qwen3-32b-v1:0` | 131K | $0.15 | $0.60 | Chat, reasoning, function calling |
| Vertex AI | `vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas` | 262K | $0.15 | $1.20 | Chat, function calling |
| Vertex AI | `vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas` | 262K | $0.15 | $1.20 | Chat, function calling |
| Vertex AI | `vertex_ai/deepseek-ai/deepseek-v3.1-maas` | 164K | $1.35 | $5.40 | Chat, reasoning, function calling |
| OpenRouter | `openrouter/x-ai/grok-4-fast:free` | 2M | $0.00 | $0.00 | Chat, reasoning, function calling |
| XAI | `xai/grok-4-fast-reasoning` | 2M | $0.20 | $0.50 | Chat, reasoning, function calling |
| XAI | `xai/grok-4-fast-non-reasoning` | 2M | $0.20 | $0.50 | Chat, function calling |
#### Features
- **[Gemini](../../docs/providers/gemini)**
- Added Gemini 2.5 Flash and Flash-lite preview models (September 2025 release) with improved pricing - [PR #14948](https://github.com/BerriAI/litellm/pull/14948)
- Added new Anthropic web fetch tool support - [PR #14951](https://github.com/BerriAI/litellm/pull/14951)
- **[XAI](../../docs/providers/xai)**
- Add xai/grok-4-fast models - [PR #14833](https://github.com/BerriAI/litellm/pull/14833)
- **[Anthropic](../../docs/providers/anthropic)**
- Updated Claude Sonnet 4 configs to reflect million-token context window pricing - [PR #14639](https://github.com/BerriAI/litellm/pull/14639)
- Added supported text field to anthropic citation response - [PR #14164](https://github.com/BerriAI/litellm/pull/14164)
- **[Bedrock](../../docs/providers/bedrock)**
- Added support for Qwen models family & Deepseek 3.1 to Amazon Bedrock - [PR #14845](https://github.com/BerriAI/litellm/pull/14845)
- Support requestMetadata in Bedrock Converse API - [PR #14570](https://github.com/BerriAI/litellm/pull/14570)
- **[Vertex AI](../../docs/providers/vertex)**
- Added vertex_ai/qwen models and azure/gpt-5-codex - [PR #14844](https://github.com/BerriAI/litellm/pull/14844)
- Update vertex ai qwen model pricing - [PR #14828](https://github.com/BerriAI/litellm/pull/14828)
- Vertex AI Context Caching: use Vertex ai API v1 instead of v1beta1 and accept 'cachedContent' param - [PR #14831](https://github.com/BerriAI/litellm/pull/14831)
- **[SambaNova](../../docs/providers/sambanova)**
- Add sambanova deepseek v3.1 and gpt-oss-120b - [PR #14866](https://github.com/BerriAI/litellm/pull/14866)
- **[OpenAI](../../docs/providers/openai)**
- Fix inconsistent token configs for gpt-5 models - [PR #14942](https://github.com/BerriAI/litellm/pull/14942)
- GPT-3.5-Turbo price updated - [PR #14858](https://github.com/BerriAI/litellm/pull/14858)
- **[OpenRouter](../../docs/providers/openrouter)**
- Add gpt-5 and gpt-5-codex to OpenRouter cost map - [PR #14879](https://github.com/BerriAI/litellm/pull/14879)
- **[VLLM](../../docs/providers/vllm)**
- Fix vllm passthrough - [PR #14778](https://github.com/BerriAI/litellm/pull/14778)
- **[Flux](../../docs/image_generation)**
- Support flux image edit - [PR #14790](https://github.com/BerriAI/litellm/pull/14790)
### Bug Fixes
- **[Anthropic](../../docs/providers/anthropic)**
- Fix: Support claude code auth via subscription (anthropic) - [PR #14821](https://github.com/BerriAI/litellm/pull/14821)
- Fix Anthropic streaming IDs - [PR #14965](https://github.com/BerriAI/litellm/pull/14965)
- Revert incorrect changes to sonnet-4 max output tokens - [PR #14933](https://github.com/BerriAI/litellm/pull/14933)
- **[OpenAI](../../docs/providers/openai)**
- Fix a bug where openai image edit silently ignores multiple images - [PR #14893](https://github.com/BerriAI/litellm/pull/14893)
- **[VLLM](../../docs/providers/vllm)**
- Fix: vLLM provider's rerank endpoint from /v1/rerank to /rerank - [PR #14938](https://github.com/BerriAI/litellm/pull/14938)
#### New Provider Support
- **[W&B Inference](../../docs/providers/wandb)**
- Add W&B Inference to LiteLLM - [PR #14416](https://github.com/BerriAI/litellm/pull/14416)
---
## LLM API Endpoints
#### Features
- **General**
- Add SDK support for additional headers - [PR #14761](https://github.com/BerriAI/litellm/pull/14761)
- Add shared_session parameter for aiohttp ClientSession reuse - [PR #14721](https://github.com/BerriAI/litellm/pull/14721)
#### Bugs
- **General**
- Fix: Streaming tool call index assignment for multiple tool calls - [PR #14587](https://github.com/BerriAI/litellm/pull/14587)
- Fix load credentials in token counter proxy - [PR #14808](https://github.com/BerriAI/litellm/pull/14808)
---
## Management Endpoints / UI
#### Features
- **Proxy CLI Auth**
- Allow re-using cli auth token - [PR #14780](https://github.com/BerriAI/litellm/pull/14780)
- Create a python method to login using litellm proxy - [PR #14782](https://github.com/BerriAI/litellm/pull/14782)
- Fixes for LiteLLM Proxy CLI to Auth to Gateway - [PR #14836](https://github.com/BerriAI/litellm/pull/14836)
**Virtual Keys**
- Initial support for scheduled key rotations - [PR #14877](https://github.com/BerriAI/litellm/pull/14877)
- Allow scheduling key rotations when creating virtual keys - [PR #14960](https://github.com/BerriAI/litellm/pull/14960)
**Models + Endpoints**
- Fix: added Oracle to provider's list - [PR #14835](https://github.com/BerriAI/litellm/pull/14835)
#### Bugs
- **SSO** - Fix: SSO "Clear" button writes empty values instead of removing SSO config - [PR #14826](https://github.com/BerriAI/litellm/pull/14826)
- **Admin Settings** - Remove useful links from admin settings - [PR #14918](https://github.com/BerriAI/litellm/pull/14918)
- **Management Routes** - Add /user/list to management routes - [PR #14868](https://github.com/BerriAI/litellm/pull/14868)
---
## Logging / Guardrail / Prompt Management Integrations
#### Features
- **[DataDog](../../docs/proxy/logging#datadog)**
- Logging - `datadog` callback Log message content w/o sending to datadog - [PR #14909](https://github.com/BerriAI/litellm/pull/14909)
- **[Langfuse](../../docs/proxy/logging#langfuse)**
- Adding langfuse usage details for cached tokens - [PR #10955](https://github.com/BerriAI/litellm/pull/10955)
- **[Opik](../../docs/proxy/logging#opik)**
- Improve opik integration code - [PR #14888](https://github.com/BerriAI/litellm/pull/14888)
- **[SQS](../../docs/proxy/logging#sqs)**
- Error logging support for SQS Logger - [PR #14974](https://github.com/BerriAI/litellm/pull/14974)
#### Guardrails
- **LakeraAI v2 Guardrail** - Ensure exception is raised correctly - [PR #14867](https://github.com/BerriAI/litellm/pull/14867)
- **Presidio Guardrail** - Support custom entity types in Presidio guardrail with Union[PiiEntityType, str] - [PR #14899](https://github.com/BerriAI/litellm/pull/14899)
- **Noma Guardrail** - Add noma guardrail provider to ui - [PR #14415](https://github.com/BerriAI/litellm/pull/14415)
#### Prompt Management
- **BitBucket Integration** - Add BitBucket Integration for Prompt Management - [PR #14882](https://github.com/BerriAI/litellm/pull/14882)
---
## Spend Tracking, Budgets and Rate Limiting
- **Service Tier Pricing** - Add service_tier based pricing support for openai (BOTH Service & Priority Support) - [PR #14796](https://github.com/BerriAI/litellm/pull/14796)
- **Cost Tracking** - Show input, output, tool call cost breakdown in StandardLoggingPayload - [PR #14921](https://github.com/BerriAI/litellm/pull/14921)
- **Parallel Request Limiter v3**
- Ensure Lua scripts can execute on redis cluster - [PR #14968](https://github.com/BerriAI/litellm/pull/14968)
- Fix: get metadata info from both metadata and litellm_metadata fields - [PR #14783](https://github.com/BerriAI/litellm/pull/14783)
- **Priority Reservation** - Fix: Priority Reservation: keys without priority metadata receive higher priority than keys with explicit priority configurations - [PR #14832](https://github.com/BerriAI/litellm/pull/14832)
---
## MCP Gateway
- **MCP Configuration** - Enable custom fields in mcp_info configuration - [PR #14794](https://github.com/BerriAI/litellm/pull/14794)
- **MCP Tools** - Remove server_name prefix from list_tools - [PR #14720](https://github.com/BerriAI/litellm/pull/14720)
- **OAuth Flow** - Initial commit for v2 oauth flow - [PR #14964](https://github.com/BerriAI/litellm/pull/14964)
---
## Performance / Loadbalancing / Reliability improvements
- **Memory Leak Fix** - Fix InMemoryCache unbounded growth when TTLs are set - [PR #14869](https://github.com/BerriAI/litellm/pull/14869)
- **Cache Performance** - Fix: cache root cause - [PR #14827](https://github.com/BerriAI/litellm/pull/14827)
- **Concurrency Fix** - Fix concurrency/scaling when many Python threads do streaming using *sync* completions - [PR #14816](https://github.com/BerriAI/litellm/pull/14816)
- **Performance Optimization** - Fix: reduce get_deployment cost to O(1) - [PR #14967](https://github.com/BerriAI/litellm/pull/14967)
- **Performance Optimization** - Fix: remove slow string operation - [PR #14955](https://github.com/BerriAI/litellm/pull/14955)
- **DB Connection Management** - Fix: DB connection state retries - [PR #14925](https://github.com/BerriAI/litellm/pull/14925)
---
## Documentation Updates
- **Provider Documentation** - Fix docs for provider_specific_params.md - [PR #14787](https://github.com/BerriAI/litellm/pull/14787)
- **Model References** - Update model references from gemini-pro to gemini-2.5-pro - [PR #14775](https://github.com/BerriAI/litellm/pull/14775)
- **Letta Guide** - Add Letta Guide documentation - [PR #14798](https://github.com/BerriAI/litellm/pull/14798)
- **README** - Make the README document clearer - [PR #14860](https://github.com/BerriAI/litellm/pull/14860)
- **Session Management** - Update docs for session management availability - [PR #14914](https://github.com/BerriAI/litellm/pull/14914)
- **Cost Documentation** - Add documentation for additional cost-related keys in custom pricing - [PR #14949](https://github.com/BerriAI/litellm/pull/14949)
- **Azure Passthrough** - Add azure passthrough documentation - [PR #14958](https://github.com/BerriAI/litellm/pull/14958)
- **General Documentation** - Doc updates sept 2025 - [PR #14769](https://github.com/BerriAI/litellm/pull/14769)
- Clarified bridging between endpoints and mode in docs.
- Added Vertex AI Gemini API configuration as an alternative in relevant guides.
Linked AWS authentication info in the Bedrock guardrails documentation.
- Added Cancel Response API usage with code snippets
- Clarified that SSO (Single Sign-On) is free for up to 5 users:
- Alphabetized sidebar, leaving quick start / intros at top of categories
- Documented max_connections under cache_params.
- Clarified IAM AssumeRole Policy requirements.
- Added transform utilities example to Getting Started (showing request transformation).
- Added references to models.litellm.ai as the full models list in various docs.
- Added a code snippet for async_post_call_success_hook.
- Removed broken links to callbacks management guide. - Reformatted and linked cookbooks + other relevant docs
- **Documentation Corrections** - Corrected docs updates sept 2025 - [PR #14916](https://github.com/BerriAI/litellm/pull/14916)
---
## New Contributors
* @uzaxirr made their first contribution in [PR #14761](https://github.com/BerriAI/litellm/pull/14761)
* @xprilion made their first contribution in [PR #14416](https://github.com/BerriAI/litellm/pull/14416)
* @CH-GAGANRAJ made their first contribution in [PR #14779](https://github.com/BerriAI/litellm/pull/14779)
* @otaviofbrito made their first contribution in [PR #14778](https://github.com/BerriAI/litellm/pull/14778)
* @danielmklein made their first contribution in [PR #14639](https://github.com/BerriAI/litellm/pull/14639)
* @Jetemple made their first contribution in [PR #14826](https://github.com/BerriAI/litellm/pull/14826)
* @akshoop made their first contribution in [PR #14818](https://github.com/BerriAI/litellm/pull/14818)
* @hazyone made their first contribution in [PR #14821](https://github.com/BerriAI/litellm/pull/14821)
* @leventov made their first contribution in [PR #14816](https://github.com/BerriAI/litellm/pull/14816)
* @fabriciojoc made their first contribution in [PR #10955](https://github.com/BerriAI/litellm/pull/10955)
* @onlylonly made their first contribution in [PR #14845](https://github.com/BerriAI/litellm/pull/14845)
* @Copilot made their first contribution in [PR #14869](https://github.com/BerriAI/litellm/pull/14869)
* @arsh72 made their first contribution in [PR #14899](https://github.com/BerriAI/litellm/pull/14899)
* @berri-teddy made their first contribution in [PR #14914](https://github.com/BerriAI/litellm/pull/14914)
* @vpbill made their first contribution in [PR #14415](https://github.com/BerriAI/litellm/pull/14415)
* @kgritesh made their first contribution in [PR #14893](https://github.com/BerriAI/litellm/pull/14893)
* @oytunkutrup1 made their first contribution in [PR #14858](https://github.com/BerriAI/litellm/pull/14858)
* @nherment made their first contribution in [PR #14933](https://github.com/BerriAI/litellm/pull/14933)
* @deepanshululla made their first contribution in [PR #14974](https://github.com/BerriAI/litellm/pull/14974)
* @TeddyAmkie made their first contribution in [PR #14758](https://github.com/BerriAI/litellm/pull/14758)
* @SmartManoj made their first contribution in [PR #14775](https://github.com/BerriAI/litellm/pull/14775)
* @uc4w6c made their first contribution in [PR #14720](https://github.com/BerriAI/litellm/pull/14720)
* @luizrennocosta made their first contribution in [PR #14783](https://github.com/BerriAI/litellm/pull/14783)
* @AlexsanderHamir made their first contribution in [PR #14827](https://github.com/BerriAI/litellm/pull/14827)
* @dharamendrak made their first contribution in [PR #14721](https://github.com/BerriAI/litellm/pull/14721)
* @TomeHirata made their first contribution in [PR #14164](https://github.com/BerriAI/litellm/pull/14164)
* @mrFranklin made their first contribution in [PR #14860](https://github.com/BerriAI/litellm/pull/14860)
* @luisfucros made their first contribution in [PR #14866](https://github.com/BerriAI/litellm/pull/14866)
* @huangyafei made their first contribution in [PR #14879](https://github.com/BerriAI/litellm/pull/14879)
* @thiswillbeyourgithub made their first contribution in [PR #14949](https://github.com/BerriAI/litellm/pull/14949)
* @Maximgitman made their first contribution in [PR #14965](https://github.com/BerriAI/litellm/pull/14965)
* @subnet-dev made their first contribution in [PR #14938](https://github.com/BerriAI/litellm/pull/14938)
* @22mSqRi made their first contribution in [PR #14972](https://github.com/BerriAI/litellm/pull/14972)
---
## **[Full Changelog](https://github.com/BerriAI/litellm/compare/v1.77.3.rc.1...v1.77.5.rc.1)**

7
enterprise/litellm_enterprise/integrations/prometheus.py
View file

@ -2262,9 +2262,12 @@ def get_custom_labels_from_metadata(metadata: dict) -> Dict[str, str]:
keys_parts = key.split(".")
# Traverse through the dictionary using the parts
value = metadata
value: Any = metadata
for part in keys_parts:
value = value.get(part, None) # Get the value, return None if not found
if isinstance(value, dict):
value = value.get(part, None) # Get the value, return None if not found
else:
value = None
if value is None:
break

9
litellm/__init__.py
View file

@ -377,7 +377,9 @@ public_model_groups: Optional[List[str]] = None
public_model_groups_links: Dict[str, str] = {}
#### REQUEST PRIORITIZATION ######
priority_reservation: Optional[Dict[str, float]] = None
priority_reservation_settings: "PriorityReservationSettings" = PriorityReservationSettings()
priority_reservation_settings: "PriorityReservationSettings" = (
PriorityReservationSettings()
)
######## Networking Settings ########
@ -443,7 +445,7 @@ def identify(event_details):
####### ADDITIONAL PARAMS ################### configurable params if you use proxy models like Helicone, map spend to org id, etc.
api_base: Optional[str] = None
headers = None
api_version = None
api_version: Optional[str] = None
organization = None
project = None
config_path = None
@ -494,7 +496,7 @@ azure_ai_models: Set = set()
jina_ai_models: Set = set()
voyage_models: Set = set()
infinity_models: Set = set()
heroku_models: Set = set()
heroku_models: Set = set()
databricks_models: Set = set()
cloudflare_models: Set = set()
codestral_models: Set = set()
@ -1357,6 +1359,7 @@ from .passthrough import allm_passthrough_route, llm_passthrough_route
### GLOBAL CONFIG ###
global_bitbucket_config: Optional[Dict[str, Any]] = None
def set_global_bitbucket_config(config: Dict[str, Any]) -> None:
"""Set global BitBucket configuration for prompt management."""
global global_bitbucket_config

325
litellm/litellm_core_utils/litellm_logging.py
View file

@ -301,9 +301,9 @@ class Logging(LiteLLMLoggingBaseClass):
self.litellm_trace_id: str = litellm_trace_id or str(uuid.uuid4())
self.function_id = function_id
self.streaming_chunks: List[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: List[
Any
] = [] # for generating complete stream response
self.sync_streaming_chunks: List[Any] = (
[]
) # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
# Initialize dynamic callbacks
@ -344,7 +344,7 @@ class Logging(LiteLLMLoggingBaseClass):
litellm_params = scrub_sensitive_keys_in_metadata(litellm_params)
self.litellm_params = litellm_params
# Initialize cost breakdown field
self.cost_breakdown: Optional[CostBreakdown] = None
@ -676,9 +676,9 @@ class Logging(LiteLLMLoggingBaseClass):
if anthropic_cache_control_logger := AnthropicCacheControlHook.get_custom_logger_for_anthropic_cache_control_hook(
non_default_params
):
self.model_call_details[
"prompt_integration"
] = anthropic_cache_control_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
anthropic_cache_control_logger.__class__.__name__
)
return anthropic_cache_control_logger
#########################################################
@ -690,9 +690,9 @@ class Logging(LiteLLMLoggingBaseClass):
internal_usage_cache=None,
llm_router=None,
)
self.model_call_details[
"prompt_integration"
] = vector_store_custom_logger.__class__.__name__
self.model_call_details["prompt_integration"] = (
vector_store_custom_logger.__class__.__name__
)
return vector_store_custom_logger
return None
@ -744,9 +744,9 @@ class Logging(LiteLLMLoggingBaseClass):
model
): # if model name was changes pre-call, overwrite the initial model call name with the new one
self.model_call_details["model"] = model
self.model_call_details["litellm_params"][
"api_base"
] = self._get_masked_api_base(additional_args.get("api_base", ""))
self.model_call_details["litellm_params"]["api_base"] = (
self._get_masked_api_base(additional_args.get("api_base", ""))
)
def pre_call(self, input, api_key, model=None, additional_args={}): # noqa: PLR0915
# Log the exact input to the LLM API
@ -775,10 +775,10 @@ class Logging(LiteLLMLoggingBaseClass):
try:
# [Non-blocking Extra Debug Information in metadata]
if turn_off_message_logging is True:
_metadata[
"raw_request"
] = "redacted by litellm. \
_metadata["raw_request"] = (
"redacted by litellm. \
'litellm.turn_off_message_logging=True'"
)
else:
curl_command = self._get_request_curl_command(
api_base=additional_args.get("api_base", ""),
@ -789,32 +789,32 @@ class Logging(LiteLLMLoggingBaseClass):
_metadata["raw_request"] = str(curl_command)
# split up, so it's easier to parse in the UI
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
ignore_sensitive_headers=True,
),
error=None,
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
raw_request_api_base=str(
additional_args.get("api_base") or ""
),
raw_request_body=self._get_raw_request_body(
additional_args.get("complete_input_dict", {})
),
raw_request_headers=self._get_masked_headers(
additional_args.get("headers", {}) or {},
ignore_sensitive_headers=True,
),
error=None,
)
)
except Exception as e:
self.model_call_details[
"raw_request_typed_dict"
] = RawRequestTypedDict(
error=str(e),
self.model_call_details["raw_request_typed_dict"] = (
RawRequestTypedDict(
error=str(e),
)
)
_metadata[
"raw_request"
] = "Unable to Log \
_metadata["raw_request"] = (
"Unable to Log \
raw request: {}".format(
str(e)
str(e)
)
)
if getattr(self, "logger_fn", None) and callable(self.logger_fn):
try:
@ -1115,13 +1115,13 @@ class Logging(LiteLLMLoggingBaseClass):
for callback in callbacks:
try:
if isinstance(callback, CustomLogger):
response: Optional[
MCPPostCallResponseObject
] = await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
response: Optional[MCPPostCallResponseObject] = (
await callback.async_post_mcp_tool_call_hook(
kwargs=kwargs,
response_obj=post_mcp_tool_call_response_obj,
start_time=start_time,
end_time=end_time,
)
)
######################################################################
# if any of the callbacks modify the response, use the modified response
@ -1168,19 +1168,19 @@ class Logging(LiteLLMLoggingBaseClass):
) -> None:
"""
Helper method to store cost breakdown in the logging object.
Args:
input_cost: Cost of input/prompt tokens
output_cost: Cost of output/completion tokens
output_cost: Cost of output/completion tokens
cost_for_built_in_tools_cost_usd_dollar: Cost of built-in tools
total_cost: Total cost of request
"""
self.cost_breakdown = CostBreakdown(
input_cost=input_cost,
output_cost=output_cost,
total_cost=total_cost,
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar
tool_usage_cost=cost_for_built_in_tools_cost_usd_dollar,
)
verbose_logger.debug(
f"Cost breakdown set - input: {input_cost}, output: {output_cost}, cost_for_built_in_tools_cost_usd_dollar: {cost_for_built_in_tools_cost_usd_dollar}, total: {total_cost}"
@ -1259,9 +1259,11 @@ class Logging(LiteLLMLoggingBaseClass):
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,
"litellm_logging_obj": self,
"service_tier": self.optional_params.get("service_tier")
if self.optional_params
else None,
"service_tier": (
self.optional_params.get("service_tier")
if self.optional_params
else None
),
}
except Exception as e: # error creating kwargs for cost calculation
debug_info = StandardLoggingModelCostFailureDebugInformation(
@ -1271,9 +1273,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
try:
@ -1298,9 +1300,9 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
f"response_cost_failure_debug_information: {debug_info}"
)
self.model_call_details[
"response_cost_failure_debug_information"
] = debug_info
self.model_call_details["response_cost_failure_debug_information"] = (
debug_info
)
return None
@ -1444,9 +1446,9 @@ class Logging(LiteLLMLoggingBaseClass):
end_time = datetime.datetime.now()
if self.completion_start_time is None:
self.completion_start_time = end_time
self.model_call_details[
"completion_start_time"
] = self.completion_start_time
self.model_call_details["completion_start_time"] = (
self.completion_start_time
)
self.model_call_details["log_event_type"] = "successful_api_call"
self.model_call_details["end_time"] = end_time
self.model_call_details["cache_hit"] = cache_hit
@ -1499,39 +1501,39 @@ class Logging(LiteLLMLoggingBaseClass):
"response_cost"
]
else:
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=logging_result)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=logging_result)
)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=logging_result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
elif isinstance(result, dict) or isinstance(result, list):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=result,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
elif standard_logging_object is not None:
self.model_call_details[
"standard_logging_object"
] = standard_logging_object
self.model_call_details["standard_logging_object"] = (
standard_logging_object
)
else: # streaming chunks + image gen.
self.model_call_details["response_cost"] = None
@ -1682,23 +1684,23 @@ class Logging(LiteLLMLoggingBaseClass):
verbose_logger.debug(
"Logging Details LiteLLM-Success Call streaming complete"
)
self.model_call_details[
"complete_streaming_response"
] = complete_streaming_response
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(result=complete_streaming_response)
self.model_call_details["complete_streaming_response"] = (
complete_streaming_response
)
self.model_call_details["response_cost"] = (
self._response_cost_calculator(result=complete_streaming_response)
)
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_success_callbacks,
@ -2026,10 +2028,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
openMeterLogger.log_success_event(
@ -2068,10 +2070,10 @@ class Logging(LiteLLMLoggingBaseClass):
)
else:
if self.stream and complete_streaming_response:
self.model_call_details[
"complete_response"
] = self.model_call_details.get(
"complete_streaming_response", {}
self.model_call_details["complete_response"] = (
self.model_call_details.get(
"complete_streaming_response", {}
)
)
result = self.model_call_details["complete_response"]
@ -2209,9 +2211,9 @@ class Logging(LiteLLMLoggingBaseClass):
if complete_streaming_response is not None:
print_verbose("Async success callbacks: Got a complete streaming response")
self.model_call_details[
"async_complete_streaming_response"
] = complete_streaming_response
self.model_call_details["async_complete_streaming_response"] = (
complete_streaming_response
)
try:
if self.model_call_details.get("cache_hit", False) is True:
@ -2222,10 +2224,10 @@ class Logging(LiteLLMLoggingBaseClass):
model_call_details=self.model_call_details
)
# base_model defaults to None if not set on model_info
self.model_call_details[
"response_cost"
] = self._response_cost_calculator(
result=complete_streaming_response
self.model_call_details["response_cost"] = (
self._response_cost_calculator(
result=complete_streaming_response
)
)
verbose_logger.debug(
@ -2238,16 +2240,16 @@ class Logging(LiteLLMLoggingBaseClass):
self.model_call_details["response_cost"] = None
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj=complete_streaming_response,
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="success",
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
callbacks = self.get_combined_callback_list(
dynamic_success_callbacks=self.dynamic_async_success_callbacks,
@ -2460,18 +2462,18 @@ class Logging(LiteLLMLoggingBaseClass):
## STANDARDIZED LOGGING PAYLOAD
self.model_call_details[
"standard_logging_object"
] = get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
self.model_call_details["standard_logging_object"] = (
get_standard_logging_object_payload(
kwargs=self.model_call_details,
init_response_obj={},
start_time=start_time,
end_time=end_time,
logging_obj=self,
status="failure",
error_str=str(exception),
original_exception=exception,
standard_built_in_tools_params=self.standard_built_in_tools_params,
)
)
return start_time, end_time
@ -2979,14 +2981,17 @@ class Logging(LiteLLMLoggingBaseClass):
- For Non-streaming responses, we need to transform the response to a ModelResponse object.
- For streaming responses, anthropic_messages handler calls success_handler with a assembled ModelResponse.
"""
import httpx
if self.stream and isinstance(result, ModelResponse):
return result
elif isinstance(result, ModelResponse):
return result
if "httpx_response" in self.model_call_details:
httpx_response = self.model_call_details.get("httpx_response", None)
if httpx_response and isinstance(httpx_response, httpx.Response):
result = litellm.AnthropicConfig().transform_response(
raw_response=self.model_call_details.get("httpx_response", None),
raw_response=httpx_response,
model_response=litellm.ModelResponse(),
model=self.model,
messages=[],
@ -3355,9 +3360,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
endpoint=arize_config.endpoint,
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"space_id={arize_config.space_key},api_key={arize_config.api_key}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, ArizeLogger)
@ -3381,9 +3386,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
# auth can be disabled on local deployments of arize phoenix
if arize_phoenix_config.otlp_auth_headers is not None:
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = arize_phoenix_config.otlp_auth_headers
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
arize_phoenix_config.otlp_auth_headers
)
for callback in _in_memory_loggers:
if (
@ -3515,9 +3520,9 @@ def _init_custom_logger_compatible_class( # noqa: PLR0915
exporter="otlp_http",
endpoint="https://langtrace.ai/api/trace",
)
os.environ[
"OTEL_EXPORTER_OTLP_TRACES_HEADERS"
] = f"api_key={os.getenv('LANGTRACE_API_KEY')}"
os.environ["OTEL_EXPORTER_OTLP_TRACES_HEADERS"] = (
f"api_key={os.getenv('LANGTRACE_API_KEY')}"
)
for callback in _in_memory_loggers:
if (
isinstance(callback, OpenTelemetry)
@ -4197,10 +4202,10 @@ class StandardLoggingPayloadSetup:
for key in StandardLoggingHiddenParams.__annotations__.keys():
if key in hidden_params:
if key == "additional_headers":
clean_hidden_params[
"additional_headers"
] = StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
clean_hidden_params["additional_headers"] = (
StandardLoggingPayloadSetup.get_additional_headers(
hidden_params[key]
)
)
else:
clean_hidden_params[key] = hidden_params[key] # type: ignore
@ -4252,9 +4257,9 @@ class StandardLoggingPayloadSetup:
if (
custom_logger
and hasattr(custom_logger, "s3_path")
and custom_logger.s3_path
and getattr(custom_logger, "s3_path")
):
s3_path = custom_logger.s3_path
s3_path = getattr(custom_logger, "s3_path")
except Exception:
# If any error occurs in getting the logger instance, use default empty s3_path
pass
@ -4704,9 +4709,9 @@ def scrub_sensitive_keys_in_metadata(litellm_params: Optional[dict]):
):
for k, v in metadata["user_api_key_metadata"].items():
if k == "logging": # prevent logging user logging keys
cleaned_user_api_key_metadata[
k
] = "scrubbed_by_litellm_for_sensitive_keys"
cleaned_user_api_key_metadata[k] = (
"scrubbed_by_litellm_for_sensitive_keys"
)
else:
cleaned_user_api_key_metadata[k] = v

15
litellm/litellm_core_utils/llm_response_utils/convert_dict_to_response.py
View file

@ -2,11 +2,11 @@ import asyncio
import json
import time
import traceback
from litellm._uuid import uuid
from typing import Dict, Iterable, List, Literal, Optional, Tuple, Union
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.constants import RESPONSE_FORMAT_TOOL_NAME
from litellm.litellm_core_utils.prompt_templates.common_utils import (
_extract_reasoning_content,
@ -31,6 +31,7 @@ from litellm.types.utils import Logprobs as TextCompletionLogprobs
from litellm.types.utils import (
Message,
ModelResponse,
ModelResponseStream,
RerankResponse,
StreamingChoices,
TextChoices,
@ -108,12 +109,12 @@ async def convert_to_streaming_response_async(response_object: Optional[dict] =
if response_object is None:
raise Exception("Error in response object format")
model_response_object = ModelResponse(stream=True)
model_response_object = ModelResponseStream()
if model_response_object is None:
raise Exception("Error in response creating model response object")
choice_list = []
choice_list: List[StreamingChoices] = []
for idx, choice in enumerate(response_object["choices"]):
if (
@ -182,8 +183,8 @@ def convert_to_streaming_response(response_object: Optional[dict] = None):
if response_object is None:
raise Exception("Error in response object format")
model_response_object = ModelResponse(stream=True)
choice_list = []
model_response_object = ModelResponseStream()
choice_list: List[StreamingChoices] = []
for idx, choice in enumerate(response_object["choices"]):
delta = Delta(**choice["message"])
finish_reason = choice.get("finish_reason", None)
@ -460,7 +461,7 @@ def convert_to_model_response_object( # noqa: PLR0915
if stream is True:
# for returning cached responses, we need to yield a generator
return convert_to_streaming_response(response_object=response_object)
choice_list = []
choice_list: List[Choices] = []
assert response_object["choices"] is not None and isinstance(
response_object["choices"], Iterable
@ -564,7 +565,7 @@ def convert_to_model_response_object( # noqa: PLR0915
provider_specific_fields=provider_specific_fields,
)
choice_list.append(choice)
model_response_object.choices = choice_list
model_response_object.choices = choice_list # type: ignore
if "usage" in response_object and response_object["usage"] is not None:
usage_object = litellm.Usage(**response_object["usage"])

11
litellm/litellm_core_utils/streaming_handler.py
View file

@ -5,7 +5,6 @@ import json
import threading
import time
import traceback
from litellm._uuid import uuid
from typing import Any, Callable, Dict, List, Optional, Union, cast
import httpx
@ -13,6 +12,7 @@ from pydantic import BaseModel
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.model_response_utils import (
is_model_response_stream_empty,
)
@ -1024,7 +1024,7 @@ class CustomStreamWrapper:
return
def chunk_creator(self, chunk: Any): # type: ignore # noqa: PLR0915
if hasattr(chunk, 'id'):
if hasattr(chunk, "id"):
self.response_id = chunk.id
model_response = self.model_response_creator()
response_obj: Dict[str, Any] = {}
@ -1365,12 +1365,13 @@ class CustomStreamWrapper:
f"model_response finish reason 3: {self.received_finish_reason}; response_obj={response_obj}"
)
## FUNCTION CALL PARSING
original_chunk = (
response_obj.get("original_chunk") if response_obj is not None else None
)
if (
response_obj is not None
and response_obj.get("original_chunk", None) is not None
original_chunk is not None
): # function / tool calling branch - only set for openai/azure compatible endpoints
# enter this branch when no content has been passed in response
original_chunk = response_obj.get("original_chunk", None)
if hasattr(original_chunk, "id"):
model_response = self.set_model_id(
original_chunk.id, model_response

8
litellm/llms/anthropic/completion/transformation.py
View file

@ -55,9 +55,9 @@ class AnthropicTextConfig(BaseConfig):
to pass metadata to anthropic, it's {"user_id": "any-relevant-information"}
"""
max_tokens_to_sample: Optional[
int
] = litellm.max_tokens # anthropic requires a default
max_tokens_to_sample: Optional[int] = (
litellm.max_tokens
) # anthropic requires a default
stop_sequences: Optional[list] = None
temperature: Optional[int] = None
top_p: Optional[int] = None
@ -291,7 +291,7 @@ class AnthropicTextCompletionResponseIterator(BaseModelResponseIterator):
_chunk_text = chunk.get("completion", None)
if _chunk_text is not None and isinstance(_chunk_text, str):
text = _chunk_text
finish_reason = chunk.get("stop_reason", None)
finish_reason = chunk.get("stop_reason") or ""
if finish_reason is not None:
is_finished = True
returned_chunk = GenericStreamingChunk(

2
litellm/llms/anthropic/cost_calculation.py
View file

@ -49,7 +49,7 @@ def get_cost_for_anthropic_web_search(
## Get the cost per web search request
search_context_pricing: SearchContextCostPerQuery = (
model_info.get("search_context_cost_per_query", {}) or {}
model_info.get("search_context_cost_per_query") or SearchContextCostPerQuery()
)
cost_per_web_search_request = search_context_pricing.get(
"search_context_size_medium", 0.0

12
litellm/llms/azure/azure.py
View file

@ -182,12 +182,12 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
model: str,
messages: list,
model_response: ModelResponse,
api_key: str,
api_key: Optional[str],
api_base: str,
api_version: str,
api_type: str,
azure_ad_token: str,
azure_ad_token_provider: Callable,
azure_ad_token: Optional[str],
azure_ad_token_provider: Optional[Callable],
dynamic_params: bool,
print_verbose: Callable,
timeout: Union[float, httpx.Timeout],
@ -372,7 +372,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
async def acompletion(
self,
api_key: str,
api_key: Optional[str],
api_version: str,
model: str,
api_base: str,
@ -477,7 +477,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
dynamic_params: bool,
data: dict,
@ -555,7 +555,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
self,
logging_obj: LiteLLMLoggingObj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
dynamic_params: bool,
data: dict,

4
litellm/llms/azure/common_utils.py
View file

@ -162,8 +162,8 @@ def get_azure_ad_token_from_username_password(
def get_azure_ad_token_from_oidc(
azure_ad_token: str,
azure_client_id: Optional[str],
azure_tenant_id: Optional[str],
azure_client_id: Optional[str] = None,
azure_tenant_id: Optional[str] = None,
scope: Optional[str] = None,
) -> str:
"""

12
litellm/llms/azure/completion/handler.py
View file

@ -30,11 +30,11 @@ class AzureTextCompletion(BaseAzureLLM):
model: str,
messages: list,
model_response: ModelResponse,
api_key: str,
api_key: Optional[str],
api_base: str,
api_version: str,
api_type: str,
azure_ad_token: str,
azure_ad_token: Optional[str],
azure_ad_token_provider: Optional[Callable],
print_verbose: Callable,
timeout,
@ -59,7 +59,7 @@ class AzureTextCompletion(BaseAzureLLM):
### CHECK IF CLOUDFLARE AI GATEWAY ###
### if so - set the model as part of the base url
if "gateway.ai.cloudflare.com" in api_base:
if api_base is not None and "gateway.ai.cloudflare.com" in api_base:
## build base url - assume api base includes resource name
client = self._init_azure_client_for_cloudflare_ai_gateway(
api_key=api_key,
@ -196,7 +196,7 @@ class AzureTextCompletion(BaseAzureLLM):
async def acompletion(
self,
api_key: str,
api_key: Optional[str],
api_version: str,
model: str,
api_base: str,
@ -263,7 +263,7 @@ class AzureTextCompletion(BaseAzureLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
data: dict,
model: str,
@ -320,7 +320,7 @@ class AzureTextCompletion(BaseAzureLLM):
self,
logging_obj,
api_base: str,
api_key: str,
api_key: Optional[str],
api_version: str,
data: dict,
model: str,

30
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -3,14 +3,15 @@ Transformation for Bedrock Invoke Agent
https://docs.aws.amazon.com/bedrock/latest/APIReference/API_agent-runtime_InvokeAgent.html
"""
import base64
import json
from litellm._uuid import uuid
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -22,6 +23,11 @@ from litellm.types.llms.bedrock_invoke_agents import (
InvokeAgentEvent,
InvokeAgentEventHeaders,
InvokeAgentEventList,
InvokeAgentMetadata,
InvokeAgentModelInvocationInput,
InvokeAgentModelInvocationOutput,
InvokeAgentOrchestrationTrace,
InvokeAgentPreProcessingTrace,
InvokeAgentTrace,
InvokeAgentTracePayload,
InvokeAgentUsage,
@ -389,15 +395,22 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
self, trace_data: InvokeAgentTrace, usage_info: InvokeAgentUsage
) -> None:
"""Extract usage information from preprocessing trace."""
pre_processing = trace_data.get("preProcessingTrace", {})
pre_processing: Optional[InvokeAgentPreProcessingTrace] = trace_data.get(
"preProcessingTrace"
)
if not pre_processing:
return
model_output = pre_processing.get("modelInvocationOutput", {})
model_output: Optional[InvokeAgentModelInvocationOutput] = (
pre_processing.get("modelInvocationOutput")
or InvokeAgentModelInvocationOutput()
)
if not model_output:
return
metadata = model_output.get("metadata", {})
metadata: Optional[InvokeAgentMetadata] = (
model_output.get("metadata") or InvokeAgentMetadata()
)
if not metadata:
return
@ -412,11 +425,16 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
self, trace_data: InvokeAgentTrace
) -> Optional[str]:
"""Extract model information from orchestration trace."""
orchestration_trace = trace_data.get("orchestrationTrace", {})
orchestration_trace: Optional[InvokeAgentOrchestrationTrace] = trace_data.get(
"orchestrationTrace"
)
if not orchestration_trace:
return None
model_invocation = orchestration_trace.get("modelInvocationInput", {})
model_invocation: Optional[InvokeAgentModelInvocationInput] = (
orchestration_trace.get("modelInvocationInput")
or InvokeAgentModelInvocationInput()
)
if not model_invocation:
return None

32
litellm/llms/bedrock/chat/invoke_handler.py
View file

@ -7,7 +7,6 @@ import json
import time
import types
import urllib.parse
from litellm._uuid import uuid
from functools import partial
from typing import (
Any,
@ -26,6 +25,7 @@ import httpx # type: ignore
import litellm
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.caching.caching import InMemoryCache
from litellm.litellm_core_utils.core_helpers import map_finish_reason
from litellm.litellm_core_utils.litellm_logging import Logging
@ -498,9 +498,9 @@ class BedrockLLM(BaseAWSLLM):
content=None,
)
model_response.choices[0].message = _message # type: ignore
model_response._hidden_params[
"original_response"
] = outputText # allow user to access raw anthropic tool calling response
model_response._hidden_params["original_response"] = (
outputText # allow user to access raw anthropic tool calling response
)
if (
_is_function_call is True
and stream is not None
@ -808,9 +808,9 @@ class BedrockLLM(BaseAWSLLM):
): # completion(top_k=3) > anthropic_config(top_k=3) <- allows for dynamic variables to be passed in
inference_params[k] = v
if stream is True:
inference_params[
"stream"
] = True # cohere requires stream = True in inference params
inference_params["stream"] = (
True # cohere requires stream = True in inference params
)
data = json.dumps({"prompt": prompt, **inference_params})
elif provider == "anthropic":
if model.startswith("anthropic.claude-3"):
@ -1352,9 +1352,11 @@ class AWSEventStreamDecoder:
"name": None,
"arguments": delta_obj["toolUse"]["input"],
},
"index": self.tool_calls_index
if self.tool_calls_index is not None
else index,
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "reasoningContent" in delta_obj:
provider_specific_fields = {
@ -1384,9 +1386,11 @@ class AWSEventStreamDecoder:
"name": None,
"arguments": "{}",
},
"index": self.tool_calls_index
if self.tool_calls_index is not None
else index,
"index": (
self.tool_calls_index
if self.tool_calls_index is not None
else index
),
}
elif "stopReason" in chunk_data:
finish_reason = map_finish_reason(chunk_data.get("stopReason", "stop"))
@ -1448,7 +1452,7 @@ class AWSEventStreamDecoder:
######### /bedrock/invoke nova mappings ###############
elif "contentBlockDelta" in chunk_data:
# when using /bedrock/invoke/nova, the chunk_data is nested under "contentBlockDelta"
_chunk_data = chunk_data.get("contentBlockDelta", None)
_chunk_data = chunk_data.get("contentBlockDelta", {})
return self.converse_chunk_parser(chunk_data=_chunk_data)
######## bedrock.mistral mappings ###############
elif "outputs" in chunk_data:

15
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -89,6 +89,7 @@ from litellm.utils import (
if TYPE_CHECKING:
from aiohttp import ClientSession
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
@ -281,7 +282,7 @@ class BaseLLMHTTPHandler:
self,
model: str,
messages: list,
api_base: str,
api_base: Optional[str],
custom_llm_provider: str,
model_response: ModelResponse,
encoding,
@ -750,7 +751,7 @@ class BaseLLMHTTPHandler:
model_response: EmbeddingResponse,
api_key: Optional[str] = None,
client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
aembedding: bool = False,
aembedding: Optional[bool] = False,
headers: Optional[Dict[str, Any]] = None,
) -> EmbeddingResponse:
provider_config = ProviderConfigManager.get_provider_embedding_config(
@ -3100,7 +3101,10 @@ class BaseLLMHTTPHandler:
_is_async: bool = False,
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Handles image edit requests.
@ -3290,7 +3294,10 @@ class BaseLLMHTTPHandler:
fake_stream: bool = False,
litellm_metadata: Optional[Dict[str, Any]] = None,
api_key: Optional[str] = None,
) -> Union[ImageResponse, Coroutine[Any, Any, ImageResponse],]:
) -> Union[
ImageResponse,
Coroutine[Any, Any, ImageResponse],
]:
"""
Handles image generation requests.
When _is_async=True, returns a coroutine instead of making the call directly.

62
litellm/llms/gemini/google_genai/transformation.py
View file

@ -1,6 +1,7 @@
"""
Transformation for Calling Google models in their native format.
"""
from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast
import httpx
@ -25,27 +26,29 @@ else:
GenerateContentContentListUnionDict = Any
GenerateContentResponse = Any
ToolConfigDict = Any
from ..common_utils import get_api_key_from_env
class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
Configuration for calling Google models in their native format.
"""
##############################
# Constants
##############################
XGOOGLE_API_KEY = "x-goog-api-key"
##############################
@property
def custom_llm_provider(self) -> Literal["gemini", "vertex_ai"]:
return "gemini"
def __init__(self):
super().__init__()
VertexLLM.__init__(self)
def get_supported_generate_content_optional_params(self, model: str) -> List[str]:
"""
Get the list of supported Google GenAI parameters for the model.
@ -58,7 +61,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
return [
"http_options",
"system_instruction",
"system_instruction",
"temperature",
"top_p",
"top_k",
@ -84,10 +87,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"speech_config",
"audio_timestamp",
"automatic_function_calling",
"thinking_config"
"thinking_config",
]
def map_generate_content_optional_params(
self,
generate_content_config_dict: GenerateContentConfigDict,
@ -103,26 +105,29 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Returns:
Mapped parameters for the provider
"""
from litellm.types.google_genai.main import GenerateContentConfigDict
_generate_content_config_dict = GenerateContentConfigDict()
supported_google_genai_params = self.get_supported_generate_content_optional_params(model)
_generate_content_config_dict: Dict[str, Any] = {}
supported_google_genai_params = (
self.get_supported_generate_content_optional_params(model)
)
for param, value in generate_content_config_dict.items():
if param in supported_google_genai_params:
_generate_content_config_dict[param] = value
return dict(_generate_content_config_dict)
return _generate_content_config_dict
def validate_environment(
self,
self,
api_key: Optional[str],
headers: Optional[dict],
model: str,
litellm_params: Optional[Union[GenericLiteLLMParams, dict]]
litellm_params: Optional[Union[GenericLiteLLMParams, dict]],
) -> dict:
default_headers = {
"Content-Type": "application/json",
}
# Use the passed api_key first, then fall back to litellm_params and environment
gemini_api_key = api_key or self._get_google_ai_studio_api_key(dict(litellm_params or {}))
gemini_api_key = api_key or self._get_google_ai_studio_api_key(
dict(litellm_params or {})
)
if gemini_api_key is not None:
default_headers[self.XGOOGLE_API_KEY] = gemini_api_key
if headers is not None:
@ -137,14 +142,14 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
or get_api_key_from_env()
or litellm.api_key
)
def _get_common_auth_components(
self,
litellm_params: dict,
) -> Tuple[Any, Optional[str], Optional[str]]:
"""
Get common authentication components used by both sync and async methods.
Returns:
Tuple of (vertex_credentials, vertex_project, vertex_location)
"""
@ -152,7 +157,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
vertex_project = self.get_vertex_ai_project(litellm_params)
vertex_location = self.get_vertex_ai_location(litellm_params)
return vertex_credentials, vertex_project, vertex_location
def _build_final_headers_and_url(
self,
model: str,
@ -168,7 +173,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Build final headers and API URL from auth components.
"""
gemini_api_key = self._get_google_ai_studio_api_key(litellm_params)
auth_header, api_base = self._get_token_and_url(
model=model,
gemini_api_key=gemini_api_key,
@ -201,7 +206,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
"""
Sync version of get_auth_token_and_url.
"""
vertex_credentials, vertex_project, vertex_location = self._get_common_auth_components(litellm_params)
vertex_credentials, vertex_project, vertex_location = (
self._get_common_auth_components(litellm_params)
)
_auth_header, vertex_project = self._ensure_access_token(
credentials=vertex_credentials,
@ -238,7 +245,9 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Returns:
Tuple of headers and API base
"""
vertex_credentials, vertex_project, vertex_location = self._get_common_auth_components(litellm_params)
vertex_credentials, vertex_project, vertex_location = (
self._get_common_auth_components(litellm_params)
)
_auth_header, vertex_project = await self._ensure_access_token_async(
credentials=vertex_credentials,
@ -256,7 +265,6 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
api_base=api_base,
litellm_params=litellm_params,
)
def transform_generate_content_request(
self,
@ -269,6 +277,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
GenerateContentConfigDict,
GenerateContentRequestDict,
)
typed_generate_content_request = GenerateContentRequestDict(
model=model,
contents=contents,
@ -279,7 +288,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
request_dict = cast(dict, typed_generate_content_request)
return request_dict
def transform_generate_content_response(
self,
model: str,
@ -297,6 +306,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
Transformed response data
"""
from litellm.types.google_genai.main import GenerateContentResponse
try:
response = raw_response.json()
except Exception as e:
@ -305,7 +315,7 @@ class GoogleGenAIConfig(BaseGoogleGenAIGenerateContentConfig, VertexLLM):
status_code=raw_response.status_code,
headers=raw_response.headers,
)
logging_obj.model_call_details["httpx_response"] = raw_response
return GenerateContentResponse(**response)
return GenerateContentResponse(**response)

26
litellm/llms/huggingface/embedding/transformation.py
View file

@ -40,17 +40,17 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
Reference: https://huggingface.github.io/text-generation-inference/#/Text%20Generation%20Inference/compat_generate
"""
hf_task: Optional[
hf_tasks
] = None # litellm-specific param, used to know the api spec to use when calling huggingface api
hf_task: Optional[hf_tasks] = (
None # litellm-specific param, used to know the api spec to use when calling huggingface api
)
best_of: Optional[int] = None
decoder_input_details: Optional[bool] = None
details: Optional[bool] = True # enables returning logprobs + best of
max_new_tokens: Optional[int] = None
repetition_penalty: Optional[float] = None
return_full_text: Optional[
bool
] = False # by default don't return the input as part of the output
return_full_text: Optional[bool] = (
False # by default don't return the input as part of the output
)
seed: Optional[int] = None
temperature: Optional[float] = None
top_k: Optional[int] = None
@ -120,9 +120,9 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
optional_params["top_p"] = value
if param == "n":
optional_params["best_of"] = value
optional_params[
"do_sample"
] = True # Need to sample if you want best of for hf inference endpoints
optional_params["do_sample"] = (
True # Need to sample if you want best of for hf inference endpoints
)
if param == "stream":
optional_params["stream"] = value
if param == "stop":
@ -268,7 +268,7 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
# check if the model has a registered custom prompt
model_prompt_details = litellm.custom_prompt_dict[model]
prompt = custom_prompt(
role_dict=model_prompt_details.get("roles", None),
role_dict=model_prompt_details.get("roles") or {},
initial_prompt_value=model_prompt_details.get(
"initial_prompt_value", ""
),
@ -363,9 +363,9 @@ class HuggingFaceEmbeddingConfig(BaseConfig):
"content-type": "application/json",
}
if api_key is not None:
default_headers[
"Authorization"
] = f"Bearer {api_key}" # Huggingface Inference Endpoint default is to accept bearer tokens
default_headers["Authorization"] = (
f"Bearer {api_key}" # Huggingface Inference Endpoint default is to accept bearer tokens
)
headers = {**headers, **default_headers}
return headers

12
litellm/llms/openai/completion/transformation.py
View file

@ -1,5 +1,5 @@
"""
Support for gpt model family
Support for gpt model family
"""
from typing import List, Optional, Union
@ -87,7 +87,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
## RESPONSE OBJECT
if response_object is None or model_response_object is None:
raise ValueError("Error in response object format")
choice_list = []
choice_list: List[Choices] = []
for idx, choice in enumerate(response_object["choices"]):
message = Message(
content=choice["text"],
@ -100,7 +100,7 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
logprobs=choice.get("logprobs", None),
)
choice_list.append(choice)
model_response_object.choices = choice_list
model_response_object.choices = choice_list # type: ignore
if "usage" in response_object:
setattr(model_response_object, "usage", response_object["usage"])
@ -111,9 +111,9 @@ class OpenAITextCompletionConfig(BaseTextCompletionConfig, OpenAIGPTConfig):
if "model" in response_object:
model_response_object.model = response_object["model"]
model_response_object._hidden_params[
"original_response"
] = response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
model_response_object._hidden_params["original_response"] = (
response_object # track original response, if users make a litellm.text_completion() request, we can return the original response
)
return model_response_object
except Exception as e:
raise e

14
litellm/llms/openai/image_edit/transformation.py
View file

@ -91,21 +91,22 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
# Handle image parameter
if _image_list is not None:
image_list = [_image_list] if not isinstance(_image_list, list) else _image_list
image_list = (
[_image_list] if not isinstance(_image_list, list) else _image_list
)
for _image in image_list:
if _image is not None:
image_content_type: str = ImageEditRequestUtils.get_image_content_type(
_image
image_content_type: str = (
ImageEditRequestUtils.get_image_content_type(_image)
)
if isinstance(_image, BufferedReader):
files_list.append(
("image", (_image.name, _image, image_content_type))
("image[]", (_image.name, _image, image_content_type))
)
else:
files_list.append(
("image", ("image.png", _image, image_content_type))
("image[]", ("image.png", _image, image_content_type))
)
# Handle mask parameter if provided
if _mask is not None:
# Handle case where mask can be a list (extract first mask)
@ -120,6 +121,7 @@ class OpenAIImageEditConfig(BaseImageEditConfig):
files_list.append(("mask", (_mask.name, _mask, mask_content_type)))
else:
files_list.append(("mask", ("mask.png", _mask, mask_content_type)))
return data_without_files, files_list
def transform_image_edit_response(

16
litellm/llms/vertex_ai/fine_tuning/handler.py
View file

@ -64,9 +64,9 @@ class VertexFineTuningAPI(VertexLLM):
)
if create_fine_tuning_job_data.validation_file:
supervised_tuning_spec[
"validation_dataset"
] = create_fine_tuning_job_data.validation_file
supervised_tuning_spec["validation_dataset"] = (
create_fine_tuning_job_data.validation_file
)
_vertex_hyperparameters = (
self._transform_openai_hyperparameters_to_vertex_hyperparameters(
@ -140,7 +140,9 @@ class VertexFineTuningAPI(VertexLLM):
fine_tuned_model=response.get("tunedModelDisplayName", ""),
finished_at=None,
hyperparameters=self._translate_vertex_response_hyperparameters(
vertex_hyper_parameters=_supervisedTuningSpec.get("hyperParameters", {})
vertex_hyper_parameters=_supervisedTuningSpec.get(
"hyperParameters", FineTuneHyperparameters()
)
or {}
),
model=response.get("baseModel", "") or "",
@ -343,9 +345,9 @@ class VertexFineTuningAPI(VertexLLM):
elif "cachedContents" in request_route:
_model = request_data.get("model")
if _model is not None and "/publishers/google/models/" not in _model:
request_data[
"model"
] = f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}"
request_data["model"] = (
f"projects/{vertex_project}/locations/{vertex_location}/publishers/google/models/{_model}"
)
url = f"https://{vertex_location}-aiplatform.googleapis.com/v1beta1/projects/{vertex_project}/locations/{vertex_location}{request_route}"
else:

2
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py
View file

@ -43,7 +43,7 @@ class GoogleBatchEmbeddings(VertexLLM):
vertex_project=None,
vertex_location=None,
vertex_credentials=None,
aembedding=False,
aembedding: Optional[bool] = False,
timeout=300,
client=None,
) -> EmbeddingResponse:

14
litellm/llms/vertex_ai/google_genai/transformation.py
View file

@ -1,7 +1,8 @@
"""
Transformation for Calling Google models in their native format.
"""
from typing import Dict, Literal, Optional, Union
from typing import Any, Dict, Literal, Optional, Union
from litellm.llms.gemini.google_genai.transformation import GoogleGenAIConfig
from litellm.types.router import GenericLiteLLMParams
@ -58,22 +59,21 @@ class VertexAIGoogleGenAIConfig(GoogleGenAIConfig):
Returns:
Mapped parameters for the provider
"""
from litellm.types.google_genai.main import GenerateContentConfigDict
_generate_content_config_dict = GenerateContentConfigDict()
_generate_content_config_dict: Dict = {}
for param, value in generate_content_config_dict.items():
camel_case_key = self._camel_to_snake(param)
_generate_content_config_dict[camel_case_key] = value
return dict(_generate_content_config_dict)
return _generate_content_config_dict
def transform_generate_content_request(
self,
model: str,
contents: any,
tools: Optional[any],
contents: Any,
tools: Optional[Any],
generate_content_config_dict: Dict,
system_instruction: Optional[any] = None,
system_instruction: Optional[Any] = None,
) -> dict:
"""
Transform the generate content request for Vertex AI.

2
litellm/llms/vertex_ai/multimodal_embeddings/embedding_handler.py
View file

@ -46,7 +46,7 @@ class VertexMultimodalEmbedding(VertexLLM):
vertex_project=None,
vertex_location=None,
vertex_credentials=None,
aembedding=False,
aembedding: Optional[bool] = False,
timeout=300,
client=None,
) -> EmbeddingResponse:

14
litellm/llms/vertex_ai/vertex_embeddings/embedding_handler.py
View file

@ -36,7 +36,7 @@ class VertexEmbedding(VertexBase):
timeout: Optional[Union[float, httpx.Timeout]],
api_key: Optional[str] = None,
encoding=None,
aembedding=False,
aembedding: Optional[bool] = False,
api_base: Optional[str] = None,
client: Optional[Union[AsyncHTTPHandler, HTTPHandler]] = None,
vertex_project: Optional[str] = None,
@ -86,8 +86,10 @@ class VertexEmbedding(VertexBase):
mode="embedding",
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
vertex_request: VertexEmbeddingRequest = (
litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
)
)
_client_params = {}
@ -176,8 +178,10 @@ class VertexEmbedding(VertexBase):
mode="embedding",
)
headers = self.set_headers(auth_header=auth_header, extra_headers=extra_headers)
vertex_request: VertexEmbeddingRequest = litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
vertex_request: VertexEmbeddingRequest = (
litellm.vertexAITextEmbeddingConfig.transform_openai_request_to_vertex_embedding_request(
input=input, optional_params=optional_params, model=model
)
)
_async_client_params = {}

4
litellm/llms/watsonx/chat/handler.py
View file

@ -21,7 +21,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler):
*,
model: str,
messages: list,
api_base: str,
api_base: Optional[str],
custom_llm_provider: str,
custom_prompt_dict: dict,
model_response: ModelResponse,
@ -70,7 +70,7 @@ class WatsonXChatHandler(OpenAILikeChatHandler):
)
return super().completion(
model=watsonx_auth_payload.get("model_id", None),
model=watsonx_auth_payload.get("model_id") or "",
messages=messages,
api_base=api_base,
custom_llm_provider=custom_llm_provider,

60
litellm/main.py
View file

@ -17,12 +17,12 @@ import random
import sys
import time
import traceback
from litellm._uuid import uuid
from concurrent import futures
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
from copy import deepcopy
from functools import partial
from typing import (
TYPE_CHECKING,
Any,
Callable,
Coroutine,
@ -36,9 +36,10 @@ from typing import (
Union,
cast,
get_args,
TYPE_CHECKING,
)
from litellm._uuid import uuid
if TYPE_CHECKING:
from aiohttp import ClientSession
@ -721,12 +722,15 @@ async def _sleep_for_timeout_async(timeout: Union[float, str, httpx.Timeout]):
await asyncio.sleep(timeout.connect)
MOCK_RESPONSE_TYPE = Union[str, Exception, dict]
def mock_completion(
model: str,
messages: List,
stream: Optional[bool] = False,
n: Optional[int] = None,
mock_response: Union[str, Exception, dict] = "This is a mock request",
mock_response: Optional[MOCK_RESPONSE_TYPE] = "This is a mock request",
mock_tool_calls: Optional[List] = None,
mock_timeout: Optional[bool] = False,
logging=None,
@ -1007,7 +1011,7 @@ def completion( # type: ignore # noqa: PLR0915
######### unpacking kwargs #####################
args = locals()
api_base = kwargs.get("api_base", None)
mock_response = kwargs.get("mock_response", None)
mock_response: Optional[MOCK_RESPONSE_TYPE] = kwargs.get("mock_response", None)
mock_tool_calls = kwargs.get("mock_tool_calls", None)
mock_timeout = cast(Optional[bool], kwargs.get("mock_timeout", None))
force_timeout = kwargs.get("force_timeout", 600) ## deprecated
@ -1114,7 +1118,7 @@ def completion( # type: ignore # noqa: PLR0915
api_base = base_url
if num_retries is not None:
max_retries = num_retries
logging = litellm_logging_obj
logging: Logging = cast(Logging, litellm_logging_obj)
fallbacks = fallbacks or litellm.model_fallbacks
if fallbacks is not None:
return completion_with_fallbacks(**args)
@ -1427,7 +1431,7 @@ def completion( # type: ignore # noqa: PLR0915
api_version = (
api_version
or litellm.api_version
or get_secret("AZURE_API_VERSION")
or get_secret_str("AZURE_API_VERSION")
or litellm.AZURE_DEFAULT_API_VERSION
)
@ -1435,13 +1439,13 @@ def completion( # type: ignore # noqa: PLR0915
api_key
or litellm.api_key
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.get("extra_body", {}).pop(
"azure_ad_token", None
) or get_secret("AZURE_AD_TOKEN")
) or get_secret_str("AZURE_AD_TOKEN")
azure_ad_token_provider = litellm_params.get(
"azure_ad_token_provider", None
@ -1529,25 +1533,32 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "azure_text":
# azure configs
api_type = get_secret("AZURE_API_TYPE") or "azure"
api_type = get_secret_str("AZURE_API_TYPE") or "azure"
api_base = api_base or litellm.api_base or get_secret("AZURE_API_BASE")
api_base = api_base or litellm.api_base or get_secret_str("AZURE_API_BASE")
if api_base is None:
raise ValueError(
"api_base is required for Azure OpenAI LLM provider. Either set it dynamically or set the AZURE_API_BASE environment variable."
)
api_version = (
api_version or litellm.api_version or get_secret("AZURE_API_VERSION")
api_version
or litellm.api_version
or get_secret_str("AZURE_API_VERSION")
)
api_key = (
api_key
or litellm.api_key
or litellm.azure_key
or get_secret("AZURE_OPENAI_API_KEY")
or get_secret("AZURE_API_KEY")
or get_secret_str("AZURE_OPENAI_API_KEY")
or get_secret_str("AZURE_API_KEY")
)
azure_ad_token = optional_params.get("extra_body", {}).pop(
"azure_ad_token", None
) or get_secret("AZURE_AD_TOKEN")
) or get_secret_str("AZURE_AD_TOKEN")
azure_ad_token_provider = litellm_params.get(
"azure_ad_token_provider", None
@ -1573,7 +1584,7 @@ def completion( # type: ignore # noqa: PLR0915
headers=headers,
api_key=api_key,
api_base=api_base,
api_version=api_version,
api_version=cast(str, api_version),
api_type=api_type,
azure_ad_token=azure_ad_token,
azure_ad_token_provider=azure_ad_token_provider,
@ -2545,15 +2556,10 @@ def completion( # type: ignore # noqa: PLR0915
)
elif custom_llm_provider == "compactifai":
api_key = (
api_key
or get_secret_str("COMPACTIFAI_API_KEY")
or litellm.api_key
api_key or get_secret_str("COMPACTIFAI_API_KEY") or litellm.api_key
)
api_base = (
api_base
or "https://api.compactif.ai/v1"
)
api_base = api_base or "https://api.compactif.ai/v1"
## COMPLETION CALL
response = base_llm_http_handler.completion(
@ -2860,7 +2866,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=custom_llm_provider, # type: ignore
client=client,
api_base=api_base,
extra_headers=extra_headers,
@ -2929,7 +2935,7 @@ def completion( # type: ignore # noqa: PLR0915
logging_obj=logging,
acompletion=acompletion,
timeout=timeout,
custom_llm_provider=custom_llm_provider,
custom_llm_provider=custom_llm_provider, # type: ignore
client=client,
api_base=api_base,
extra_headers=extra_headers,
@ -3935,7 +3941,7 @@ def embedding( # noqa: PLR0915
litellm_logging_obj: LiteLLMLoggingObj = kwargs.get("litellm_logging_obj") # type: ignore
mock_response: Optional[List[float]] = kwargs.get("mock_response", None) # type: ignore
azure_ad_token_provider = kwargs.get("azure_ad_token_provider", None)
aembedding = kwargs.get("aembedding", None)
aembedding: Optional[bool] = kwargs.get("aembedding", None)
extra_headers = kwargs.get("extra_headers", None)
headers = kwargs.get("headers", None)
### CUSTOM MODEL COST ###
@ -5615,7 +5621,7 @@ def speech( # noqa: PLR0915
if max_retries is None:
max_retries = litellm.num_retries or openai.DEFAULT_MAX_RETRIES
litellm_params_dict = get_litellm_params(**kwargs)
logging_obj = kwargs.get("litellm_logging_obj", None)
logging_obj: Logging = cast(Logging, kwargs.get("litellm_logging_obj"))
logging_obj.update_environment_variables(
model=model,
user=user,

36
litellm/proxy/_experimental/mcp_server/cost_calculator.py
View file

@ -1,6 +1,7 @@
"""
Cost calculator for MCP tools.
"""
from typing import TYPE_CHECKING, Any, Optional, cast
from litellm.types.mcp import MCPServerCostInfo
@ -13,11 +14,12 @@ if TYPE_CHECKING:
else:
LitellmLoggingObject = Any
class MCPCostCalculator:
@staticmethod
def calculate_mcp_tool_call_cost(
litellm_logging_obj: Optional[LitellmLoggingObject],
) -> float:
) -> float:
"""
Calculate the cost of an MCP tool call.
@ -25,29 +27,43 @@ class MCPCostCalculator:
"""
if litellm_logging_obj is None:
return 0.0
#########################################################
# Get the response cost from logging object model_call_details
# This is set when a user modifies the response in a post_mcp_tool_call_hook
#########################################################
response_cost = litellm_logging_obj.model_call_details.get("response_cost", None)
response_cost = litellm_logging_obj.model_call_details.get(
"response_cost", None
)
if response_cost is not None:
return response_cost
#########################################################
# Unpack the mcp_tool_call_metadata
#########################################################
mcp_tool_call_metadata: StandardLoggingMCPToolCall = cast(StandardLoggingMCPToolCall, litellm_logging_obj.model_call_details.get("mcp_tool_call_metadata", {})) or {}
mcp_server_cost_info_raw = mcp_tool_call_metadata.get("mcp_server_cost_info", {}) or {}
mcp_server_cost_info: MCPServerCostInfo = cast(MCPServerCostInfo, mcp_server_cost_info_raw)
mcp_tool_call_metadata: StandardLoggingMCPToolCall = (
cast(
StandardLoggingMCPToolCall,
litellm_logging_obj.model_call_details.get(
"mcp_tool_call_metadata", {}
),
)
or {}
)
mcp_server_cost_info: MCPServerCostInfo = (
mcp_tool_call_metadata.get("mcp_server_cost_info") or MCPServerCostInfo()
)
#########################################################
# User defined cost per query
#########################################################
default_cost_per_query = mcp_server_cost_info.get("default_cost_per_query", None)
tool_name_to_cost_per_query: dict = mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
default_cost_per_query = mcp_server_cost_info.get(
"default_cost_per_query", None
)
tool_name_to_cost_per_query: dict = (
mcp_server_cost_info.get("tool_name_to_cost_per_query", {}) or {}
)
tool_name = mcp_tool_call_metadata.get("name", "")
#########################################################
# 1. If tool_name is in tool_name_to_cost_per_query, use the cost per query
# 2. If tool_name is not in tool_name_to_cost_per_query, use the default cost per query

4
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -1,5 +1,5 @@
import json
from typing import Optional
from typing import Optional, Tuple
from urllib.parse import urlencode, urlparse, urlunparse
from fastapi import APIRouter, Form, HTTPException, Request
@ -36,7 +36,7 @@ def encode_state_with_base_url(base_url: str, original_state: str) -> str:
return encrypted_state
def decode_state_hash(encrypted_state: str) -> tuple[str, str]:
def decode_state_hash(encrypted_state: str) -> Tuple[str, str]:
"""
Decode an encrypted state to retrieve the base_url and original state.

0
litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_buildManifest.js → litellm/proxy/_experimental/out/_next/static/Ap4Kq4vtq74RgOyxD-zii/_buildManifest.js
View file

0
litellm/proxy/_experimental/out/_next/static/0oPk2eYtSaTLaPyVixqA8/_ssgManifest.js → litellm/proxy/_experimental/out/_next/static/Ap4Kq4vtq74RgOyxD-zii/_ssgManifest.js
View file

2
litellm/proxy/_experimental/out/_next/static/chunks/162-4e7640b4d68e1ae4.js → litellm/proxy/_experimental/out/_next/static/chunks/162-dd6427ff1a4ad9f4.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/172-0f7049c565983c4d.js
View file

File diff suppressed because one or more lines are too long

8
litellm/proxy/_experimental/out/_next/static/chunks/220-89d73a525e307735.js → litellm/proxy/_experimental/out/_next/static/chunks/313-0025fb08e386c4b8.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/851-bbe6d02cf41bb87a.js Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js → litellm/proxy/_experimental/out/_next/static/chunks/app/model_hub_table/page-0b693f691bf0309f.js
View file

@ -1 +1 @@
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(36172);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,172,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);
(self.webpackChunk_N_E=self.webpackChunk_N_E||[]).push([[25],{38520:function(e,n,u){Promise.resolve().then(u.bind(u,22775))},22775:function(e,n,u){"use strict";u.r(n),u.d(n,{default:function(){return f}});var t=u(57437),s=u(2265),r=u(99376),c=u(97851);function f(){let e=(0,r.useSearchParams)().get("key"),[n,u]=(0,s.useState)(null);return(0,s.useEffect)(()=>{e&&u(e)},[e]),(0,t.jsx)(c.Z,{accessToken:n,publicPage:!0,premiumUser:!1,userRole:null})}}},function(e){e.O(0,[50,521,866,154,162,851,971,117,744],function(){return e(e.s=38520)}),_N_E=e.O()}]);

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-46f79791404274c7.js Normal file
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/_next/static/chunks/app/page-73b19c9fbf8cc64f.js
View file

File diff suppressed because one or more lines are too long

1
litellm/proxy/_experimental/out/assets/logos/oracle.svg Normal file
View file

@ -0,0 +1 @@
<svg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 231 30' preserveAspectRatio='xMinYMid'><path d='M99.61,19.52h15.24l-8.05-13L92,30H85.27l18-28.17a4.29,4.29,0,0,1,7-.05L128.32,30h-6.73l-3.17-5.25H103l-3.36-5.23m69.93,5.23V0.28h-5.72V27.16a2.76,2.76,0,0,0,.85,2,2.89,2.89,0,0,0,2.08.87h26l3.39-5.25H169.54M75,20.38A10,10,0,0,0,75,.28H50V30h5.71V5.54H74.65a4.81,4.81,0,0,1,0,9.62H58.54L75.6,30h8.29L72.43,20.38H75M14.88,30H32.15a14.86,14.86,0,0,0,0-29.71H14.88a14.86,14.86,0,1,0,0,29.71m16.88-5.23H15.26a9.62,9.62,0,0,1,0-19.23h16.5a9.62,9.62,0,1,1,0,19.23M140.25,30h17.63l3.34-5.23H140.64a9.62,9.62,0,1,1,0-19.23h16.75l3.38-5.25H140.25a14.86,14.86,0,1,0,0,29.71m69.87-5.23a9.62,9.62,0,0,1-9.26-7h24.42l3.36-5.24H200.86a9.61,9.61,0,0,1,9.26-7h16.76l3.35-5.25h-20.5a14.86,14.86,0,0,0,0,29.71h17.63l3.35-5.23h-20.6' transform='translate(-0.02 0)' style='fill:#C74634'/></svg>
Side by side

After

Width:  |  Height:  |  Size: 874 B

2
litellm/proxy/_experimental/out/index.html
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/index.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[85617,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","220","static/chunks/220-89d73a525e307735.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","931","static/chunks/app/page-73b19c9fbf8cc64f.js"],"default",1]
3:I[55139,["665","static/chunks/3014691f-b7b79b78e27792f3.js","990","static/chunks/13b76428-ebdf3012af0e4489.js","50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","313","static/chunks/313-0025fb08e386c4b8.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","851","static/chunks/851-bbe6d02cf41bb87a.js","931","static/chunks/app/page-46f79791404274c7.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["__PAGE__",{}]},"$undefined","$undefined",true],["",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

4
litellm/proxy/_experimental/out/model_hub.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","418","static/chunks/app/model_hub/page-13b00ef4a072d920.js"],"default",1]
3:I[52829,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","418","static/chunks/app/model_hub/page-13b00ef4a072d920.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["model_hub",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

2
litellm/proxy/_experimental/out/model_hub_table/index.html → litellm/proxy/_experimental/out/model_hub_table.html
View file

File diff suppressed because one or more lines are too long

4
litellm/proxy/_experimental/out/model_hub_table.txt
View file

@ -1,7 +1,7 @@
2:I[19107,[],"ClientPageRoot"]
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-4e7640b4d68e1ae4.js","172","static/chunks/172-0f7049c565983c4d.js","25","static/chunks/app/model_hub_table/page-304b7041a3fa39f7.js"],"default",1]
3:I[22775,["50","static/chunks/50-d0da2dd7acce2eb9.js","521","static/chunks/521-d97d355792d44830.js","866","static/chunks/866-9e1803a09e9ae8da.js","154","static/chunks/154-b1f2a106d0e0d77b.js","162","static/chunks/162-dd6427ff1a4ad9f4.js","851","static/chunks/851-bbe6d02cf41bb87a.js","25","static/chunks/app/model_hub_table/page-0b693f691bf0309f.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["model_hub_table",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["model_hub_table",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","model_hub_table","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

1
litellm/proxy/_experimental/out/onboarding.html Normal file
View file

File diff suppressed because one or more lines are too long

2
litellm/proxy/_experimental/out/onboarding.txt
View file

@ -2,6 +2,6 @@
3:I[12011,["665","static/chunks/3014691f-b7b79b78e27792f3.js","50","static/chunks/50-d0da2dd7acce2eb9.js","154","static/chunks/154-b1f2a106d0e0d77b.js","461","static/chunks/app/onboarding/page-d0d85032bb87ba51.js"],"default",1]
4:I[4707,[],""]
5:I[36423,[],""]
0:["0oPk2eYtSaTLaPyVixqA8",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
0:["Ap4Kq4vtq74RgOyxD-zii",[[["",{"children":["onboarding",{"children":["__PAGE__",{}]}]},"$undefined","$undefined",true],["",{"children":["onboarding",{"children":["__PAGE__",{},[["$L1",["$","$L2",null,{"props":{"params":{},"searchParams":{}},"Component":"$3"}],null],null],null]},[null,["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children","onboarding","children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":"$undefined","notFoundStyles":"$undefined"}]],null]},[[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/349654da14372cd9.css","precedence":"next","crossOrigin":"$undefined"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/css/4103fa525703177b.css","precedence":"next","crossOrigin":"$undefined"}]],["$","html",null,{"lang":"en","children":["$","body",null,{"className":"__className_1c856b","children":["$","$L4",null,{"parallelRouterKey":"children","segmentPath":["children"],"error":"$undefined","errorStyles":"$undefined","errorScripts":"$undefined","template":["$","$L5",null,{}],"templateStyles":"$undefined","templateScripts":"$undefined","notFound":[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":"404"}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],"notFoundStyles":[]}]}]}]],null],null],["$L6",null]]]]
6:[["$","meta","0",{"name":"viewport","content":"width=device-width, initial-scale=1"}],["$","meta","1",{"charSet":"utf-8"}],["$","title","2",{"children":"LiteLLM Dashboard"}],["$","meta","3",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","4",{"rel":"icon","href":"/favicon.ico","type":"image/x-icon","sizes":"16x16"}],["$","link","5",{"rel":"icon","href":"./favicon.ico"}],["$","meta","6",{"name":"next-size-adjust"}]]
1:null

130
litellm/proxy/_types.py
View file

@ -1,16 +1,7 @@
import enum
import json
from datetime import datetime
from typing import (
TYPE_CHECKING,
Any,
Callable,
Dict,
List,
Literal,
Optional,
Union,
)
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literal, Optional, Union
import httpx
from pydantic import (
@ -26,11 +17,7 @@ from typing_extensions import Required, TypedDict
from litellm._uuid import uuid
from litellm.types.integrations.slack_alerting import AlertType
from litellm.types.llms.openai import AllMessageValues, OpenAIFileObject
from litellm.types.mcp import (
MCPAuthType,
MCPTransport,
MCPTransportType,
)
from litellm.types.mcp import MCPAuthType, MCPTransport, MCPTransportType
from litellm.types.mcp_server.mcp_server_manager import MCPInfo
from litellm.types.router import RouterErrors, UpdateRouterConfig
from litellm.types.secret_managers.main import KeyManagementSystem
@ -404,16 +391,16 @@ class LiteLLMRoutes(enum.Enum):
]
key_management_routes = [
KeyManagementRoutes.KEY_GENERATE,
KeyManagementRoutes.KEY_UPDATE,
KeyManagementRoutes.KEY_DELETE,
KeyManagementRoutes.KEY_INFO,
KeyManagementRoutes.KEY_REGENERATE,
KeyManagementRoutes.KEY_GENERATE_SERVICE_ACCOUNT,
KeyManagementRoutes.KEY_REGENERATE_WITH_PATH_PARAM,
KeyManagementRoutes.KEY_LIST,
KeyManagementRoutes.KEY_BLOCK,
KeyManagementRoutes.KEY_UNBLOCK,
KeyManagementRoutes.KEY_GENERATE.value,
KeyManagementRoutes.KEY_UPDATE.value,
KeyManagementRoutes.KEY_DELETE.value,
KeyManagementRoutes.KEY_INFO.value,
KeyManagementRoutes.KEY_REGENERATE.value,
KeyManagementRoutes.KEY_GENERATE_SERVICE_ACCOUNT.value,
KeyManagementRoutes.KEY_REGENERATE_WITH_PATH_PARAM.value,
KeyManagementRoutes.KEY_LIST.value,
KeyManagementRoutes.KEY_BLOCK.value,
KeyManagementRoutes.KEY_UNBLOCK.value,
]
management_routes = [
@ -747,9 +734,9 @@ class GenerateRequestBase(LiteLLMPydanticObjectBase):
allowed_cache_controls: Optional[list] = []
config: Optional[dict] = {}
permissions: Optional[dict] = {}
model_max_budget: Optional[
dict
] = {} # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_max_budget: Optional[dict] = (
{}
) # {"gpt-4": 5.0, "gpt-3.5-turbo": 5.0}, defaults to {}
model_config = ConfigDict(protected_namespaces=())
model_rpm_limit: Optional[dict] = None
@ -788,12 +775,11 @@ class GenerateKeyRequest(KeyRequestBase):
description="Type of key that determines default allowed routes.",
)
auto_rotate: Optional[bool] = Field(
default=False,
description="Whether this key should be automatically rotated"
default=False, description="Whether this key should be automatically rotated"
)
rotation_interval: Optional[str] = Field(
default=None,
description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True"
description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True",
)
@ -1157,12 +1143,12 @@ class NewCustomerRequest(BudgetNewRequest):
blocked: bool = False # allow/disallow requests for this end-user
budget_id: Optional[str] = None # give either a budget_id or max_budget
spend: Optional[float] = None
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
@model_validator(mode="before")
@classmethod
@ -1184,12 +1170,12 @@ class UpdateCustomerRequest(LiteLLMPydanticObjectBase):
blocked: bool = False # allow/disallow requests for this end-user
max_budget: Optional[float] = None
budget_id: Optional[str] = None # give either a budget_id or max_budget
allowed_model_region: Optional[
AllowedModelRegion
] = None # require all user requests to use models in this specific region
default_model: Optional[
str
] = None # if no equivalent model in allowed region - default all requests to this model
allowed_model_region: Optional[AllowedModelRegion] = (
None # require all user requests to use models in this specific region
)
default_model: Optional[str] = (
None # if no equivalent model in allowed region - default all requests to this model
)
class DeleteCustomerRequest(LiteLLMPydanticObjectBase):
@ -1263,15 +1249,15 @@ class NewTeamRequest(TeamBase):
guardrails: Optional[List[str]] = None
prompts: Optional[List[str]] = None
object_permission: Optional[LiteLLM_ObjectPermissionBase] = None
team_member_budget: Optional[
float
] = None # allow user to set a budget for all team members
team_member_rpm_limit: Optional[
int
] = None # allow user to set RPM limit for all team members
team_member_tpm_limit: Optional[
int
] = None # allow user to set TPM limit for all team members
team_member_budget: Optional[float] = (
None # allow user to set a budget for all team members
)
team_member_rpm_limit: Optional[int] = (
None # allow user to set RPM limit for all team members
)
team_member_tpm_limit: Optional[int] = (
None # allow user to set TPM limit for all team members
)
team_member_key_duration: Optional[str] = None # e.g. "1d", "1w", "1m"
model_config = ConfigDict(protected_namespaces=())
@ -1350,9 +1336,9 @@ class BlockKeyRequest(LiteLLMPydanticObjectBase):
class AddTeamCallback(LiteLLMPydanticObjectBase):
callback_name: str
callback_type: Optional[
Literal["success", "failure", "success_and_failure"]
] = "success_and_failure"
callback_type: Optional[Literal["success", "failure", "success_and_failure"]] = (
"success_and_failure"
)
callback_vars: Dict[str, str]
@model_validator(mode="before")
@ -1621,9 +1607,9 @@ class ConfigList(LiteLLMPydanticObjectBase):
stored_in_db: Optional[bool]
field_default_value: Any
premium_field: bool = False
nested_fields: Optional[
List[FieldDetail]
] = None # For nested dictionary or Pydantic fields
nested_fields: Optional[List[FieldDetail]] = (
None # For nested dictionary or Pydantic fields
)
class UserHeaderMapping(LiteLLMPydanticObjectBase):
@ -1931,7 +1917,7 @@ class UserAPIKeyAuth(
key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME,
)
@classmethod
def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth":
"""
@ -1947,7 +1933,7 @@ class UserAPIKeyAuth(
key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME,
team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME,
)
@classmethod
def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth":
"""
@ -1990,9 +1976,9 @@ class LiteLLM_OrganizationMembershipTable(LiteLLMPydanticObjectBase):
budget_id: Optional[str] = None
created_at: datetime
updated_at: datetime
user: Optional[
Any
] = None # You might want to replace 'Any' with a more specific type if available
user: Optional[Any] = (
None # You might want to replace 'Any' with a more specific type if available
)
litellm_budget_table: Optional[LiteLLM_BudgetTable] = None
model_config = ConfigDict(protected_namespaces=())
@ -2887,9 +2873,9 @@ class TeamModelDeleteRequest(BaseModel):
# Organization Member Requests
class OrganizationMemberAddRequest(OrgMemberAddRequest):
organization_id: str
max_budget_in_organization: Optional[
float
] = None # Users max budget within the organization
max_budget_in_organization: Optional[float] = (
None # Users max budget within the organization
)
class OrganizationMemberDeleteRequest(MemberDeleteRequest):
@ -3099,9 +3085,9 @@ class ProviderBudgetResponse(LiteLLMPydanticObjectBase):
Maps provider names to their budget configs.
"""
providers: Dict[
str, ProviderBudgetResponseObject
] = {} # Dictionary mapping provider names to their budget configurations
providers: Dict[str, ProviderBudgetResponseObject] = (
{}
) # Dictionary mapping provider names to their budget configurations
class ProxyStateVariables(TypedDict):
@ -3235,9 +3221,9 @@ class LiteLLM_JWTAuth(LiteLLMPydanticObjectBase):
enforce_rbac: bool = False
roles_jwt_field: Optional[str] = None # v2 on role mappings
role_mappings: Optional[List[RoleMapping]] = None
object_id_jwt_field: Optional[
str
] = None # can be either user / team, inferred from the role mapping
object_id_jwt_field: Optional[str] = (
None # can be either user / team, inferred from the role mapping
)
scope_mappings: Optional[List[ScopeMapping]] = None
enforce_scope_based_access: bool = False
enforce_team_based_model_access: bool = False

6
litellm/proxy/auth/oauth2_proxy_hook.py
View file

@ -14,9 +14,9 @@ async def handle_oauth2_proxy_request(request: Request) -> UserAPIKeyAuth:
verbose_proxy_logger.debug("Handling oauth2 proxy request")
# Define the OAuth2 config mappings
oauth2_config_mappings: Dict[str, str] = general_settings.get(
"oauth2_config_mappings", {}
) or {}
oauth2_config_mappings: Dict[str, str] = (
general_settings.get("oauth2_config_mappings") or {}
)
verbose_proxy_logger.debug(f"Oauth2 config mappings: {oauth2_config_mappings}")
if not oauth2_config_mappings:

24
litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py
View file

@ -22,9 +22,7 @@ from fastapi import HTTPException
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.caching import DualCache
from litellm.integrations.custom_guardrail import (
CustomGuardrail,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.llms.custom_httpx.http_handler import (
get_async_httpx_client,
@ -597,11 +595,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 2. Update the messages with the guardrail response ##########
#########################################################
data[
"messages"
] = self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
data["messages"] = (
self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
)
)
#########################################################
@ -652,11 +650,11 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM):
#########################################################
########## 2. Update the messages with the guardrail response ##########
#########################################################
data[
"messages"
] = self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
data["messages"] = (
self._update_messages_with_updated_bedrock_guardrail_response(
messages=new_messages,
bedrock_guardrail_response=bedrock_guardrail_response,
)
)
#########################################################

64
litellm/proxy/hooks/parallel_request_limiter_v3.py
View file

@ -303,18 +303,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if "{" in key and "}" in key:
start = key.find("{")
end = key.find("}", start)
hash_tag = key[start:end+1]
hash_tag = key[start : end + 1]
else:
# Fallback for keys without hash tags
hash_tag = "no_hash_tag"
if hash_tag not in groups:
groups[hash_tag] = []
groups[hash_tag].append(key)
return groups
async def _execute_redis_batch_rate_limiter_script(
self,
keys_to_fetch: List[str],
@ -332,10 +331,10 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"""
if self.batch_rate_limiter_script is None:
return []
key_groups = self._group_keys_by_hash_tag(keys_to_fetch)
all_cache_values = []
for hash_tag, group_keys in key_groups.items():
try:
group_cache_values = await self.batch_rate_limiter_script(
@ -354,7 +353,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
window_size=self.window_size,
)
all_cache_values.extend(group_cache_values)
return all_cache_values
async def should_rate_limit(
@ -378,7 +377,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for descriptor in descriptors:
descriptor_key = descriptor["key"]
descriptor_value = descriptor["value"]
rate_limit: Optional[RateLimitDescriptorRateLimitObject] = descriptor.get("rate_limit", {}) or {}
rate_limit: RateLimitDescriptorRateLimitObject = (
descriptor.get("rate_limit") or RateLimitDescriptorRateLimitObject()
)
requests_limit = rate_limit.get("requests_per_unit")
tokens_limit = rate_limit.get("tokens_per_unit")
max_parallel_requests_limit = rate_limit.get("max_parallel_requests")
@ -632,26 +633,28 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
for i, status in enumerate(response["statuses"]):
if status["code"] == "OVER_LIMIT":
descriptor = descriptors[floor(i / 2)]
# Calculate reset time (window_start + window_size)
now = datetime.now().timestamp()
reset_time = now + self.window_size # Conservative estimate
reset_time_formatted = datetime.fromtimestamp(reset_time).strftime("%Y-%m-%d %H:%M:%S UTC")
reset_time_formatted = datetime.fromtimestamp(
reset_time
).strftime("%Y-%m-%d %H:%M:%S UTC")
# Handle negative remaining values more gracefully
remaining_display = max(0, status['limit_remaining'])
remaining_display = max(0, status["limit_remaining"])
# Create detailed error message
rate_limit_type = status['rate_limit_type']
current_limit = status['current_limit']
rate_limit_type = status["rate_limit_type"]
current_limit = status["current_limit"]
detail = (
f"Rate limit exceeded for {descriptor['key']}: {descriptor['value']}. "
f"Limit type: {rate_limit_type}. "
f"Current limit: {current_limit}, Remaining: {remaining_display}. "
f"Limit resets at: {reset_time_formatted}"
)
raise HTTPException(
status_code=429,
detail=detail,
@ -693,7 +696,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
return pipeline_operations
async def _execute_token_increment_script(
self,
pipeline_operations: List["RedisPipelineIncrementOperation"],
@ -703,15 +706,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
"""
if self.token_increment_script is None:
return
# Group operations by hash tag for Redis cluster compatibility
operation_keys = [op["key"] for op in pipeline_operations]
key_groups = self._group_keys_by_hash_tag(operation_keys)
for _hash_tag, group_keys in key_groups.items():
# Get operations for this hash tag group
group_operations = [op for op in pipeline_operations if op["key"] in group_keys]
group_operations = [
op for op in pipeline_operations if op["key"] in group_keys
]
keys = []
args = []
@ -731,7 +736,6 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
args=args,
)
async def async_increment_tokens_with_ttl_preservation(
self,
pipeline_operations: List["RedisPipelineIncrementOperation"],
@ -757,7 +761,7 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
try:
await self._execute_token_increment_script(pipeline_operations)
verbose_proxy_logger.debug(
f"Successfully executed TTL-preserving increment for {len(pipeline_operations)} keys"
)
@ -811,7 +815,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
)
# Get metadata from kwargs
litellm_metadata = kwargs["litellm_params"].get(get_metadata_variable_name_from_kwargs(kwargs), {})
litellm_metadata = kwargs["litellm_params"].get(
get_metadata_variable_name_from_kwargs(kwargs), {}
)
if litellm_metadata is None:
return
user_api_key = litellm_metadata.get("user_api_key")
@ -825,7 +831,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
# Get total tokens from response
total_tokens = 0
# spot fix for /responses api
if (isinstance(response_obj, ModelResponse) or isinstance(response_obj, BaseLiteLLMOpenAIResponseObject)):
if isinstance(response_obj, ModelResponse) or isinstance(
response_obj, BaseLiteLLMOpenAIResponseObject
):
_usage = getattr(response_obj, "usage", None)
if _usage and isinstance(_usage, Usage):
if rate_limit_type == "output":
@ -943,7 +951,9 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
_get_parent_otel_span_from_kwargs(kwargs)
)
litellm_metadata = kwargs["litellm_params"]["metadata"]
user_api_key = litellm_metadata.get("user_api_key") if litellm_metadata else None
user_api_key = (
litellm_metadata.get("user_api_key") if litellm_metadata else None
)
pipeline_operations: List[RedisPipelineIncrementOperation] = []
if user_api_key:

123
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -10,7 +10,6 @@ Has all /sso/* routes
import asyncio
import os
from litellm._uuid import uuid
from copy import deepcopy
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
@ -19,6 +18,7 @@ from fastapi.responses import RedirectResponse
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.caching import DualCache
from litellm.constants import MAX_SPENDLOG_ROWS_TO_QUERY
from litellm.llms.custom_httpx.http_handler import (
@ -115,7 +115,10 @@ def process_sso_jwt_access_token(
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
async def google_login(
request: Request, source: Optional[str] = None, key: Optional[str] = None, existing_key: Optional[str] = None
request: Request,
source: Optional[str] = None,
key: Optional[str] = None,
existing_key: Optional[str] = None,
): # noqa: PLR0915
"""
Create Proxy API Keys using Google Workspace SSO. Requires setting PROXY_BASE_URL in .env
@ -664,17 +667,20 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
status_code=401,
detail="Result not returned by SSO provider.",
)
if state and state.startswith(f"{LITELLM_CLI_SESSION_TOKEN_PREFIX}:"):
# Extract the key ID from the state
key_id = state.split(":", 1)[1]
# Get existing_key from query parameters if provided
existing_key = request.query_params.get("existing_key")
verbose_proxy_logger.info(f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}")
return await cli_sso_callback(request=request, key=key_id, existing_key=existing_key, result=result)
verbose_proxy_logger.info(
f"CLI SSO callback detected for key: {key_id}, existing_key: {existing_key}"
)
return await cli_sso_callback(
request=request, key=key_id, existing_key=existing_key, result=result
)
return await SSOAuthenticationHandler.get_redirect_response_from_openid(
result=result,
@ -685,30 +691,30 @@ async def auth_callback(request: Request, state: Optional[str] = None): # noqa:
)
async def _regenerate_cli_key(existing_key: str, new_key: str, user_id: Optional[str] = None) -> None:
async def _regenerate_cli_key(
existing_key: str, new_key: str, user_id: Optional[str] = None
) -> None:
"""Regenerate an existing CLI key with a new token"""
from litellm.proxy._types import RegenerateKeyRequest, UserAPIKeyAuth
from litellm.proxy.management_endpoints.key_management_endpoints import (
regenerate_key_fn,
)
verbose_proxy_logger.info(f"Regenerating existing CLI key: {existing_key}")
admin_user_dict = UserAPIKeyAuth.get_litellm_cli_user_api_key_auth()
regenerate_request = RegenerateKeyRequest(
key=existing_key,
new_key=new_key,
duration="24hr",
user_id=user_id,
)
await regenerate_key_fn(
key=existing_key,
data=regenerate_request,
user_api_key_dict=admin_user_dict
key=existing_key, data=regenerate_request, user_api_key_dict=admin_user_dict
)
verbose_proxy_logger.info(f"Regenerated CLI key: {new_key}")
@ -720,9 +726,9 @@ async def _create_new_cli_key(
from litellm.proxy.management_endpoints.key_management_endpoints import (
generate_key_helper_fn,
)
verbose_proxy_logger.info("Creating new CLI key")
await generate_key_helper_fn(
request_type="key",
duration="24hr",
@ -734,13 +740,20 @@ async def _create_new_cli_key(
table_name="key",
token=key,
)
verbose_proxy_logger.info(f"Created new CLI key: {key}")
async def cli_sso_callback(request: Request, key: Optional[str] = None, existing_key: Optional[str] = None, result: Optional[Union[OpenID, dict]] = None):
async def cli_sso_callback(
request: Request,
key: Optional[str] = None,
existing_key: Optional[str] = None,
result: Optional[Union[OpenID, dict]] = None,
):
"""CLI SSO callback - regenerates existing CLI key or creates new one"""
verbose_proxy_logger.info(f"CLI SSO callback for key: {key}, existing_key: {existing_key}")
verbose_proxy_logger.info(
f"CLI SSO callback for key: {key}, existing_key: {existing_key}"
)
from litellm.proxy.proxy_server import prisma_client
@ -754,8 +767,10 @@ async def cli_sso_callback(request: Request, key: Optional[str] = None, existing
raise HTTPException(
status_code=500, detail=CommonProxyErrors.db_not_connected_error.value
)
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(result=result)
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(
result=result
)
verbose_proxy_logger.debug(f"parsed_openid_result: {parsed_openid_result}")
try:
@ -783,7 +798,9 @@ async def cli_sso_callback(request: Request, key: Optional[str] = None, existing
except Exception as e:
verbose_proxy_logger.error(f"Error with CLI key: {e}")
raise HTTPException(status_code=500, detail=f"Failed to process CLI key: {str(e)}")
raise HTTPException(
status_code=500, detail=f"Failed to process CLI key: {str(e)}"
)
@router.get("/sso/cli/poll/{key_id}", tags=["experimental"], include_in_schema=False)
@ -874,8 +891,10 @@ async def insert_sso_user(
auto_create_key=False,
)
if result_openid:
new_user_request.metadata = {"auth_provider": result_openid.provider}
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {
"auth_provider": getattr(result_openid, "provider")
}
response = await new_user(
data=new_user_request,
@ -1052,11 +1071,13 @@ class SSOAuthenticationHandler:
# or a cryptographicly signed state that we can verify stateless
# For simplification we are using a static state, this is not perfect but some
# SSO providers do not allow stateless verification
redirect_params = SSOAuthenticationHandler._get_generic_sso_redirect_params(
state=state,
generic_authorization_endpoint=generic_authorization_endpoint
redirect_params = (
SSOAuthenticationHandler._get_generic_sso_redirect_params(
state=state,
generic_authorization_endpoint=generic_authorization_endpoint,
)
)
return await generic_sso.get_login_redirect(**redirect_params) # type: ignore
raise ValueError(
"Unknown SSO provider. Please setup SSO with client IDs https://docs.litellm.ai/docs/proxy/admin_ui_sso"
@ -1064,26 +1085,26 @@ class SSOAuthenticationHandler:
@staticmethod
def _get_generic_sso_redirect_params(
state: Optional[str] = None,
generic_authorization_endpoint: Optional[str] = None
state: Optional[str] = None,
generic_authorization_endpoint: Optional[str] = None,
) -> dict:
"""
Get redirect parameters for Generic SSO with proper state priority handling.
Priority order:
1. CLI state (if provided)
2. GENERIC_CLIENT_STATE environment variable
3. Generated UUID for Okta (if Okta endpoint detected)
Args:
state: Optional state parameter (e.g., CLI state)
generic_authorization_endpoint: Authorization endpoint URL
Returns:
dict: Redirect parameters for SSO login
"""
redirect_params = {}
if state:
# CLI state takes priority
# the litellm proxy cli sends the "state" parameter to the proxy server for auth. We should maintain the state parameter for the cli if it is provided
@ -1092,8 +1113,13 @@ class SSOAuthenticationHandler:
generic_client_state = os.getenv("GENERIC_CLIENT_STATE", None)
if generic_client_state:
redirect_params["state"] = generic_client_state
elif generic_authorization_endpoint and "okta" in generic_authorization_endpoint:
redirect_params["state"] = uuid.uuid4().hex # set state param for okta - required
elif (
generic_authorization_endpoint
and "okta" in generic_authorization_endpoint
):
redirect_params["state"] = (
uuid.uuid4().hex
) # set state param for okta - required
return redirect_params
@ -1127,11 +1153,11 @@ class SSOAuthenticationHandler:
redirect_url += sso_callback_route
else:
redirect_url += "/" + sso_callback_route
# Append existing_key as query parameter if provided
if existing_key:
redirect_url += f"?existing_key={existing_key}"
return redirect_url
@staticmethod
@ -1165,7 +1191,9 @@ class SSOAuthenticationHandler:
)
return user_info
except Exception as e:
verbose_proxy_logger.error(f"Error upserting SSO user into LiteLLM DB: {e}")
verbose_proxy_logger.exception(
f"Error upserting SSO user into LiteLLM DB: {e}"
)
return user_info
@staticmethod
@ -1314,7 +1342,9 @@ class SSOAuthenticationHandler:
return team_request
@staticmethod
def _get_cli_state(source: Optional[str], key: Optional[str], existing_key: Optional[str] = None) -> Optional[str]:
def _get_cli_state(
source: Optional[str], key: Optional[str], existing_key: Optional[str] = None
) -> Optional[str]:
"""
Checks the request 'source' if a cli state token was passed in
@ -1374,7 +1404,7 @@ class SSOAuthenticationHandler:
if user_email is not None and (user_id is None or len(user_id) == 0):
user_id = user_email
return ParsedOpenIDResult(
user_email=user_email,
user_id=user_id,
@ -1408,13 +1438,16 @@ class SSOAuthenticationHandler:
)
# User is Authe'd in - generate key for the UI to access Proxy
parsed_openid_result = SSOAuthenticationHandler._get_user_email_and_id_from_result(result=result, generic_client_id=generic_client_id)
parsed_openid_result = (
SSOAuthenticationHandler._get_user_email_and_id_from_result(
result=result, generic_client_id=generic_client_id
)
)
user_email = parsed_openid_result.get("user_email")
user_id = parsed_openid_result.get("user_id")
user_role = parsed_openid_result.get("user_role")
verbose_proxy_logger.info(f"SSO callback result: {result}")
user_info = None
user_id_models: List = []
max_internal_user_budget = litellm.max_internal_user_budget

2
litellm/proxy/management_helpers/team_member_permission_checks.py
View file

@ -171,7 +171,7 @@ class TeamMemberPermissionChecks:
"""
all_available_permissions = []
for route in LiteLLMRoutes.key_management_routes.value:
all_available_permissions.append(route.value)
all_available_permissions.append(route)
return all_available_permissions
@staticmethod

4
litellm/proxy/management_helpers/utils.py
View file

@ -1,6 +1,5 @@
# What is this?
## Helper utils for the management endpoints (keys/users/teams)
from litellm._uuid import uuid
from datetime import datetime
from functools import wraps
from typing import Optional, Tuple
@ -9,6 +8,7 @@ from fastapi import HTTPException, Request
import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.proxy._types import ( # key request types; user request types; team request types; customer request types
DeleteCustomerRequest,
DeleteTeamRequest,
@ -36,7 +36,7 @@ def get_new_internal_user_defaults(
user_info = litellm.default_internal_user_params or {}
returned_dict: SSOUserDefinedValues = {
"models": user_info.get("models", None),
"models": user_info.get("models") or [],
"max_budget": user_info.get("max_budget", litellm.max_internal_user_budget),
"budget_duration": user_info.get(
"budget_duration", litellm.internal_user_budget_duration

10
litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
View file

@ -459,14 +459,6 @@ async def anthropic_proxy_route(
region_name=None,
)
custom_headers = {}
if (
"authorization" not in request.headers
and "x-api-key" not in request.headers
and anthropic_api_key is not None
):
custom_headers["x-api-key"] = "{}".format(anthropic_api_key)
## check for streaming
is_streaming_request = await is_streaming_request_fn(request)
@ -474,7 +466,7 @@ async def anthropic_proxy_route(
endpoint_func = create_pass_through_route(
endpoint=endpoint,
target=str(updated_url),
custom_headers=custom_headers,
custom_headers={"x-api-key": "{}".format(anthropic_api_key)},
_forward_headers=True,
) # dynamically construct pass-through endpoint based on incoming path
received_value = await endpoint_func(

26
litellm/proxy/pass_through_endpoints/pass_through_endpoints.py
View file

@ -3,7 +3,6 @@ import asyncio
import copy
import json
import traceback
from litellm._uuid import uuid
from base64 import b64encode
from datetime import datetime
from typing import Dict, List, Optional, Tuple, Union
@ -25,6 +24,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile
import litellm
from litellm._logging import verbose_proxy_logger
from litellm._uuid import uuid
from litellm.constants import MAXIMUM_TRACEBACK_LINES_TO_LOG
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
@ -424,10 +424,10 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
for field_name, field_value in form_data.items():
if isinstance(field_value, (StarletteUploadFile, UploadFile)):
files[
field_name
] = await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
upload_file=field_value
files[field_name] = (
await HttpPassThroughEndpointHelpers._build_request_files_from_upload_file(
upload_file=field_value
)
)
else:
form_data_dict[field_name] = field_value
@ -476,7 +476,11 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
user_api_key_request_route=user_api_key_dict.request_route,
user_api_key_spend=user_api_key_dict.spend,
user_api_key_max_budget=user_api_key_dict.max_budget,
user_api_key_budget_reset_at=user_api_key_dict.budget_reset_at.isoformat() if user_api_key_dict.budget_reset_at else None,
user_api_key_budget_reset_at=(
user_api_key_dict.budget_reset_at.isoformat()
if user_api_key_dict.budget_reset_at
else None
),
)
)
@ -496,7 +500,7 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
kwargs = {
"litellm_params": {
**litellm_params_in_body,
**litellm_params_in_body, # type: ignore
"metadata": _metadata,
"proxy_server_request": {
"url": str(request.url),
@ -509,9 +513,9 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils):
"passthrough_logging_payload": passthrough_logging_payload,
}
logging_obj.model_call_details[
"passthrough_logging_payload"
] = passthrough_logging_payload
logging_obj.model_call_details["passthrough_logging_payload"] = (
passthrough_logging_payload
)
return kwargs
@ -923,7 +927,6 @@ def create_pass_through_route(
):
# check if target is an adapter.py or a url
from litellm._uuid import uuid
from litellm.proxy.types_utils.utils import get_instance_fn
try:
@ -1367,7 +1370,6 @@ async def create_pass_through_endpoints(
Create new pass-through endpoint
"""
from litellm._uuid import uuid
from litellm.proxy.proxy_server import (
get_config_general_settings,
update_config_general_settings,

11
litellm/proxy/proxy_server.py
View file

@ -637,11 +637,6 @@ async def proxy_startup_event(app: FastAPI):
user_api_key_cache=user_api_key_cache,
)
if use_background_health_checks:
asyncio.create_task(
_run_background_health_check()
) # start the background health check coroutine.
if prompt_injection_detection_obj is not None: # [TODO] - REFACTOR THIS
prompt_injection_detection_obj.update_environment(router=llm_router)
@ -664,6 +659,12 @@ async def proxy_startup_event(app: FastAPI):
await ProxyStartupEvent._update_default_team_member_budget()
# Start background health checks AFTER models are loaded and index is built
if use_background_health_checks:
asyncio.create_task(
_run_background_health_check()
) # start the background health check coroutine.
## [Optional] Initialize dd tracer
ProxyStartupEvent._init_dd_tracer()

22
litellm/proxy/spend_tracking/spend_tracking_utils.py
View file

@ -10,7 +10,7 @@ from pydantic import BaseModel
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELM_STRING, MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB, REDACTED_BY_LITELM_STRING
from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy._types import SpendLogsMetadata, SpendLogsPayload
@ -21,6 +21,7 @@ from litellm.types.utils import (
StandardLoggingModelInformation,
StandardLoggingPayload,
StandardLoggingVectorStoreRequest,
VectorStoreSearchResponse,
)
from litellm.utils import get_end_user_id_for_cost_tracking
@ -297,7 +298,9 @@ def get_logging_payload( # noqa: PLR0915
id = f"{id}_cache_hit{time.time()}" # SpendLogs does not allow duplicate request_id
mcp_namespaced_tool_name = None
mcp_tool_call_metadata = clean_metadata.get("mcp_tool_call_metadata", {})
mcp_tool_call_metadata: Optional[StandardLoggingMCPToolCall] = clean_metadata.get(
"mcp_tool_call_metadata"
)
if mcp_tool_call_metadata is not None:
mcp_namespaced_tool_name = mcp_tool_call_metadata.get(
"namespaced_tool_name", None
@ -505,23 +508,23 @@ def _sanitize_request_body_for_spend_logs_payload(
# This split ensures we keep more context from the end of conversations
start_ratio = 0.35
end_ratio = 0.65
# Calculate character distribution
start_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * start_ratio)
end_chars = int(MAX_STRING_LENGTH_PROMPT_IN_DB * end_ratio)
# Ensure we don't exceed the total limit
total_keep = start_chars + end_chars
if total_keep > MAX_STRING_LENGTH_PROMPT_IN_DB:
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars
# If the string length is less than what we want to keep, just truncate normally
if len(value) <= MAX_STRING_LENGTH_PROMPT_IN_DB:
return value
# Calculate how many characters are being skipped
skipped_chars = len(value) - total_keep
# Build the truncated string: beginning + truncation marker + end
truncated_value = (
f"{value[:start_chars]}"
@ -567,8 +570,9 @@ def _get_vector_store_request_for_spend_logs_payload(
if vector_store_request_metadata is None:
return None
for vector_store_request in vector_store_request_metadata:
vector_store_search_response = (
vector_store_request.get("vector_store_search_response", {}) or {}
vector_store_search_response: VectorStoreSearchResponse = (
vector_store_request.get("vector_store_search_response")
or VectorStoreSearchResponse()
)
response_data = vector_store_search_response.get("data", []) or []
for response_item in response_data:

148
litellm/router.py
View file

@ -17,7 +17,6 @@ import logging
import threading
import time
import traceback
from litellm._uuid import uuid
from collections import defaultdict
from functools import lru_cache
from typing import (
@ -45,6 +44,7 @@ import litellm.litellm_core_utils
import litellm.litellm_core_utils.exception_mapping_utils
from litellm import get_secret_str
from litellm._logging import verbose_router_logger
from litellm._uuid import uuid
from litellm.caching.caching import (
DualCache,
InMemoryCache,
@ -409,7 +409,12 @@ class Router:
) # {"TEAM_ID": PatternMatchRouter}
self.auto_routers: Dict[str, "AutoRouter"] = {}
# Initialize model ID to deployment index mapping for O(1) lookups
self.model_id_to_deployment_index_map: Dict[str, int] = {}
if model_list is not None:
# Build model index immediately to enable O(1) lookups from the start
self._build_model_id_to_deployment_index_map(model_list)
model_list = copy.deepcopy(model_list)
self.set_model_list(model_list)
self.healthy_deployments: List = self.model_list # type: ignore
@ -2005,11 +2010,17 @@ class Router:
# Filter out prompt management specific parameters from data before merging
prompt_management_params = {
"bitbucket_config", "dotprompt_config", "prompt_id",
"prompt_variables", "prompt_label", "prompt_version"
"bitbucket_config",
"dotprompt_config",
"prompt_id",
"prompt_variables",
"prompt_label",
"prompt_version",
}
filtered_data = {k: v for k, v in data.items() if k not in prompt_management_params}
filtered_data = {
k: v for k, v in data.items() if k not in prompt_management_params
}
kwargs = {**filtered_data, **kwargs, **optional_params}
kwargs["model"] = model
kwargs["messages"] = messages
@ -3436,7 +3447,7 @@ class Router:
*[try_retrieve_batch(model) for model in filtered_model_list]
)
final_results = {
final_results: Dict = {
"object": "list",
"data": [],
"first_id": None,
@ -4108,7 +4119,9 @@ class Router:
"""
model_group = kwargs.get("model")
response = original_function(*args, **kwargs)
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(response):
if coroutine_checker.is_async_callable(response) or inspect.isawaitable(
response
):
response = await response
## PROCESS RESPONSE HEADERS
response = await self.set_response_headers(
@ -4517,7 +4530,9 @@ class Router:
_time_to_cooldown = self.cooldown_time
if isinstance(_model_info, dict):
deployment_id = _model_info.get("id", None)
deployment_id: Optional[str] = _model_info.get("id")
if deployment_id is None:
return False
increment_deployment_failures_for_current_minute(
litellm_router_instance=self,
deployment_id=deployment_id,
@ -4974,7 +4989,7 @@ class Router:
model = deployment.to_json(exclude_none=True)
self.model_list.append(model)
self._add_model_to_list_and_index_map(model=model, model_id=deployment.model_info.id)
return deployment
except Exception as e:
if self.ignore_invalid_deployments:
@ -5085,6 +5100,7 @@ class Router:
def set_model_list(self, model_list: list):
original_model_list = copy.deepcopy(model_list)
self.model_list = []
self.model_id_to_deployment_index_map = {} # Reset the index
# we add api_base/api_key each model so load balancing between azure/gpt on api_base1 and api_base2 works
for model in original_model_list:
@ -5134,12 +5150,12 @@ class Router:
# Check if this is a prompt management model before validating as LLM provider
litellm_model = deployment.litellm_params.model
is_prompt_management_model = False
if "/" in litellm_model:
split_litellm_model = litellm_model.split("/")[0]
if split_litellm_model in litellm._known_custom_logger_compatible_callbacks:
is_prompt_management_model = True
if is_prompt_management_model:
# For prompt management models, skip LLM provider validation
# The actual model will be resolved at runtime from the prompt file
@ -5229,11 +5245,12 @@ class Router:
# litellm_router_instance=self, model=deployment.to_json(exclude_none=True)
# )
self._initialize_deployment_for_pass_through(
deployment=deployment,
custom_llm_provider=custom_llm_provider,
model=deployment.litellm_params.model,
)
if custom_llm_provider is not None:
self._initialize_deployment_for_pass_through(
deployment=deployment,
custom_llm_provider=custom_llm_provider,
model=deployment.litellm_params.model,
)
#########################################################
# Check if this is an auto-router deployment
@ -5323,10 +5340,42 @@ class Router:
self._add_deployment(deployment=deployment)
# add to model names
self.model_list.append(_deployment)
self._add_model_to_list_and_index_map(model=_deployment, model_id=deployment.model_info.id)
self.model_names.append(deployment.model_name)
return deployment
def _update_deployment_indices_after_removal(self, model_id: str, removal_idx: int) -> None:
"""
Helper method to update deployment indices after a deployment has been removed from model_list.
Parameters:
- model_id: str - the id of the deployment that was removed
- removal_idx: int - the index where the deployment was removed from model_list
"""
# Update indices for all models after the removed one
for deployment_id, idx in self.model_id_to_deployment_index_map.items():
if idx > removal_idx:
self.model_id_to_deployment_index_map[deployment_id] = idx - 1
# Remove the deleted model from index
if model_id in self.model_id_to_deployment_index_map:
del self.model_id_to_deployment_index_map[model_id]
def _add_model_to_list_and_index_map(self, model: dict, model_id: Optional[str] = None) -> None:
"""
Helper method to add a model to the model_list and update the model_id_to_deployment_index_map.
Parameters:
- model: dict - the model to add to the list
- model_id: Optional[str] - the model ID to use for indexing. If None, will try to get from model["model_info"]["id"]
"""
self.model_list.append(model)
# Update model index for O(1) lookup
if model_id is not None:
self.model_id_to_deployment_index_map[model_id] = len(self.model_list) - 1
elif model.get("model_info", {}).get("id") is not None:
self.model_id_to_deployment_index_map[model["model_info"]["id"]] = len(self.model_list) - 1
def upsert_deployment(self, deployment: Deployment) -> Optional[Deployment]:
"""
Add or update deployment
@ -5352,12 +5401,15 @@ class Router:
# if there is a new litellm param -> then update the deployment
# remove the previous deployment
removal_idx: Optional[int] = None
for idx, model in enumerate(self.model_list):
if model["model_info"]["id"] == deployment.model_info.id:
removal_idx = idx
deployment_id = deployment.model_info.id
deployment_fast_mapping = self.model_id_to_deployment_index_map
if deployment_id in deployment_fast_mapping:
removal_idx = deployment_fast_mapping[deployment_id]
if removal_idx is not None:
self.model_list.pop(removal_idx)
if removal_idx is not None:
self.model_list.pop(removal_idx)
self._update_deployment_indices_after_removal(model_id=deployment_id, removal_idx=removal_idx)
# if the model_id is not in router
self.add_deployment(deployment=deployment)
@ -5381,13 +5433,14 @@ class Router:
- OR None (if deleted deployment not found)
"""
deployment_idx = None
for idx, m in enumerate(self.model_list):
if m["model_info"]["id"] == id:
deployment_idx = idx
if id in self.model_id_to_deployment_index_map:
deployment_idx = self.model_id_to_deployment_index_map[id]
try:
if deployment_idx is not None:
# Pop the item from the list first
item = self.model_list.pop(deployment_idx)
self._update_deployment_indices_after_removal(model_id=id, removal_idx=deployment_idx)
return item
else:
return None
@ -5400,15 +5453,17 @@ class Router:
Raise Exception -> if model found in invalid format
"""
for model in self.model_list:
if "model_info" in model and "id" in model["model_info"]:
if model_id == model["model_info"]["id"]:
if isinstance(model, dict):
return Deployment(**model)
elif isinstance(model, Deployment):
return model
else:
raise Exception("Model invalid format - {}".format(type(model)))
# Use O(1) lookup via model_id_to_deployment_index_map only
if model_id in self.model_id_to_deployment_index_map:
idx = self.model_id_to_deployment_index_map[model_id]
model = self.model_list[idx]
if isinstance(model, dict):
return Deployment(**model)
elif isinstance(model, Deployment):
return model
else:
raise Exception("Model invalid format - {}".format(type(model)))
return None
def get_deployment_credentials(self, model_id: str) -> Optional[dict]:
@ -6026,6 +6081,31 @@ class Router:
additional_headers[header] = value
return response
def _build_model_id_to_deployment_index_map(self, model_list: list):
"""
Build model index from model list to enable O(1) lookups immediately.
This is called during initialization to avoid the race condition where
requests arrive before model_id_to_deployment_index_map is populated.
"""
# First populate the model_list
self.model_list = []
for _, model in enumerate(model_list):
# Extract model_info from the model dict
model_info = model.get("model_info", {})
model_id = model_info.get("id")
# If no ID exists, generate one using the same logic as set_model_list
if model_id is None:
model_name = model.get("model_name", "")
litellm_params = model.get("litellm_params", {})
model_id = self._generate_model_id(model_name, litellm_params)
# Update the model_info in the original list
if "model_info" not in model:
model["model_info"] = {}
model["model_info"]["id"] = model_id
self._add_model_to_list_and_index_map(model=model, model_id=model_id)
def get_model_ids(
self, model_name: Optional[str] = None, exclude_team_models: bool = False
) -> List[str]:

9
litellm/types/llms/openai.py
View file

@ -43,9 +43,12 @@ from openai.types.responses.response import (
# Handle OpenAI SDK version compatibility for Text type
try:
from openai.types.responses.response_create_params import (
Text as ResponseText, # type: ignore
# fmt: off
from openai.types.responses.response_create_params import ( # type: ignore[attr-defined]
Text as ResponseText, # type: ignore[attr-defined]
)
# fmt: on
except (ImportError, AttributeError):
# Fall back to the concrete config type available in all SDK versions
from openai.types.responses.response_text_config_param import (
@ -1308,7 +1311,7 @@ class MCPListToolsFailedEvent(BaseLiteLLMOpenAIResponseObject):
item_id: str
# MCP Call Events
# MCP Call Events
class MCPCallInProgressEvent(BaseLiteLLMOpenAIResponseObject):
type: Literal[ResponsesAPIStreamEvents.MCP_CALL_IN_PROGRESS]
sequence_number: int

4
pyproject.toml
View file

@ -1,6 +1,6 @@
[tool.poetry]
name = "litellm"
version = "1.77.4"
version = "1.77.5"
description = "Library to easily interface with LLM API providers"
authors = ["BerriAI"]
license = "MIT"
@ -157,7 +157,7 @@ requires = ["poetry-core", "wheel"]
build-backend = "poetry.core.masonry.api"
[tool.commitizen]
version = "1.77.4"
version = "1.77.5"
version_files = [
"pyproject.toml:^version"
]

BIN
test_image_edit.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 1.5 MiB

2
tests/enterprise/litellm_enterprise/integrations/test_prometheus_unit_tests.py
View file

@ -166,7 +166,7 @@ async def test_prometheus_metric_tracking():
{
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),

13
tests/image_gen_tests/test_image_generation.py
View file

@ -118,19 +118,6 @@ class TestVertexImageGeneration(BaseImageGenTest):
"n": 1,
}
class TestBedrockSd3(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
litellm.in_memory_llm_clients_cache = InMemoryCache()
return {"model": "bedrock/stability.sd3-large-v1:0"}
class TestBedrockSd1(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
litellm.in_memory_llm_clients_cache = InMemoryCache()
return {"model": "bedrock/stability.sd3-large-v1:0"}
class TestBedrockNovaCanvasTextToImage(BaseImageGenTest):
def get_base_image_generation_call_args(self) -> dict:
litellm.in_memory_llm_clients_cache = InMemoryCache()

26
tests/llm_translation/test_azure_ai.py
View file

@ -313,3 +313,29 @@ async def test_azure_gpt5_reasoning(model):
)
print("response: ", response)
assert response.choices[0].message.content is not None
def test_completion_azure():
try:
litellm.set_verbose = False
## Test azure call
response = completion(
model="azure/gpt-4.1-nano",
messages=[
{
"role": "user",
"content": "Hello, how are you?",
}
],
api_key="os.environ/AZURE_API_KEY",
)
print(f"response: {response}")
print(f"response hidden params: {response._hidden_params}")
print(response)
cost = completion_cost(completion_response=response)
assert cost > 0.0
print("Cost for azure completion request", cost)
except Exception as e:
pytest.fail(f"Error occurred: {e}")

2
tests/llm_translation/test_azure_openai.py
View file

@ -137,7 +137,7 @@ def test_azure_extra_headers(input, call_type, header_value):
func = image_generation
data = {
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com",
"api_version": "2023-07-01-preview",
"api_key": "my-azure-api-key",

2
tests/local_testing/example_config_yaml/azure_config.yaml
View file

@ -1,7 +1,7 @@
model_list:
- model_name: gpt-4-team1
litellm_params:
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
api_base: https://openai-gpt-4-test-v-1.openai.azure.com/
api_version: "2023-05-15"
api_key: os.environ/AZURE_API_KEY

6
tests/local_testing/test_acooldowns_router.py
View file

@ -26,7 +26,7 @@ model_list = [
{ # list of model deployments
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_key": "bad-key",
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),
@ -180,7 +180,7 @@ async def test_cooldown_same_model_name(sync_mode):
model_ids.append(model["model_info"]["id"])
print("\n litellm model ids ", model_ids)
# example litellm_model_names ['azure/chatgpt-v-3-ModelID-64321', 'azure/chatgpt-v-3-ModelID-63960']
# example litellm_model_names ['azure/gpt-4.1-nano-ModelID-64321', 'azure/gpt-4.1-nano-ModelID-63960']
assert (
model_ids[0] != model_ids[1]
) # ensure both models have a uuid added, and they have different names
@ -197,7 +197,7 @@ async def test_cooldown_same_model_name(sync_mode):
model_ids.append(model["model_info"]["id"])
print("\n litellm model ids ", model_ids)
# example litellm_model_names ['azure/chatgpt-v-3-ModelID-64321', 'azure/chatgpt-v-3-ModelID-63960']
# example litellm_model_names ['azure/gpt-4.1-nano-ModelID-64321', 'azure/gpt-4.1-nano-ModelID-63960']
assert (
model_ids[0] != model_ids[1]
) # ensure both models have a uuid added, and they have different names

2
tests/local_testing/test_alangfuse.py
View file

@ -194,7 +194,7 @@ def create_async_task(**completion_kwargs):
By default a standard set of arguments are used for the litellm.acompletion function.
"""
completion_args = {
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_version": "2024-02-01",
"messages": [{"role": "user", "content": "This is a test"}],
"max_tokens": 5,

2
tests/local_testing/test_amazing_vertex_completion.py
View file

@ -764,7 +764,7 @@ def test_gemini_pro_grounding(value_in_dict):
# @pytest.mark.skip(reason="exhausted vertex quota. need to refactor to mock the call")
@pytest.mark.parametrize("model", ["vertex_ai_beta/gemini-1.5-pro"]) # "vertex_ai",
@pytest.mark.parametrize("model", ["vertex_ai_beta/gemini-2.5-flash-lite"]) # "vertex_ai",
@pytest.mark.parametrize("sync_mode", [True]) # "vertex_ai",
@pytest.mark.asyncio
@pytest.mark.flaky(retries=3, delay=1)

4
tests/local_testing/test_azure_openai.py
View file

@ -47,7 +47,7 @@ async def test_aaaaazure_tenant_id_auth(respx_mock: MockRouter):
{
"model_name": "gpt-3.5-turbo",
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_base": os.getenv("AZURE_API_BASE"),
"tenant_id": os.getenv("AZURE_TENANT_ID"),
"client_id": os.getenv("AZURE_CLIENT_ID"),
@ -96,6 +96,6 @@ async def test_aaaaazure_tenant_id_auth(respx_mock: MockRouter):
assert json_body == {
"messages": [{"role": "user", "content": "Hello world!"}],
"model": "chatgpt-v-3",
"model": "gpt-4.1-nano",
"stream": False,
}

2
tests/local_testing/test_azure_perf.py
View file

@ -18,7 +18,7 @@
# {
# "model_name": "azure-test",
# "litellm_params": {
# "model": "azure/chatgpt-v-3",
# "model": "azure/gpt-4.1-nano",
# "api_key": os.getenv("AZURE_API_KEY"),
# "api_base": os.getenv("AZURE_API_BASE"),
# "api_version": os.getenv("AZURE_API_VERSION"),

10
tests/local_testing/test_caching.py
View file

@ -1627,7 +1627,7 @@ def test_get_cache_key():
embedding_cache_key = cache_instance.get_cache_key(
**{
"model": "azure/azure-embedding-model",
"model": "azure/text-embedding-ada-002",
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
"api_key": "",
"api_version": "2023-07-01-preview",
@ -1642,19 +1642,19 @@ def test_get_cache_key():
print(embedding_cache_key)
embedding_cache_key_str = (
"model: azure/azure-embedding-modelinput: ['hi who is ishaan']"
"model: azure/text-embedding-ada-002input: ['hi who is ishaan']"
)
hash_object = hashlib.sha256(embedding_cache_key_str.encode())
# Hexadecimal representation of the hash
hash_hex = hash_object.hexdigest()
assert (
embedding_cache_key == hash_hex
), f"{embedding_cache_key} != 'model: azure/azure-embedding-modelinput: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs"
), f"{embedding_cache_key} != 'model: azure/text-embedding-ada-002input: ['hi who is ishaan']'. The same kwargs should have the same cache key across runs"
# Proxy - embedding cache, test if embedding key, gets model_group and not model
embedding_cache_key_2 = cache_instance.get_cache_key(
**{
"model": "azure/azure-embedding-model",
"model": "azure/text-embedding-ada-002",
"api_base": "https://openai-gpt-4-test-v-1.openai.azure.com/",
"api_key": "",
"api_version": "2023-07-01-preview",
@ -1689,7 +1689,7 @@ def test_get_cache_key():
"content-length": "80",
},
"model_group": "EMBEDDING_MODEL_GROUP",
"deployment": "azure/azure-embedding-model-ModelID-azure/azure-embedding-modelhttps://openai-gpt-4-test-v-1.openai.azure.com/2023-07-01-preview",
"deployment": "azure/text-embedding-ada-002-ModelID-azure/text-embedding-ada-002https://openai-gpt-4-test-v-1.openai.azure.com/2023-07-01-preview",
},
"model_info": {
"mode": "embedding",

2
tests/local_testing/test_caching_ssl.py
View file

@ -58,7 +58,7 @@ def test_caching_router():
{
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),

4
tests/local_testing/test_class.py
View file

@ -55,7 +55,7 @@
# # {
# # "model_name": "gpt-3.5-turbo", # openai model name
# # "litellm_params": { # params for litellm completion/embedding call
# # "model": "azure/chatgpt-v-3",
# # "model": "azure/gpt-4.1-nano",
# # "api_key": os.getenv("AZURE_API_KEY"),
# # "api_version": os.getenv("AZURE_API_VERSION"),
# # "api_base": os.getenv("AZURE_API_BASE"),
@ -93,7 +93,7 @@
# # {
# # "model_name": "gpt-3.5-turbo", # openai model name
# # "litellm_params": { # params for litellm completion/embedding call
# # "model": "azure/chatgpt-v-3",
# # "model": "azure/gpt-4.1-nano",
# # "api_key": os.getenv("AZURE_API_KEY"),
# # "api_version": os.getenv("AZURE_API_VERSION"),
# # "api_base": os.getenv("AZURE_API_BASE"),

85
tests/local_testing/test_completion.py
View file

@ -712,7 +712,7 @@ def encode_image(image_path):
"model",
[
"gpt-4o",
"azure/gpt-4o-new-test",
"azure/gpt-4.1-nano",
"anthropic/claude-3-opus-20240229",
],
) #
@ -1746,7 +1746,7 @@ def test_completion_openai():
"model, api_version",
[
# ("gpt-4o-2024-08-06", None),
# ("azure/chatgpt-v-3", None),
# ("azure/gpt-4.1-nano", None),
("bedrock/anthropic.claude-3-sonnet-20240229-v1:0", None),
# ("azure/gpt-4o-new-test", "2024-08-01-preview"),
],
@ -2417,7 +2417,7 @@ def test_completion_azure_extra_headers():
litellm.client_session = http_client
try:
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
api_base=os.getenv("AZURE_API_BASE"),
api_version="2023-07-01-preview",
@ -2466,7 +2466,7 @@ def test_completion_azure_ad_token():
litellm.client_session = http_client
try:
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
azure_ad_token="my-special-token",
)
@ -2497,7 +2497,7 @@ def test_completion_azure_key_completion_arg():
litellm.set_verbose = True
## Test azure call
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
api_key=old_key,
logprobs=True,
@ -2514,31 +2514,6 @@ def test_completion_azure_key_completion_arg():
pytest.fail(f"Error occurred: {e}")
# test_completion_azure_key_completion_arg()
def test_azure_instruct():
litellm.set_verbose = True
response = completion(
model="azure_text/instruct-model",
messages=[{"role": "user", "content": "What is the weather like in Boston?"}],
max_tokens=10,
)
print("response", response)
@pytest.mark.asyncio
async def test_azure_instruct_stream():
litellm.set_verbose = False
response = await litellm.acompletion(
model="azure_text/instruct-model",
messages=[{"role": "user", "content": "What is the weather like in Boston?"}],
max_tokens=10,
stream=True,
)
print("response", response)
async for chunk in response:
print(chunk)
async def test_re_use_azure_async_client():
@ -2555,7 +2530,7 @@ async def test_re_use_azure_async_client():
## Test azure call
for _ in range(3):
response = await litellm.acompletion(
model="azure/chatgpt-v-3", messages=messages, client=client
model="azure/gpt-4.1-nano", messages=messages, client=client
)
print(f"response: {response}")
except Exception as e:
@ -2581,37 +2556,6 @@ def test_re_use_openaiClient():
pytest.fail("got Exception", e)
def test_completion_azure():
try:
litellm.set_verbose = False
## Test azure call
response = completion(
model="azure/gpt-4o-new-test",
messages=messages,
api_key="os.environ/AZURE_API_KEY",
)
print(f"response: {response}")
print(f"response hidden params: {response._hidden_params}")
## Test azure flag for backwards-compat
# response = completion(
# model="chatgpt-v-3",
# messages=messages,
# azure=True,
# max_tokens=10
# )
# Add any assertions here to check the response
print(response)
cost = completion_cost(completion_response=response)
assert cost > 0.0
print("Cost for azure completion request", cost)
except Exception as e:
pytest.fail(f"Error occurred: {e}")
# test_completion_azure()
@pytest.mark.skip(
reason="this is bad test. It doesn't actually fail if the token is not set in the header. "
)
@ -2633,7 +2577,7 @@ def test_azure_openai_ad_token():
litellm.input_callback = [tester]
try:
response = litellm.completion(
model="azure/chatgpt-v-3", # e.g. gpt-35-instant
model="azure/gpt-4.1-nano", # e.g. gpt-35-instant
messages=[
{
"role": "user",
@ -2671,7 +2615,7 @@ def test_completion_azure2():
## Test azure call
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
api_base=api_base,
api_key=api_key,
@ -2708,7 +2652,7 @@ def test_completion_azure3():
## Test azure call
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
max_tokens=10,
)
@ -2756,7 +2700,7 @@ def test_completion_azure_with_litellm_key():
openai.api_key = "ymca"
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
)
# Add any assertions here to check the response
@ -2784,7 +2728,7 @@ def test_completion_azure_deployment_id():
try:
litellm.set_verbose = True
response = completion(
deployment_id="gpt-4o-new-test",
deployment_id="gpt-4.1-nano",
model="gpt-3.5-turbo",
messages=messages,
)
@ -3695,8 +3639,7 @@ def test_completion_volcengine():
"model",
[
# "gemini-1.0-pro",
"gemini-1.5-pro",
# "gemini-2.5-flash-lite",
"gemini-2.5-flash-lite",
],
)
@pytest.mark.flaky(retries=3, delay=1)
@ -4098,7 +4041,7 @@ async def test_completion_ai21_chat():
@pytest.mark.parametrize(
"model",
["gpt-4o", "azure/chatgpt-v-3"],
["gpt-4o", "azure/gpt-4.1-nano"],
)
@pytest.mark.parametrize(
"stream",
@ -4120,7 +4063,7 @@ def test_completion_response_ratelimit_headers(model, stream):
assert "x-ratelimit-remaining-requests" in additional_headers
assert "x-ratelimit-remaining-tokens" in additional_headers
if model == "azure/chatgpt-v-3":
if model == "azure/gpt-4.1-nano":
# Azure OpenAI header
assert "llm_provider-azureml-model-session" in additional_headers
if model == "claude-3-sonnet-20240229":

2
tests/local_testing/test_completion_cost.py
View file

@ -283,7 +283,7 @@ def test_cost_azure_embedding():
async def _test():
response = await litellm.aembedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm", "gm"],
)

6
tests/local_testing/test_config.py
View file

@ -46,7 +46,7 @@ async def test_delete_deployment():
import base64
litellm_params = LiteLLM_Params(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
api_key=os.getenv("AZURE_API_KEY"),
api_base=os.getenv("AZURE_API_BASE"),
api_version=os.getenv("AZURE_API_VERSION"),
@ -232,7 +232,7 @@ async def test_db_error_new_model_check():
litellm_params = LiteLLM_Params(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
api_key=os.getenv("AZURE_API_KEY"),
api_base=os.getenv("AZURE_API_BASE"),
api_version=os.getenv("AZURE_API_VERSION"),
@ -250,7 +250,7 @@ def _create_model_list(flag_value: Literal[0, 1], master_key: str):
import base64
new_litellm_params = LiteLLM_Params(
model="azure/chatgpt-v-3-3",
model="azure/gpt-4.1-nano-3",
api_key=os.getenv("AZURE_API_KEY"),
api_base=os.getenv("AZURE_API_BASE"),
api_version=os.getenv("AZURE_API_VERSION"),

6
tests/local_testing/test_configs/test_bad_config.yaml
View file

@ -5,17 +5,17 @@ model_list:
model: gpt-3.5-turbo
- model_name: working-azure-gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
- model_name: azure-gpt-3.5-turbo
litellm_params:
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
api_base: os.environ/AZURE_API_BASE
api_key: bad-key
- model_name: azure-embedding
litellm_params:
model: azure/azure-embedding-model
model: azure/text-embedding-ada-002
api_base: os.environ/AZURE_API_BASE
api_key: bad-key

2
tests/local_testing/test_configs/test_cloudflare_azure_with_cache_config.yaml
View file

@ -1,7 +1,7 @@
model_list:
- model_name: azure-cloudflare
litellm_params:
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
api_key: os.environ/AZURE_API_KEY
api_version: 2023-07-01-preview

6
tests/local_testing/test_configs/test_config_no_auth.yaml
View file

@ -12,7 +12,7 @@ model_list:
- litellm_params:
api_base: https://gateway.ai.cloudflare.com/v1/0399b10e77ac6668c80404a5ff49eb37/litellm-test/azure-openai/openai-gpt-4-test-v-1
api_key: os.environ/AZURE_API_KEY
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
model_name: azure-cloudflare-model
- litellm_params:
api_base: https://openai-france-1234.openai.azure.com
@ -52,7 +52,7 @@ model_list:
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: 2023-07-01-preview
model: azure/azure-embedding-model
model: azure/text-embedding-ada-002
model_info:
mode: embedding
model_name: azure-embedding-model
@ -105,7 +105,7 @@ model_list:
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: 2023-07-01-preview
model: azure/azure-embedding-model
model: azure/text-embedding-ada-002
model_info:
base_model: text-embedding-ada-002
mode: embedding

4
tests/local_testing/test_configs/test_custom_logger.yaml
View file

@ -1,7 +1,7 @@
model_list:
- model_name: Azure OpenAI GPT-4 Canada
litellm_params:
model: azure/chatgpt-v-3
model: azure/gpt-4.1-nano
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: "2023-07-01-preview"
@ -11,7 +11,7 @@ model_list:
id: gm
- model_name: azure-embedding-model
litellm_params:
model: azure/azure-embedding-model
model: azure/text-embedding-ada-002
api_base: os.environ/AZURE_API_BASE
api_key: os.environ/AZURE_API_KEY
api_version: "2023-07-01-preview"

26
tests/local_testing/test_custom_callback_input.py
View file

@ -450,12 +450,12 @@ def test_chat_azure_stream():
customHandler = CompletionCustomHandler()
litellm.callbacks = [customHandler]
response = litellm.completion(
model="azure/gpt-4o-new-test",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm sync azure"}],
)
# test streaming
response = litellm.completion(
model="azure/gpt-4o-new-test",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm sync azure"}],
stream=True,
)
@ -464,7 +464,7 @@ def test_chat_azure_stream():
# test failure callback
try:
response = litellm.completion(
model="azure/gpt-4o-new-test",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm sync azure"}],
api_key="my-bad-key",
stream=True,
@ -491,12 +491,12 @@ async def test_async_chat_azure_stream():
customHandler = CompletionCustomHandler()
litellm.callbacks = [customHandler]
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm async azure"}],
)
## test streaming
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm async azure"}],
stream=True,
)
@ -507,7 +507,7 @@ async def test_async_chat_azure_stream():
# test failure callback
try:
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "Hi 👋 - i'm async azure"}],
api_key="my-bad-key",
stream=True,
@ -774,7 +774,8 @@ async def test_async_embedding_openai():
customHandler_failure = CompletionCustomHandler()
litellm.callbacks = [customHandler_success]
response = await litellm.aembedding(
model="azure/azure-embedding-model", input=["good morning from litellm"]
model="text-embedding-ada-002",
input=["good morning from litellm"],
)
await asyncio.sleep(1)
print(f"customHandler_success.errors: {customHandler_success.errors}")
@ -811,7 +812,7 @@ def test_amazing_sync_embedding():
customHandler_failure = CompletionCustomHandler()
litellm.callbacks = [customHandler_success]
response = litellm.embedding(
model="azure/azure-embedding-model", input=["good morning from litellm"]
model="azure/text-embedding-ada-002", input=["good morning from litellm"]
)
print(f"customHandler_success.errors: {customHandler_success.errors}")
print(f"customHandler_success.states: {customHandler_success.states}")
@ -823,7 +824,7 @@ def test_amazing_sync_embedding():
litellm.callbacks = [customHandler_failure]
try:
response = litellm.embedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm"],
api_key="my-bad-key",
)
@ -846,7 +847,7 @@ async def test_async_embedding_azure():
customHandler_failure = CompletionCustomHandler()
litellm.callbacks = [customHandler_success]
response = await litellm.aembedding(
model="azure/azure-embedding-model", input=["good morning from litellm"]
model="azure/text-embedding-ada-002", input=["good morning from litellm"]
)
await asyncio.sleep(1)
print(f"customHandler_success.errors: {customHandler_success.errors}")
@ -858,7 +859,7 @@ async def test_async_embedding_azure():
litellm.callbacks = [customHandler_failure]
try:
response = await litellm.aembedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm"],
api_key="my-bad-key",
)
@ -914,7 +915,6 @@ async def test_async_embedding_bedrock():
pytest.fail(f"An exception occurred: {str(e)}")
# Image Generation
@ -1004,7 +1004,7 @@ def test_turn_off_message_logging():
"model",
[
"ft:gpt-3.5-turbo:my-org:custom_suffix:id"
], # "gpt-3.5-turbo", "azure/chatgpt-v-3",
], # "gpt-3.5-turbo", "azure/gpt-4.1-nano",
)
@pytest.mark.parametrize(
"turn_off_message_logging",

6
tests/local_testing/test_custom_logger.py
View file

@ -160,7 +160,7 @@ def test_completion_azure_stream_moderation_failure():
]
try:
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=messages,
mock_response="Exception: content_filter_policy",
stream=True,
@ -195,7 +195,7 @@ def test_async_custom_handler_stream():
async def test_1():
nonlocal complete_streaming_response
response = await litellm.acompletion(
model="azure/chatgpt-v-3", messages=messages, stream=True
model="azure/gpt-4.1-nano", messages=messages, stream=True
)
async for chunk in response:
complete_streaming_response += (
@ -239,7 +239,7 @@ def test_azure_completion_stream():
complete_streaming_response = ""
response = litellm.completion(
model="azure/chatgpt-v-3", messages=messages, stream=True
model="azure/gpt-4.1-nano", messages=messages, stream=True
)
for chunk in response:
complete_streaming_response += chunk["choices"][0]["delta"]["content"] or ""

20
tests/local_testing/test_embedding.py
View file

@ -107,7 +107,7 @@ def test_openai_embedding_3():
@pytest.mark.parametrize(
"model, api_base, api_key",
[
# ("azure/azure-embedding-model", None, None),
# ("azure/text-embedding-ada-002", None, None),
("together_ai/togethercomputer/m2-bert-80M-8k-retrieval", None, None),
],
)
@ -253,7 +253,7 @@ async def test_azure_ai_embedding_image(model, api_base, api_key, sync_mode):
def test_openai_azure_embedding_timeouts():
try:
response = embedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm"],
timeout=0.00001,
)
@ -301,7 +301,7 @@ def test_openai_azure_embedding():
os.environ["AZURE_API_KEY"] = ""
response = embedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm", "this is another item"],
api_key=api_key,
api_base=api_base,
@ -380,8 +380,16 @@ def test_openai_azure_embedding_optional_arg():
azure_ad_token="test",
)
assert mock_client.called_once_with(model="test", input=["test"], timeout=600)
mock_client.assert_called_once_with(
model="test",
input=["test"],
extra_body={"azure_ad_token": "test"},
timeout=600,
extra_headers={"X-Stainless-Raw-Response": "true"}
)
# Verify azure_ad_token is passed in extra_body, not as a direct parameter
assert "azure_ad_token" not in mock_client.call_args.kwargs
assert mock_client.call_args.kwargs["extra_body"]["azure_ad_token"] == "test"
# test_openai_azure_embedding()
@ -726,7 +734,7 @@ def test_aembedding_azure():
async def embedding_call():
try:
response = await litellm.aembedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input=["good morning from litellm", "this is another item"],
)
print(response)
@ -1099,7 +1107,7 @@ async def test_lm_studio_embedding(monkeypatch, sync_mode):
"model",
[
"text-embedding-ada-002",
"azure/azure-embedding-model",
"azure/text-embedding-ada-002",
],
)
def test_embedding_response_ratelimit_headers(model):

24
tests/local_testing/test_exceptions.py
View file

@ -51,7 +51,7 @@ async def test_content_policy_exception_azure():
# this is ony a test - we needed some way to invoke the exception :(
litellm.set_verbose = True
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "where do I buy lethal drugs from"}],
mock_response="Exception: content_filter_policy",
)
@ -124,7 +124,7 @@ def test_context_window_with_fallbacks(model):
ctx_window_fallback_dict = {
"command-nightly": "claude-2.1",
"gpt-3.5-turbo-instruct": "gpt-3.5-turbo-16k",
"azure/chatgpt-v-3": "gpt-3.5-turbo-16k",
"azure/gpt-4.1-nano": "gpt-3.5-turbo-16k",
}
sample_text = "how does a court case get to the Supreme Court?" * 1000
messages = [{"content": sample_text, "role": "user"}]
@ -161,7 +161,7 @@ def invalid_auth(model): # set the model key to an invalid key, depending on th
os.environ["AWS_REGION_NAME"] = "bad-key"
temporary_secret_key = os.environ["AWS_SECRET_ACCESS_KEY"]
os.environ["AWS_SECRET_ACCESS_KEY"] = "bad-key"
elif model == "azure/chatgpt-v-3":
elif model == "azure/gpt-4.1-nano":
temporary_key = os.environ["AZURE_API_KEY"]
os.environ["AZURE_API_KEY"] = "bad-key"
elif model == "claude-3-5-haiku-20241022":
@ -262,7 +262,7 @@ def test_completion_azure_exception():
old_azure_key = os.environ["AZURE_API_KEY"]
os.environ["AZURE_API_KEY"] = "good morning"
response = completion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "hello"}],
)
os.environ["AZURE_API_KEY"] = old_azure_key
@ -282,7 +282,7 @@ def test_azure_embedding_exceptions():
try:
response = litellm.embedding(
model="azure/azure-embedding-model",
model="azure/text-embedding-ada-002",
input="hello",
mock_response="error",
)
@ -306,7 +306,7 @@ async def asynctest_completion_azure_exception():
old_azure_key = os.environ["AZURE_API_KEY"]
os.environ["AZURE_API_KEY"] = "good morning"
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "hello"}],
)
print(f"response: {response}")
@ -525,7 +525,7 @@ def test_content_policy_violation_error_streaming():
async def test_get_response():
try:
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[{"role": "user", "content": "say 1"}],
temperature=0,
top_p=1,
@ -554,7 +554,7 @@ def test_content_policy_violation_error_streaming():
async def test_get_error():
try:
response = await litellm.acompletion(
model="azure/chatgpt-v-3",
model="azure/gpt-4.1-nano",
messages=[
{"role": "user", "content": "where do i buy lethal drugs from"}
],
@ -751,7 +751,7 @@ def test_litellm_predibase_exception():
# return False
# # Repeat each model 500 times
# # extended_models = [model for model in models for _ in range(250)]
# extended_models = ["azure/chatgpt-v-3" for _ in range(250)]
# extended_models = ["azure/gpt-4.1-nano" for _ in range(250)]
# def worker(model):
# return test_model_call(model)
@ -1023,7 +1023,7 @@ def _pre_call_utils_httpx(
("openai", "gpt-3.5-turbo", "chat_completion", False),
("openai", "gpt-3.5-turbo", "chat_completion", True),
("openai", "gpt-3.5-turbo-instruct", "completion", True),
("azure", "azure/chatgpt-v-3", "chat_completion", True),
("azure", "azure/gpt-4.1-nano", "chat_completion", True),
("azure", "azure/text-embedding-ada-002", "embedding", True),
("azure", "azure_text/gpt-3.5-turbo-instruct", "completion", True),
],
@ -1298,7 +1298,7 @@ async def test_exception_with_headers_httpx(
@pytest.mark.asyncio
@pytest.mark.parametrize("model", ["azure/chatgpt-v-3", "openai/gpt-3.5-turbo"])
@pytest.mark.parametrize("model", ["azure/gpt-4.1-nano", "openai/gpt-3.5-turbo"])
async def test_bad_request_error_contains_httpx_response(model):
"""
Test that the BadRequestError contains the httpx response
@ -1349,7 +1349,7 @@ def test_context_window_exceeded_error_from_litellm_proxy():
@pytest.mark.parametrize("sync_mode", [True, False])
@pytest.mark.parametrize("stream_mode", [True, False])
@pytest.mark.parametrize("model", ["azure/gpt-4o-new-test"]) # "gpt-4o-mini",
@pytest.mark.parametrize("model", ["gpt-4.1-nano"]) # "gpt-4o-mini",
@pytest.mark.asyncio
async def test_exception_bubbling_up(sync_mode, stream_mode, model):
"""

2
tests/local_testing/test_function_calling.py
View file

@ -48,7 +48,7 @@ def get_current_weather(location, unit="fahrenheit"):
"gpt-3.5-turbo-1106",
"mistral/mistral-large-latest",
"claude-3-haiku-20240307",
"gemini/gemini-1.5-pro",
"gemini/gemini-2.5-flash-lite",
"anthropic.claude-3-sonnet-20240229-v1:0",
"cohere_chat/command-r",
],

6
tests/local_testing/test_gcs_bucket.py
View file

@ -108,7 +108,7 @@ async def test_aaabasic_gcs_logger():
},
"endpoint": "http://localhost:4000/chat/completions",
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
"model_info": {
"id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
"db_model": False,
@ -216,7 +216,7 @@ async def test_basic_gcs_logger_failure():
},
"endpoint": "http://localhost:4000/chat/completions",
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
"model_info": {
"id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
"db_model": False,
@ -626,7 +626,7 @@ async def test_basic_gcs_logger_with_folder_in_bucket_name():
},
"endpoint": "http://localhost:4000/chat/completions",
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
"model_info": {
"id": "4bad40a1eb6bebd1682800f16f44b9f06c52a6703444c99c7f9f32e9de3693b4",
"db_model": False,

2
tests/local_testing/test_helicone_integration.py
View file

@ -78,7 +78,7 @@ async def make_async_calls(metadata=None, **completion_kwargs):
def create_async_task(**completion_kwargs):
completion_args = {
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_version": "2024-02-01",
"messages": [{"role": "user", "content": "This is a test"}],
"max_tokens": 5,

6
tests/local_testing/test_least_busy_routing.py
View file

@ -33,7 +33,7 @@ def test_model_added():
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": "1234"},
}
@ -47,7 +47,7 @@ def test_get_available_deployments():
test_cache = DualCache()
least_busy_logger = LeastBusyLoggingHandler(router_cache=test_cache, model_list=[])
model_group = "gpt-3.5-turbo"
deployment = "azure/chatgpt-v-3"
deployment = "azure/gpt-4.1-nano"
kwargs = {
"litellm_params": {
"metadata": {
@ -113,7 +113,7 @@ async def test_router_get_available_deployments(async_test):
router.leastbusy_logger.test_flag = True
model_group = "azure-model"
deployment = "azure/chatgpt-v-3"
deployment = "azure/gpt-4.1-nano"
request_count_dict = {1: 10, 2: 54, 3: 100}
cache_key = f"{model_group}_request_count"
if async_test is True:

2
tests/local_testing/test_load_test_router_s3.py
View file

@ -46,7 +46,7 @@
# {
# "model_name": "gpt-3.5-turbo",
# "litellm_params": {
# "model": "azure/chatgpt-v-3",
# "model": "azure/gpt-4.1-nano",
# "api_key": os.getenv("AZURE_API_KEY"),
# "api_base": os.getenv("AZURE_API_BASE"),
# "api_version": os.getenv("AZURE_API_VERSION"),

2
tests/local_testing/test_loadtest_router.py
View file

@ -38,7 +38,7 @@
# {
# "model_name": "gpt-3.5-turbo",
# "litellm_params": {
# "model": "azure/chatgpt-v-3",
# "model": "azure/gpt-4.1-nano",
# "api_key": os.getenv("AZURE_API_KEY"),
# "api_base": os.getenv("AZURE_API_BASE"),
# "api_version": os.getenv("AZURE_API_VERSION"),

2
tests/local_testing/test_lowest_cost_routing.py
View file

@ -60,7 +60,7 @@ async def test_get_available_deployments_custom_price():
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"input_cost_per_token": 0.00003,
"output_cost_per_token": 0.00003,
},

28
tests/local_testing/test_lowest_latency_routing.py
View file

@ -48,7 +48,7 @@ async def test_latency_memory_leak(sync_mode):
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -130,7 +130,7 @@ def test_latency_updated():
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -173,7 +173,7 @@ def test_latency_updated_custom_ttl():
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -200,12 +200,12 @@ def test_get_available_deployments():
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "1234"},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "5678"},
},
]
@ -219,7 +219,7 @@ def test_get_available_deployments():
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -240,7 +240,7 @@ def test_get_available_deployments():
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -275,7 +275,7 @@ async def _deploy(lowest_latency_logger, deployment_id, tokens_used, duration):
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -317,12 +317,12 @@ def test_get_available_endpoints_tpm_rpm_check_async(ans_rpm):
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "1234", "rpm": ans_rpm},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "5678", "rpm": non_ans_rpm},
},
]
@ -366,12 +366,12 @@ def test_get_available_endpoints_tpm_rpm_check(ans_rpm):
model_list = [
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "1234", "rpm": ans_rpm},
},
{
"model_name": "gpt-3.5-turbo",
"litellm_params": {"model": "azure/chatgpt-v-3"},
"litellm_params": {"model": "azure/gpt-4.1-nano"},
"model_info": {"id": "5678", "rpm": non_ans_rpm},
},
]
@ -385,7 +385,7 @@ def test_get_available_endpoints_tpm_rpm_check(ans_rpm):
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}
@ -407,7 +407,7 @@ def test_get_available_endpoints_tpm_rpm_check(ans_rpm):
"litellm_params": {
"metadata": {
"model_group": "gpt-3.5-turbo",
"deployment": "azure/chatgpt-v-3",
"deployment": "azure/gpt-4.1-nano",
},
"model_info": {"id": deployment_id},
}

6
tests/local_testing/test_mem_usage.py
View file

@ -29,7 +29,7 @@
# {
# "model_name": "gpt-3.5-turbo", # openai model name
# "litellm_params": { # params for litellm completion/embedding call
# "model": "azure/chatgpt-v-3",
# "model": "azure/gpt-4.1-nano",
# "api_key": os.getenv("AZURE_API_KEY"),
# "api_version": os.getenv("AZURE_API_VERSION"),
# "api_base": os.getenv("AZURE_API_BASE"),
@ -40,7 +40,7 @@
# {
# "model_name": "bad-model", # openai model name
# "litellm_params": { # params for litellm completion/embedding call
# "model": "azure/chatgpt-v-3",
# "model": "azure/gpt-4.1-nano",
# "api_key": "bad-key",
# "api_version": os.getenv("AZURE_API_VERSION"),
# "api_base": os.getenv("AZURE_API_BASE"),
@ -51,7 +51,7 @@
# {
# "model_name": "text-embedding-ada-002",
# "litellm_params": {
# "model": "azure/azure-embedding-model",
# "model": "azure/text-embedding-ada-002",
# "api_key": os.environ["AZURE_API_KEY"],
# "api_base": os.environ["AZURE_API_BASE"],
# },

13
tests/local_testing/test_mock_request.py
View file

@ -155,15 +155,14 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
},
},
{
"model_name": "azure-gpt",
"model_name": "gpt-4.1-nano",
"litellm_params": {
"model": "azure/gpt-4.1-nano",
"api_key": os.getenv("AZURE_API_KEY"),
"api_base": os.getenv("AZURE_API_BASE"),
"model": "gpt-4.1-nano",
"api_key": os.getenv("OPENAI_API_KEY"),
},
},
],
fallbacks=[{"gpt-3.5-turbo": ["azure-gpt"]}],
fallbacks=[{"gpt-3.5-turbo": ["gpt-4.1-nano"]}],
)
response = router.completion(
model="gpt-3.5-turbo",
@ -176,5 +175,5 @@ def test_router_mock_request_with_mock_timeout_with_fallbacks():
end_time = time.time()
assert end_time - start_time >= 3, f"Time taken: {end_time - start_time}"
assert (
"gpt-3.5-turbo-0125" in response.model
), "Model should be azure gpt-3.5-turbo-0125"
"gpt-4.1-nano" in response.model
), "Model should be gpt-4.1-nano"

2
tests/local_testing/test_prompt_injection_detection.py
View file

@ -107,7 +107,7 @@ async def test_prompt_injection_llm_eval():
{
"model_name": "gpt-3.5-turbo", # openai model name
"litellm_params": { # params for litellm completion/embedding call
"model": "azure/chatgpt-v-3",
"model": "azure/gpt-4.1-nano",
"api_key": os.getenv("AZURE_API_KEY"),
"api_version": os.getenv("AZURE_API_VERSION"),
"api_base": os.getenv("AZURE_API_BASE"),

Some files were not shown because too many files have changed in this diff Show more