diff --git a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md index 8a54426dfb0..21ba3d60790 100644 --- a/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md +++ b/docs/my-website/blog/anthropic_opus_4_5_and_advanced_features/index.md @@ -3,18 +3,9 @@ slug: anthropic_advanced_features title: "Day 0 Support: Claude 4.5 Opus (+Advanced Features)" date: 2025-11-25T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to Claude Opus 4.5 and advanced features in LiteLLM: Tool Search, Programmatic Tool Calling, and Effort Parameter." tags: [anthropic, claude, tool search, programmatic tool calling, effort, advanced features] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; This guide covers Anthropic's latest model (Claude Opus 4.5) and its advanced features now available in LiteLLM: Tool Search, Programmatic Tool Calling, Tool Input Examples, and the Effort Parameter. +{/* truncate */} + --- | Feature | Supported Models | diff --git a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md index f6172cd6744..8d58e18e580 100644 --- a/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md +++ b/docs/my-website/blog/anthropic_wildcard_model_access_incident/index.md @@ -3,18 +3,9 @@ slug: anthropic-wildcard-model-access-incident title: "Incident Report: Wildcard Blocking New Models After Cost Map Reload" date: 2026-02-23T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, auth, model-access] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/authors.yml b/docs/my-website/blog/authors.yml index 2a49a736333..1b1ef4d34c4 100644 --- a/docs/my-website/blog/authors.yml +++ b/docs/my-website/blog/authors.yml @@ -4,6 +4,12 @@ litellm: url: https://github.com/BerriAI/litellm image_url: https://github.com/BerriAI.png +sameer: + name: Sameer Kankute + title: SWE @ LiteLLM (LLM Translation) + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + krrish: name: Krrish Dholakia title: CEO, LiteLLM @@ -22,3 +28,21 @@ ishaan-alt: title: CTO, LiteLLM url: https://www.linkedin.com/in/reffajnaahsi/ image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + +ryan: + name: Ryan Crabbe + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 + image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + +alexsander: + name: Alexsander Hamir + title: Performance Engineer, LiteLLM + url: https://www.linkedin.com/in/alexsander-baptista/ + image_url: https://github.com/AlexsanderHamir.png + +yuneng: + name: Yuneng Jiang + title: SWE @ LiteLLM (Full Stack) + url: https://www.linkedin.com/in/yuneng-david-jiang-455676139/ + image_url: https://avatars.githubusercontent.com/u/171294688?v=4 diff --git a/docs/my-website/blog/claude_code_beta_headers/index.md b/docs/my-website/blog/claude_code_beta_headers/index.md index 44567f616aa..ee07da79397 100644 --- a/docs/my-website/blog/claude_code_beta_headers/index.md +++ b/docs/my-website/blog/claude_code_beta_headers/index.md @@ -3,18 +3,9 @@ slug: claude-code-beta-headers-incident title: "Incident Report: Invalid beta headers with Claude Code" date: 2026-02-16T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish tags: [incident-report, anthropic, stability] hide_table_of_contents: false --- @@ -173,5 +164,5 @@ curl -X POST "https://your-proxy-url/reload/anthropic_beta_headers" \ ## Related documentation -- [Managing Anthropic Beta Headers](../proxy/sync_anthropic_beta_headers.md) - Complete configuration guide +- [Managing Anthropic Beta Headers](../../docs/proxy/sync_anthropic_beta_headers) - Complete configuration guide - [`anthropic_beta_headers_config.json`](https://github.com/BerriAI/litellm/blob/main/litellm/anthropic_beta_headers_config.json) - Current configuration file diff --git a/docs/my-website/blog/claude_opus_4_6/index.md b/docs/my-website/blog/claude_opus_4_6/index.md index e44420bd570..eeb5d5ff2f8 100644 --- a/docs/my-website/blog/claude_opus_4_6/index.md +++ b/docs/my-website/blog/claude_opus_4_6/index.md @@ -3,18 +3,9 @@ slug: claude_opus_4_6 title: "Day 0 Support: Claude Opus 4.6" date: 2026-02-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - sameer + - ishaan-alt + - krrish description: "Day 0 support for Claude Opus 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, opus 4.6] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Opus 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/claude_sonnet_4_6/index.md b/docs/my-website/blog/claude_sonnet_4_6/index.md index df54fa09792..12446c82c60 100644 --- a/docs/my-website/blog/claude_sonnet_4_6/index.md +++ b/docs/my-website/blog/claude_sonnet_4_6/index.md @@ -3,14 +3,8 @@ slug: claude_sonnet_4_6 title: "Day 0 Support: Claude Sonnet 4.6" date: 2026-02-17T10:00:00 authors: - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ishaan-alt + - krrish description: "Day 0 support for Claude Sonnet 4.6 on LiteLLM AI Gateway - use across Anthropic, Azure, Vertex AI, and Bedrock." tags: [anthropic, claude, sonnet 4.6] hide_table_of_contents: false @@ -21,6 +15,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports Claude Sonnet 4.6 on Day 0. Use it across Anthropic, Azure, Vertex AI, and Bedrock through the LiteLLM AI Gateway. +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/fastapi_middleware_performance/index.mdx b/docs/my-website/blog/fastapi_middleware_performance/index.mdx index b0c5ba13634..e373326c071 100644 --- a/docs/my-website/blog/fastapi_middleware_performance/index.mdx +++ b/docs/my-website/blog/fastapi_middleware_performance/index.mdx @@ -3,18 +3,9 @@ slug: fastapi-middleware-performance title: "Your Middleware Could Be a Bottleneck" date: 2026-02-07T10:00:00 authors: - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Ryan Crabbe - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - image_url: https://media.licdn.com/dms/image/v2/D5603AQHt1t9Z4BJ6Gw/profile-displayphoto-shrink_400_400/profile-displayphoto-shrink_400_400/0/1724453682340?e=1772064000&v=beta&t=VXdmr13rsNB05wyA2F1TENOB5UuDHUZ0FCHTolNyR5M + - krrish + - ishaan-alt + - ryan description: "How we improved LiteLLM proxy latency and throughput by replacing a single middleware base class" tags: [performance, fastapi, middleware] hide_table_of_contents: false diff --git a/docs/my-website/blog/gemin_3.1/index.md b/docs/my-website/blog/gemin_3.1/index.md index b81595e4bd5..0afccb49d98 100644 --- a/docs/my-website/blog/gemin_3.1/index.md +++ b/docs/my-website/blog/gemin_3.1/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_pro title: "DAY 0 Support: Gemini 3.1 Pro on LiteLLM" date: 2026-02-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Pro on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -28,6 +19,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports `gemini-3.1-pro-preview` and all the new API changes along with it. +{/* truncate */} + ## Deploy this version @@ -67,7 +60,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Pro on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -147,4 +140,3 @@ curl -X POST http://localhost:4000/v1/chat/completions \ | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_3/index.md b/docs/my-website/blog/gemini_3/index.md index 7263acc12c9..a5b94382b6f 100644 --- a/docs/my-website/blog/gemini_3/index.md +++ b/docs/my-website/blog/gemini_3/index.md @@ -3,18 +3,9 @@ slug: gemini_3 title: "DAY 0 Support: Gemini 3 on LiteLLM" date: 2025-11-19T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Common questions and best practices for using gemini-3-pro-preview with LiteLLM Proxy and SDK." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -29,6 +20,8 @@ This guide covers common questions and best practices for using `gemini-3-pro-pr ::: +{/* truncate */} + ## Quick Start @@ -976,8 +969,7 @@ messages.append(response.choices[0].message) # ✅ Includes thought signatures ## Additional Resources -- [Gemini Provider Documentation](../gemini.md) -- [Thought Signatures Guide](../gemini.md#thought-signatures) -- [Reasoning Content Documentation](../../reasoning_content.md) -- [Function Calling Guide](../../function_calling.md) - +- [Gemini Provider Documentation](../../docs/providers/gemini) +- [Thought Signatures Guide](../../docs/providers/gemini#thought-signatures) +- [Reasoning Content Documentation](../../docs/reasoning_content) +- [Function Calling Guide](../../docs/completion/function_call) diff --git a/docs/my-website/blog/gemini_3_1_flash_lite/index.md b/docs/my-website/blog/gemini_3_1_flash_lite/index.md index 9ef4bacb2ad..0ae79e5fa67 100644 --- a/docs/my-website/blog/gemini_3_1_flash_lite/index.md +++ b/docs/my-website/blog/gemini_3_1_flash_lite/index.md @@ -3,18 +3,9 @@ slug: gemini_3_1_flash_lite_preview title: "DAY 0 Support: Gemini 3.1 Flash Lite Preview on LiteLLM" date: 2026-03-03T08:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3.1 Flash Lite Preview on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms, supernova] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3.1-flash-lite-preview` with full day 0 support! If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -150,7 +143,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3.1 Flash Lite Preview o - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses @@ -172,4 +165,4 @@ LiteLLM automatically maps OpenAI's `reasoning_effort` parameter to Gemini's `th | `medium` | `medium` | Balanced reasoning for moderate complexity | | `high` | `high` | Maximum reasoning depth, complex problems | | `disable` | `minimal` | Disable extended reasoning | -| `none` | `minimal` | No extended reasoning | \ No newline at end of file +| `none` | `minimal` | No extended reasoning | diff --git a/docs/my-website/blog/gemini_3_flash/index.md b/docs/my-website/blog/gemini_3_flash/index.md index 830c21e5f66..5e98d2136b4 100644 --- a/docs/my-website/blog/gemini_3_flash/index.md +++ b/docs/my-website/blog/gemini_3_flash/index.md @@ -3,18 +3,9 @@ slug: gemini_3_flash title: "DAY 0 Support: Gemini 3 Flash on LiteLLM" date: 2025-12-17T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Guide to using Gemini 3 Flash on LiteLLM Proxy and SDK with day 0 support." tags: [gemini, day 0 support, llms] hide_table_of_contents: false @@ -32,6 +23,8 @@ LiteLLM now supports `gemini-3-flash-preview` and all the new API changes along If you only want cost tracking, you need no change in your current Litellm version. But if you want the support for new features introduced along with it like thinking levels, you will need to use v1.80.8-stable.1 or above. ::: +{/* truncate */} + ## Deploy this version @@ -80,7 +73,7 @@ LiteLLM provides **full end-to-end support** for Gemini 3 Flash on: - ✅ `/v1/chat/completions` - OpenAI-compatible chat completions endpoint - ✅ `/v1/responses` - OpenAI Responses API endpoint (streaming and non-streaming) - ✅ [`/v1/messages`](../../docs/anthropic_unified) - Anthropic-compatible messages endpoint -- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent.md) compatible endpoint +- ✅ `/v1/generateContent` – [Google Gemini API](../../docs/generateContent) compatible endpoint All endpoints support: - Streaming and non-streaming responses - Function calling with thought signatures @@ -252,4 +245,3 @@ If using this model via vertex_ai, keep the location as global as this is the on | `high` | `high` | | `disable` | `minimal` | | `none` | `minimal` | - diff --git a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md index 8c09432e3b6..d66de1b6c79 100644 --- a/docs/my-website/blog/gemini_embedding_2_multimodal/index.md +++ b/docs/my-website/blog/gemini_embedding_2_multimodal/index.md @@ -3,10 +3,7 @@ slug: gemini_embedding_2_multimodal title: "Gemini Embedding 2 Preview: Multimodal Embeddings on LiteLLM" date: 2025-03-11T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - sameer description: "Generate embeddings from text, images, audio, video, and PDFs with gemini-embedding-2-preview on LiteLLM via Gemini API and Vertex AI." tags: [gemini, embeddings, multimodal, vertex ai] hide_table_of_contents: false @@ -19,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports **multimodal embeddings** with `gemini-embedding-2-preview`—generating a single embedding from a mix of text, images, audio, video, and PDF content. Available via both the **Gemini API** (API key) and **Vertex AI** (GCP credentials). +{/* truncate */} + ## Supported Input Types | Modality | Supported Formats | diff --git a/docs/my-website/blog/gpt_5_3_codex/index.md b/docs/my-website/blog/gpt_5_3_codex/index.md index 850586538f6..1dfab1f7ac7 100644 --- a/docs/my-website/blog/gpt_5_3_codex/index.md +++ b/docs/my-website/blog/gpt_5_3_codex/index.md @@ -3,18 +3,9 @@ slug: gpt_5_3_codex title: "Day 0 Support: GPT-5.3-Codex" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for GPT-5.3-Codex on LiteLLM, including phase parameter handling for Responses API." tags: [openai, gpt-5.3-codex, codex, day 0 support] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports GPT-5.3-Codex on Day 0, including support for the new assistant `phase` metadata on Responses API output items. +{/* truncate */} + ## Why `phase` matters for GPT-5.3-Codex `phase` appears on assistant output items and helps distinguish preamble/commentary turns from final closeout responses. diff --git a/docs/my-website/blog/gpt_5_4/index.md b/docs/my-website/blog/gpt_5_4/index.md index de099736f00..4f7e4344157 100644 --- a/docs/my-website/blog/gpt_5_4/index.md +++ b/docs/my-website/blog/gpt_5_4/index.md @@ -3,18 +3,9 @@ slug: gpt_5_4 title: "Day 0 Support: GPT-5.4" date: 2026-03-05T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "GPT-5.4 model support in LiteLLM" tags: [openai, gpt-5.4, completion] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports fully GPT-5.4! +{/* truncate */} + ## Docker Image ```bash diff --git a/docs/my-website/blog/httpx_cache_eviction_incident/index.md b/docs/my-website/blog/httpx_cache_eviction_incident/index.md index 9e6152d0e63..7fc3789a91d 100644 --- a/docs/my-website/blog/httpx_cache_eviction_incident/index.md +++ b/docs/my-website/blog/httpx_cache_eviction_incident/index.md @@ -3,17 +3,9 @@ slug: httpx-cache-eviction-incident title: "Incident Report: Cache Eviction Closes In-Use httpx Clients" date: 2026-02-27T10:00:00 authors: - - name: Ryan Crabbe - title: Performance Engineer, LiteLLM - url: https://www.linkedin.com/in/ryan-crabbe-0b9687214 - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - ryan + - ishaan-alt + - krrish tags: [incident-report, caching, stability] hide_table_of_contents: false --- @@ -31,6 +23,8 @@ A change to improve Redis connection pool cleanup introduced a regression that c **Impact:** Any proxy instance that hit the cache TTL (default 10 minutes) or capacity limit (200 entries) would have its httpx clients closed out from under it, causing requests to LLM providers to fail with connection errors. +{/* truncate */} + --- ## Background diff --git a/docs/my-website/blog/litellm_observatory/index.md b/docs/my-website/blog/litellm_observatory/index.md index 4554f77fb85..36366e5de22 100644 --- a/docs/my-website/blog/litellm_observatory/index.md +++ b/docs/my-website/blog/litellm_observatory/index.md @@ -3,18 +3,9 @@ slug: litellm-observatory title: "Improve release stability with 24 hour load tests" date: 2026-02-06T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "How we built a long-running, release-validation system to catch regressions before they reach users." tags: [testing, observability, reliability, releases] hide_table_of_contents: false @@ -28,6 +19,8 @@ As LiteLLM adoption has grown, so have expectations around reliability, performa This post introduces **LiteLLM Observatory**, a long-running release-validation system we built to catch regressions before they reach users. +{/* truncate */} + --- ## Why We Built the Observatory @@ -133,4 +126,3 @@ Reliability is an ongoing investment. LiteLLM Observatory is one of several systems we’re building to continuously raise the bar on release quality and operational safety. As LiteLLM evolves, so will our validation tooling, informed by real-world usage and lessons learned. We’ll continue to share those improvements openly as we go. - diff --git a/docs/my-website/blog/minimax_m2_5/index.md b/docs/my-website/blog/minimax_m2_5/index.md index 50084fcc1e5..9bccbdf7979 100644 --- a/docs/my-website/blog/minimax_m2_5/index.md +++ b/docs/my-website/blog/minimax_m2_5/index.md @@ -3,18 +3,9 @@ slug: minimax_m2_5 title: "Day 0 Support: MiniMax-M2.5" date: 2026-02-12T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Day 0 support for MiniMax-M2.5 on LiteLLM" tags: [minimax, M2.5, llm] hide_table_of_contents: false @@ -25,6 +16,8 @@ import TabItem from '@theme/TabItem'; LiteLLM now supports MiniMax-M2.5 on Day 0. Use it across OpenAI-compatible and Anthropic-compatible APIs through the LiteLLM AI Gateway. +{/* truncate */} + ## Supported Models LiteLLM supports the following MiniMax models: diff --git a/docs/my-website/blog/model_cost_map_incident/index.md b/docs/my-website/blog/model_cost_map_incident/index.md index b9ff20e4128..5b4499cc31c 100644 --- a/docs/my-website/blog/model_cost_map_incident/index.md +++ b/docs/my-website/blog/model_cost_map_incident/index.md @@ -3,10 +3,7 @@ slug: model-cost-map-incident title: "Incident Report: Invalid model cost map on main" date: 2026-02-10T10:00:00 authors: - - name: Ishaan Jaffer - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/ishaanjaffer/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - ishaan tags: [incident-report, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md index 04c3d3c9097..70fc5b2c48e 100644 --- a/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md +++ b/docs/my-website/blog/realtime_webrtc_http_endpoints/index.md @@ -3,18 +3,9 @@ slug: realtime_webrtc_http_endpoints title: "Realtime WebRTC HTTP Endpoints" date: 2026-03-12T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange." tags: [realtime, webrtc, proxy, openai] hide_table_of_contents: false @@ -24,6 +15,8 @@ import WebRTCTester from '@site/src/components/WebRTCTester'; Connect to the Realtime API via WebRTC from browser/mobile clients. LiteLLM handles auth and key management. +{/* truncate */} + ## How it works ![WebRTC flow: Browser, LiteLLM Proxy, and OpenAI/Azure](../../img/webrtc_flow.png) @@ -116,4 +109,3 @@ A: Set the correct `api_version` in `litellm_params` (or via the `AZURE_API_VERS **Q: What if I get no audio?** A: Make sure you grant microphone permission, ensure `pc.ontrack` assigns the audio element with `autoplay` enabled, check your network/firewall for WebRTC traffic, and inspect the browser console for ICE or SDP errors. - diff --git a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md index 19b55898caa..fd5a9e76c42 100644 --- a/docs/my-website/blog/responses_api_encrypted_content_incident/index.md +++ b/docs/my-website/blog/responses_api_encrypted_content_incident/index.md @@ -3,18 +3,9 @@ slug: responses-api-encrypted-content-incident title: "Incident Report: Encrypted Content Failures in Multi-Region Responses API Load Balancing" date: 2026-02-24T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, proxy, responses-api, load-balancing] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/server_root_path/index.md b/docs/my-website/blog/server_root_path/index.md index d7925baf6b4..13b7365cc9e 100644 --- a/docs/my-website/blog/server_root_path/index.md +++ b/docs/my-website/blog/server_root_path/index.md @@ -3,17 +3,9 @@ slug: server-root-path-incident title: "Incident Report: SERVER_ROOT_PATH regression broke UI routing" date: 2026-02-21T10:00:00 authors: - - name: Yuneng Jiang - title: SWE @ LiteLLM (Full Stack) - url: https://www.linkedin.com/in/yunengjiang/ - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - yuneng + - ishaan-alt + - krrish tags: [incident-report, ui, stability] hide_table_of_contents: false --- diff --git a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md index 1857383363c..7f8ac086a21 100644 --- a/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md +++ b/docs/my-website/blog/sub_millisecond_proxy_overhead/index.md @@ -3,18 +3,9 @@ slug: sub-millisecond-proxy-overhead title: "Achieving Sub-Millisecond Proxy Overhead" date: 2026-02-02T10:00:00 authors: - - name: Alexsander Hamir - title: "Performance Engineer, LiteLLM" - url: https://www.linkedin.com/in/alexsander-baptista/ - image_url: https://github.com/AlexsanderHamir.png - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - alexsander + - krrish + - ishaan-alt description: "Our Q1 performance target and architectural direction for achieving sub-millisecond proxy overhead on modest hardware." tags: [performance, architecture] hide_table_of_contents: false @@ -32,6 +23,8 @@ Proxy overhead refers to the latency introduced by LiteLLM itself, independent o To measure it, we run the same workload directly against the provider and through LiteLLM at identical QPS (for example, 1,000 QPS) and compare the latency delta. To reduce noise, the load generator, LiteLLM, and a mock LLM endpoint all run on the same machine, ensuring the difference reflects proxy overhead rather than network latency. +{/* truncate */} + --- ## Where We're Coming From diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md index 263a17d7191..a0f87385d81 100644 --- a/docs/my-website/blog/video_characters_litellm/index.md +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -3,18 +3,9 @@ slug: video_characters_api title: "New Video Characters, Edit and Extension API support" date: 2026-03-16T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations." tags: [videos, characters, proxy, routing] hide_table_of_contents: false @@ -22,6 +13,8 @@ hide_table_of_contents: false LiteLLM now supoports videos character, edit and extension apis. +{/* truncate */} + ## What's New Four new endpoints for video character operations: @@ -125,4 +118,4 @@ Router knows exactly which deployment to use **Behind the scenes:** - Character ID format: `character_` - Metadata includes: provider, model_id, original_character_id -- Transparent to you - just use the ID, LiteLLM handles routing \ No newline at end of file +- Transparent to you - just use the ID, LiteLLM handles routing diff --git a/docs/my-website/blog/vllm_embeddings_incident/index.md b/docs/my-website/blog/vllm_embeddings_incident/index.md index a1ce8152857..26387e66dd0 100644 --- a/docs/my-website/blog/vllm_embeddings_incident/index.md +++ b/docs/my-website/blog/vllm_embeddings_incident/index.md @@ -3,18 +3,9 @@ slug: vllm-embeddings-incident title: "Incident Report: vLLM Embeddings Broken by encoding_format Parameter" date: 2026-02-18T10:00:00 authors: - - name: Sameer Kankute - title: SWE @ LiteLLM (LLM Translation) - url: https://www.linkedin.com/in/sameer-kankute/ - image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg - - name: Krrish Dholakia - title: "CEO, LiteLLM" - url: https://www.linkedin.com/in/krish-d/ - image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg - - name: Ishaan Jaff - title: "CTO, LiteLLM" - url: https://www.linkedin.com/in/reffajnaahsi/ - image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg + - sameer + - krrish + - ishaan-alt tags: [incident-report, embeddings, vllm] hide_table_of_contents: false --- diff --git a/docs/my-website/docs/guides/index.md b/docs/my-website/docs/guides/index.md new file mode 100644 index 00000000000..1641600dab2 --- /dev/null +++ b/docs/my-website/docs/guides/index.md @@ -0,0 +1,78 @@ +--- +title: Guides +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Guides** are focused references organized by the job you are trying to do with LiteLLM: make requests, use tools, handle media, manage context, or operate the gateway safely. + +> New to LiteLLM or not sure whether you need the SDK or Gateway path first? Start at [Learn →](/docs/learn) + +--- + +## Build With LiteLLM + + + +--- + +## Operate & Extend + + diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index ba605e316d3..ca63c9e39ff 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -1,689 +1,462 @@ +--- +id: index +title: Getting Started +sidebar_label: Quickstart +--- + import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import NavigationCards from '@site/src/components/NavigationCards'; +import Image from '@theme/IdealImage'; -# LiteLLM - Getting Started + -https://github.com/BerriAI/litellm +**LiteLLM** is an open-source library that gives you a single, unified interface to call 100+ LLMs — OpenAI, Anthropic, Vertex AI, Bedrock, and more — using the OpenAI format. -## **Call 100+ LLMs using the OpenAI Input/Output Format** +- Call any provider using the same `completion()` interface — no re-learning the API for each one +- Consistent output format regardless of which provider or model you use +- Built-in retry / fallback logic across multiple deployments via the [Router](./routing.md) +- Self-hosted [LLM Gateway (Proxy)](./simple_proxy) with virtual keys, cost tracking, and an admin UI -- Translate inputs to provider's endpoints (`/chat/completions`, `/responses`, `/embeddings`, `/images`, `/audio`, `/batches`, and more) -- [Consistent output](https://docs.litellm.ai/docs/supported_endpoints) - same response format regardless of which provider you use -- Retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - [Router](https://docs.litellm.ai/docs/routing) -- Track spend & set budgets per project [LiteLLM Proxy Server](https://docs.litellm.ai/docs/simple_proxy) +[![PyPI](https://img.shields.io/pypi/v/litellm.svg)](https://pypi.org/project/litellm/) +[![GitHub Stars](https://img.shields.io/github/stars/BerriAI/litellm?style=social)](https://github.com/BerriAI/litellm) -## How to use LiteLLM +--- -You can use LiteLLM through either the Proxy Server or Python SDK. Both gives you a unified interface to access multiple LLMs (100+ LLMs). Choose the option that best fits your needs: - - - - - - - - - - - - - - - - - - - - - - - - - - -
LiteLLM Proxy ServerLiteLLM Python SDK
Use CaseCentral service (LLM Gateway) to access multiple LLMsUse LiteLLM directly in your Python code
Who Uses It?Gen AI Enablement / ML Platform TeamsDevelopers building LLM projects
Key Features• Centralized API gateway with authentication & authorization
• Multi-tenant cost tracking and spend management per project/user
• Per-project customization (logging, guardrails, caching)
• Virtual keys for secure access control
• Admin dashboard UI for monitoring and management		• Direct Python library integration in your codebase
• Router with retry/fallback logic across multiple deployments (e.g. Azure/OpenAI) - Router
• Application-level load balancing and cost tracking
• Exception handling with OpenAI-compatible errors
• Observability callbacks (Lunary, MLflow, Langfuse, etc.)
- - -## **LiteLLM Python SDK** - -### Basic usage - - - Open In Colab - +## Installation ```shell pip install litellm ``` - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEXAI_PROJECT"] = "hardy-device-386718" -os.environ["VERTEXAI_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud" -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434" -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita-api-key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}] -) -``` - - - - - -### Response Format (OpenAI Chat Completions Format) - -```json -{ - "id": "chatcmpl-565d891b-a42e-4c39-8d14-82a1f5208885", - "created": 1734366691, - "model": "gpt-4o-2024-08-06", - "object": "chat.completion", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": "stop", - "index": 0, - "message": { - "content": "Hello! As an AI language model, I don't have feelings, but I'm operating properly and ready to assist you with any questions or tasks you may have. How can I help you today?", - "role": "assistant", - "tool_calls": null, - "function_call": null - } - } - ], - "usage": { - "completion_tokens": 43, - "prompt_tokens": 13, - "total_tokens": 56, - "completion_tokens_details": null, - "prompt_tokens_details": { - "audio_tokens": null, - "cached_tokens": 0 - }, - "cache_creation_input_tokens": 0, - "cache_read_input_tokens": 0 - } -} -``` - -### Streaming -Set `stream=True` in the `completion` args. - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" - -response = completion( - model="openai/gpt-4o", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["ANTHROPIC_API_KEY"] = "your-api-key" - -response = completion( - model="anthropic/claude-3-sonnet-20240229", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["XAI_API_KEY"] = "your-api-key" - -response = completion( - model="xai/grok-2-latest", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - -```python -from litellm import completion -import os - -# auth: run 'gcloud auth application-default' -os.environ["VERTEX_PROJECT"] = "hardy-device-386718" -os.environ["VERTEX_LOCATION"] = "us-central1" - -response = completion( - model="vertex_ai/gemini-1.5-pro", - messages=[{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["NVIDIA_NIM_API_KEY"] = "nvidia_api_key" -os.environ["NVIDIA_NIM_API_BASE"] = "nvidia_nim_endpoint_url" - -response = completion( - model="nvidia_nim/", - messages=[{ "content": "Hello, how are you?","role": "user"}] - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -os.environ["HUGGINGFACE_API_KEY"] = "huggingface_api_key" - -# e.g. Call 'WizardLM/WizardCoder-Python-34B-V1.0' hosted on HF Inference endpoints -response = completion( - model="huggingface/WizardLM/WizardCoder-Python-34B-V1.0", - messages=[{ "content": "Hello, how are you?","role": "user"}], - api_base="https://my-endpoint.huggingface.cloud", - stream=True, -) - -print(response) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["AZURE_API_KEY"] = "" -os.environ["AZURE_API_BASE"] = "" -os.environ["AZURE_API_VERSION"] = "" - -# azure call -response = completion( - "azure/", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion - -response = completion( - model="ollama/llama2", - messages = [{ "content": "Hello, how are you?","role": "user"}], - api_base="http://localhost:11434", - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables -os.environ["OPENROUTER_API_KEY"] = "openrouter_api_key" - -response = completion( - model="openrouter/google/palm-2-chat-bison", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://novita.ai/settings/key-management to get your API key -os.environ["NOVITA_API_KEY"] = "novita_api_key" - -response = completion( - model="novita/deepseek/deepseek-r1", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -```python -from litellm import completion -import os - -## set ENV variables. Visit https://vercel.com/docs/ai-gateway#using-the-ai-gateway-with-an-api-key for insturctions on obtaining a key -os.environ["VERCEL_AI_GATEWAY_API_KEY"] = "your-vercel-api-key" - -response = completion( - model="vercel_ai_gateway/openai/gpt-4o", - messages = [{ "content": "Hello, how are you?","role": "user"}], - stream=True, -) -``` - - - - - -### Streaming Response Format (OpenAI Format) - -```json -{ - "id": "chatcmpl-2be06597-eb60-4c70-9ec5-8cd2ab1b4697", - "created": 1734366925, - "model": "claude-3-sonnet-20240229", - "object": "chat.completion.chunk", - "system_fingerprint": null, - "choices": [ - { - "finish_reason": null, - "index": 0, - "delta": { - "content": "Hello", - "role": "assistant", - "function_call": null, - "tool_calls": null, - "audio": null - }, - "logprobs": null - } - ] -} -``` - -### Exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -import litellm -from litellm import completion -import os - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - completion(model="anthropic/claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except litellm.AuthenticationError as e: - # Thrown when the API key is invalid - print(f"Authentication failed: {e}") -except litellm.RateLimitError as e: - # Thrown when you've exceeded your rate limit - print(f"Rate limited: {e}") -except litellm.APIError as e: - # Thrown for general API errors - print(f"API error: {e}") -``` -### See How LiteLLM Transforms Your Requests - -Want to understand how LiteLLM parses and normalizes your LLM API requests? Use the `/utils/transform_request` endpoint to see exactly how your request is transformed internally. - -You can try it out now directly on our Demo App! -Go to the [LiteLLM API docs for transform_request](https://litellm-api.up.railway.app/#/llm%20utils/transform_request_utils_transform_request_post) - -LiteLLM will show you the normalized, provider-agnostic version of your request. This is useful for debugging, learning, and understanding how LiteLLM handles different providers and options. - - -### Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) -LiteLLM exposes pre defined callbacks to send data to Lunary, MLflow, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to lunary, mlflow, langfuse, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -### Track Costs, Usage, Latency for streaming -Use a callback function for this - more info on custom callbacks: https://docs.litellm.ai/docs/observability/custom_callback - -```python -import litellm - -# track_cost_callback -def track_cost_callback( - kwargs, # kwargs to completion - completion_response, # response from completion - start_time, end_time # start/end time -): - try: - response_cost = kwargs.get("response_cost", 0) - print("streaming response_cost", response_cost) - except: - pass -# set callback -litellm.success_callback = [track_cost_callback] # set custom callback function - -# litellm.completion() call -response = completion( - model="gpt-3.5-turbo", - messages=[ - { - "role": "user", - "content": "Hi 👋 - i'm openai" - } - ], - stream=True -) -``` - -## **LiteLLM Proxy Server (LLM Gateway)** - -Track spend across multiple projects/people - -![ui_3](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) - -The proxy provides: - -1. [Hooks for auth](https://docs.litellm.ai/docs/proxy/virtual_keys#custom-auth) -2. [Hooks for logging](https://docs.litellm.ai/docs/proxy/logging#step-1---create-your-custom-litellm-callback-class) -3. [Cost tracking](https://docs.litellm.ai/docs/proxy/virtual_keys#tracking-spend) -4. [Rate Limiting](https://docs.litellm.ai/docs/proxy/users#set-rate-limits) - -### 📖 Proxy Endpoints - [Swagger Docs](https://litellm-api.up.railway.app/) - -Go here for a complete tutorial with keys + rate limits - [**here**](./proxy/docker_quick_start.md) - -### Quick Start Proxy - CLI +To run the full Proxy Server (LLM Gateway): ```shell pip install 'litellm[proxy]' ``` -#### Step 1: Start litellm proxy +--- + +## Quick Start + +Make your first LLM call using the provider of your choice: + - +```python +from litellm import completion +import os -```shell -$ litellm --model huggingface/bigcode/starcoder +os.environ["OPENAI_API_KEY"] = "your-api-key" -#INFO: Proxy running on http://0.0.0.0:4000 +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) ``` + - +```python +from litellm import completion +import os +os.environ["ANTHROPIC_API_KEY"] = "your-api-key" -Step 1. CREATE config.yaml +response = completion( + model="anthropic/claude-3-5-sonnet-20241022", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` -Example `litellm_config.yaml` + + -```yaml +```python +from litellm import completion +import os + +# auth: run 'gcloud auth application-default login' +os.environ["VERTEXAI_PROJECT"] = "your-project-id" +os.environ["VERTEXAI_LOCATION"] = "us-central1" + +response = completion( + model="vertex_ai/gemini-1.5-pro", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AWS_ACCESS_KEY_ID"] = "your-key" +os.environ["AWS_SECRET_ACCESS_KEY"] = "your-secret" +os.environ["AWS_REGION_NAME"] = "us-east-1" + +response = completion( + model="bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion + +response = completion( + model="ollama/llama3", + messages=[{"role": "user", "content": "Hello, how are you?"}], + api_base="http://localhost:11434" +) +print(response.choices[0].message.content) +``` + + + + +```python +from litellm import completion +import os + +os.environ["AZURE_API_KEY"] = "your-key" +os.environ["AZURE_API_BASE"] = "https://your-resource.openai.azure.com" +os.environ["AZURE_API_VERSION"] = "2024-02-01" + +response = completion( + model="azure/your-deployment-name", + messages=[{"role": "user", "content": "Hello, how are you?"}] +) +print(response.choices[0].message.content) +``` + + + + +Every response follows the OpenAI Chat Completions format, regardless of provider. ✅ + +### Response Format + +Non-streaming responses return a `ModelResponse` object: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +Streaming responses (`stream=True`) yield `ModelResponseStream` chunks: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion.chunk", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "delta": { + "role": "assistant", + "content": "Hello" + }, + "finish_reason": null + } + ] +} +``` + +📖 [Full output format reference →](./completion/output) + +:::tip Open in Colab + +Open In Colab + +::: + +--- + +## New to LiteLLM? + +**Want to get started fast?** Head to [Tutorials](/docs/tutorials) for step-by-step walkthroughs — AI coding tools, agent SDKs, proxy setup, and more. + +**Need to understand a specific feature?** Check [Guides](/docs/guides) for streaming, function calling, prompt caching, and other how-tos. + +--- + +## Choose Your Path + + + +--- + +## LiteLLM Python SDK + +### Streaming + +Add `stream=True` to receive chunks as they are generated: + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +for chunk in completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Write a short poem"}], + stream=True, +): + print(chunk.choices[0].delta.content or "", end="") +``` + +### Exception Handling + +LiteLLM maps every provider's errors to the OpenAI exception types — your existing error handling works out of the box: + +```python +import litellm + +try: + litellm.completion( + model="anthropic/claude-instant-1", + messages=[{"role": "user", "content": "Hey!"}] + ) +except litellm.AuthenticationError as e: + print(f"Bad API key: {e}") +except litellm.RateLimitError as e: + print(f"Rate limited: {e}") +except litellm.APIError as e: + print(f"API error: {e}") +``` + +### Logging & Observability + +Send input/output to Langfuse, MLflow, Helicone, Lunary, and more with a single line: + +```python +import litellm + +litellm.success_callback = ["langfuse", "mlflow", "helicone"] + +response = litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hi!"}] +) +``` + +📖 [See all observability integrations →](/docs/observability/agentops_integration) + +### Track Costs & Usage + +Use a callback to capture cost per response: + +```python +import litellm + +def track_cost(kwargs, completion_response, start_time, end_time): + print("Cost:", kwargs.get("response_cost", 0)) + +litellm.success_callback = [track_cost] + +litellm.completion( + model="gpt-4o", + messages=[{"role": "user", "content": "Hello!"}], + stream=True +) +``` + +📖 [Custom callback docs →](./observability/custom_callback) + +--- + +## LiteLLM Proxy Server (LLM Gateway) + +The proxy is a self-hosted OpenAI-compatible gateway. Any client that works with OpenAI works with the proxy — no code changes needed. + +![LiteLLM Proxy Dashboard](https://github.com/BerriAI/litellm/assets/29436595/47c97d5e-b9be-4839-b28c-43d7f4f10033) + +#### Step 1 — Start the proxy + + + + +```shell +litellm --model huggingface/bigcode/starcoder +# Proxy running on http://0.0.0.0:4000 +``` + + + + +```yaml title="litellm_config.yaml" model_list: - model_name: gpt-3.5-turbo litellm_params: - model: azure/ - api_base: os.environ/AZURE_API_BASE # runs os.getenv("AZURE_API_BASE") - api_key: os.environ/AZURE_API_KEY # runs os.getenv("AZURE_API_KEY") + model: azure/your-deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY api_version: "2023-07-01-preview" ``` -Step 2. RUN Docker Image - ```shell docker run \ - -v $(pwd)/litellm_config.yaml:/app/config.yaml \ - -e AZURE_API_KEY=d6*********** \ - -e AZURE_API_BASE=https://openai-***********/ \ - -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ - --config /app/config.yaml --detailed_debug + -v $(pwd)/litellm_config.yaml:/app/config.yaml \ + -e AZURE_API_KEY=your-key \ + -e AZURE_API_BASE=https://your-resource.openai.azure.com/ \ + -p 4000:4000 \ + docker.litellm.ai/berriai/litellm:main-latest \ + --config /app/config.yaml --detailed_debug ``` - -#### Step 2: Make ChatCompletions Request to Proxy +#### Step 2 — Call it with the OpenAI client ```python -import openai # openai v1.0.0+ -client = openai.OpenAI(api_key="anything",base_url="http://0.0.0.0:4000") # set proxy to base_url -# request sent to model set on litellm proxy, `litellm --model` -response = client.chat.completions.create(model="gpt-3.5-turbo", messages = [ - { - "role": "user", - "content": "this is a test request, write a short poem" - } -]) +import openai -print(response) +client = openai.OpenAI(api_key="anything", base_url="http://0.0.0.0:4000") + +response = client.chat.completions.create( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Write a short poem"}] +) +print(response.choices[0].message.content) ``` -## More details +👉 [Full proxy quickstart with Docker →](./proxy/docker_quick_start) -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [proxy virtual keys & spend management](./proxy/virtual_keys.md) -- [E2E Tutorial for LiteLLM Proxy Server](./proxy/docker_quick_start.md) +:::tip Debugging tool +Use [**`/utils/transform_request`**](./utils/transform_request) to inspect exactly what LiteLLM sends to any provider — useful for debugging prompt formatting, header issues, and provider-specific parameters. +::: + +🔗 [Interactive API explorer (Swagger) →](https://litellm-api.up.railway.app/) + +--- + +## Agent & MCP Gateway + +LiteLLM is a unified gateway for **LLMs, agents, and MCP** — you don't need a separate agent or MCP gateway. One endpoint for 100+ models, A2A agents, and MCP tools. + + + +--- + +## What to Explore Next + + diff --git a/docs/my-website/docs/integrations/index.md b/docs/my-website/docs/integrations/index.md index 95c922cce89..0ad934d5b41 100644 --- a/docs/my-website/docs/integrations/index.md +++ b/docs/my-website/docs/integrations/index.md @@ -1,18 +1,336 @@ -# Integrations +--- +title: Integrations +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; This section covers integrations with various tools and services that can be used with LiteLLM (either Proxy or SDK). -## AI Agent Frameworks -- **[Letta](./letta.md)** - Build stateful LLM agents with persistent memory using LiteLLM Proxy +--- -## Development Tools -- **[OpenWebUI](../tutorials/openweb_ui.md)** - Self-hosted ChatGPT-style interface +## Observability -## Observability & Monitoring -- **[Langfuse](../observability/langfuse_integration.md)** - LLM observability and analytics -- **[Prometheus](../proxy/prometheus.md)** - Metrics collection and monitoring -- **[PagerDuty](../proxy/pagerduty.md)** - Incident response and alerting -- **[Datadog](../observability/datadog.md)** +Track, debug, and analyze LLM calls with observability platforms. + -Click into each section to learn more about the integrations. \ No newline at end of file +[View all observability integrations →](/docs/integrations/observability_integrations) + +--- + +## Alerting & Monitoring + +Set up alerts, metrics collection, and infrastructure monitoring. + + + +--- + +## Guardrail Providers + +Add safety and content filtering to LLM calls. + + + +[View all guardrail providers →](/docs/guardrail_providers) + +--- + +## Policies + +Define and enforce usage policies across your LLM deployment. + + + +--- + +## AI Tools + +Connect LiteLLM to AI-powered coding and productivity tools. + + + +--- + +## Agent SDKs + +Use LiteLLM with agent frameworks and SDKs. + + + +--- + +## Prompt Management + +Manage, version, and deploy prompts. + + + +--- + +## Manage with AI Agents + +Use AI agents to manage your LiteLLM deployment — create users, teams, keys, models, and more via natural language. + + diff --git a/docs/my-website/docs/integrations/letta.md b/docs/my-website/docs/integrations/letta.md index 2afb82542f2..9711999df5e 100644 --- a/docs/my-website/docs/integrations/letta.md +++ b/docs/my-website/docs/integrations/letta.md @@ -920,9 +920,9 @@ for model in models: ## Resources -- [Letta Documentation](https://docs.letta.ai/) -- [LiteLLM Proxy Documentation](../proxy/quick_start.md) -- [LiteLLM SDK Documentation](../completion/input.md) -- [Function Calling Guide](../completion/function_call.md) -- [Observability Setup](../observability/langfuse_integration.md) -- [Router Configuration](../routing.md) \ No newline at end of file +- [Letta Documentation](https://docs.letta.com/) +- [LiteLLM Proxy Documentation](/docs/simple_proxy) +- [LiteLLM SDK Documentation](/docs/#litellm-python-sdk) +- [Function Calling Guide](/docs/completion/function_call) +- [Observability Setup](/docs/integrations/observability_integrations) +- [Router Configuration](/docs/routing) \ No newline at end of file diff --git a/docs/my-website/docs/integrations/observability_index.md b/docs/my-website/docs/integrations/observability_index.md new file mode 100644 index 00000000000..8ab83950cdc --- /dev/null +++ b/docs/my-website/docs/integrations/observability_index.md @@ -0,0 +1,28 @@ +--- +title: Observability +sidebar_label: Overview +slug: observability_integrations +--- + +Track, debug, and analyze LLM calls with observability platforms. + +import NavigationCards from '@site/src/components/NavigationCards'; + +## Observability Integrations + + + +[View all observability integrations →](/docs/observability/callbacks) diff --git a/docs/my-website/docs/integrations/websearch_interception.md b/docs/my-website/docs/integrations/websearch_interception.md index 0c5d8927013..bc5e8ec0b39 100644 --- a/docs/my-website/docs/integrations/websearch_interception.md +++ b/docs/my-website/docs/integrations/websearch_interception.md @@ -375,7 +375,7 @@ search_tools: - [Search Providers](../search/index.md) - Detailed search provider setup - [Claude Code WebSearch](../tutorials/claude_code_websearch.md) - Using with Claude Code - [Tool Calling](../completion/function_call.md) - General tool calling documentation -- [Callbacks](./custom_callback.md) - Custom callback documentation +- [Callbacks](../observability/custom_callback.md) - Custom callback documentation ## Technical Details diff --git a/docs/my-website/docs/learn/gateway_quickstart.md b/docs/my-website/docs/learn/gateway_quickstart.md new file mode 100644 index 00000000000..acec259758c --- /dev/null +++ b/docs/my-website/docs/learn/gateway_quickstart.md @@ -0,0 +1,174 @@ +--- +title: Gateway Quickstart +sidebar_label: Gateway Quickstart +description: Start LiteLLM Gateway, add models and keys, then connect applications and SDKs to one shared endpoint. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you need one shared OpenAI-compatible endpoint for a team or platform. + +If you need a Docker or database-first setup, use the [Docker + Database tutorial](/docs/proxy/docker_quick_start). Otherwise, use the steps below to get to a working request fast. + +## 1. Install The Gateway + +```bash +pip install 'litellm[proxy]' +``` + +## 2. Set One Provider Key + +```bash +export OPENAI_API_KEY="your-api-key" +``` + +## 3. Create `config.yaml` + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: openai/gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +general_settings: + master_key: sk-1234 +``` + +## 4. Start The Gateway + +```bash +litellm --config config.yaml +``` + +You should see the proxy start on `http://0.0.0.0:4000`. + +## 5. Send Your First Request + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o-mini", + "messages": [ + {"role": "user", "content": "Hello from LiteLLM Gateway"} + ] + }' +``` + +## 6. Check The Response + +If the request succeeds, the proxy returns `200 OK` with an OpenAI-style response. + +The assistant text will be in: + +```json +choices[0].message.content +``` + +If your gateway is routing to OpenAI, a real response can look like this: + +```json +{ + "id": "chatcmpl-abc123", + "created": 1677858242, + "model": "gpt-4o-mini-2024-07-18", + "object": "chat.completion", + "system_fingerprint": "fp_406d6473f8", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! How can I assist you today?", + "tool_calls": null, + "function_call": null, + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 13, + "total_tokens": 22, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +`id`, `created`, the resolved model version, token counts, and message text will vary by request. Other providers may return a smaller or slightly different set of fields, but `choices[0].message.content` is the main field to read. + +## 7. Add Keys And The UI + +If you need virtual keys, spend tracking, or the admin UI, add a database next. + +- Add `database_url` under `general_settings` +- Use [Virtual keys](/docs/proxy/virtual_keys) for key creation and budgets +- Use [Admin UI](/docs/proxy/ui) to manage models and keys +- Use the [Docker + Database tutorial](/docs/proxy/docker_quick_start) if you want a fuller setup + +## 8. Pick Your Next Step + + + +## When To Use The SDK Path Instead + +If you only need to call models from one application and do not need centralized auth or shared infrastructure, start with the [SDK Quickstart](/docs/learn/sdk_quickstart) instead. diff --git a/docs/my-website/docs/learn/index.md b/docs/my-website/docs/learn/index.md new file mode 100644 index 00000000000..018aec5af00 --- /dev/null +++ b/docs/my-website/docs/learn/index.md @@ -0,0 +1,117 @@ +--- +title: Learn LiteLLM +sidebar_label: Learn +slug: /learn +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +LiteLLM gives you one OpenAI-compatible interface for 100+ LLM providers. Start with the path that matches your setup. + +--- + +## Start Here + +Pick one path first. + + + +--- + +## Common Tasks + +Jump to a specific task. + + + +--- + +## Docs Map + +Use these when you already know the type of doc you want. + + + +Not sure where to start? Use [SDK Quickstart](/docs/learn/sdk_quickstart) for app code or [Gateway Quickstart](/docs/learn/gateway_quickstart) for shared infrastructure. diff --git a/docs/my-website/docs/learn/sdk_quickstart.md b/docs/my-website/docs/learn/sdk_quickstart.md new file mode 100644 index 00000000000..bdf7b63eb5d --- /dev/null +++ b/docs/my-website/docs/learn/sdk_quickstart.md @@ -0,0 +1,174 @@ +--- +title: SDK Quickstart +sidebar_label: SDK Quickstart +description: Make your first LiteLLM SDK call, then jump to the right docs for the next feature you need. +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +Use this path if you are integrating LiteLLM directly into application code. + +## 1. Install LiteLLM + +```bash +pip install litellm +``` + +## 2. Set Provider Credentials + +Start with one provider and set its environment variables. + +- OpenAI: `OPENAI_API_KEY` +- Anthropic: `ANTHROPIC_API_KEY` +- Azure OpenAI: `AZURE_API_KEY`, `AZURE_API_BASE`, `AZURE_API_VERSION` +- Bedrock: standard AWS credentials +- Vertex AI: `VERTEXAI_PROJECT`, `VERTEXAI_LOCATION` + +If you have not picked a provider yet, browse [all supported providers](/docs/providers). + +## 3. Make Your First Call + +```python +from litellm import completion +import os + +os.environ["OPENAI_API_KEY"] = "your-api-key" + +response = completion( + model="openai/gpt-4o", + messages=[{"role": "user", "content": "Hello, how are you?"}], +) + +print(response.choices[0].message.content) +``` + +## 4. Check The Response + +The line below: + +```python +print(response.choices[0].message.content) +``` + +prints the assistant text, for example: + +```text +Hello! I'm doing well, thanks for asking. +``` + +If you print the full object with: + +```python +print(response) +``` + +you will see a Python `ModelResponse(...)` object. For an OpenAI-backed model, it can look like this: + +```python +ModelResponse( + id='chatcmpl-abc123', + created=1773782130, + model='gpt-4o-2024-08-06', + object='chat.completion', + system_fingerprint='fp_4ff89bf575', + choices=[ + Choices( + finish_reason='stop', + index=0, + message=Message( + content="Hello! I'm just a program, but I'm here to help you. How can I assist you today?", + role='assistant', + tool_calls=None, + function_call=None, + provider_specific_fields={'refusal': None}, + annotations=[] + ), + provider_specific_fields={} + ) + ], + usage=Usage( + completion_tokens=21, + prompt_tokens=13, + total_tokens=34, + completion_tokens_details=CompletionTokensDetailsWrapper(...), + prompt_tokens_details=PromptTokensDetailsWrapper(...) + ), + service_tier='default' +) +``` + +The same response follows an OpenAI-style shape. Conceptually, it looks like this: + +```json +{ + "id": "chatcmpl-abc123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4o", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Hello! I'm doing well, thanks for asking." + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 12, + "total_tokens": 25 + } +} +``` + +`id`, `created`, token counts, and message text will vary by request. + +If you call an OpenAI-backed model, you may also see extra fields such as `system_fingerprint`, `service_tier`, `tool_calls`, `function_call`, `annotations`, `provider_specific_fields`, and detailed token usage. For the full output reference, see [completion output](/docs/completion/output). + +Need more provider examples? See the main [Getting Started](/docs/#quick-start) page. + +## 5. Pick Your Next Step + + + +## When To Use Gateway Instead + +Use LiteLLM Gateway if you need centralized auth, virtual keys, spend tracking, shared logging, or one OpenAI-compatible endpoint for multiple apps. + +[Go to Gateway Quickstart →](/docs/learn/gateway_quickstart) diff --git a/docs/my-website/docs/load_test_advanced.md b/docs/my-website/docs/load_test_advanced.md index d35b5f74784..d7bc35e74e0 100644 --- a/docs/my-website/docs/load_test_advanced.md +++ b/docs/my-website/docs/load_test_advanced.md @@ -11,7 +11,7 @@ Tutorial on how to get to 1K+ RPS with LiteLLM Proxy on locust - [Github releases](https://github.com/BerriAI/litellm/releases) - [litellm docker containers](https://github.com/BerriAI/litellm/pkgs/container/litellm) - [litellm database docker container](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) -- [ ] Ensure you're following **ALL** [best practices for production](./proxy/production_setup.md) +- [ ] Ensure you're following **ALL** [best practices for production](./proxy/prod.md) - [ ] Locust - Ensure you're Locust instance can create 1K+ requests per second - 👉 You can use our **[maintained locust instance here](https://locust-load-tester-production.up.railway.app/)** - If you're self hosting locust @@ -222,4 +222,4 @@ class MyUser(HttpUser): def on_start(self): self.api_key = os.getenv('API_KEY', 'sk-1234') self.client.headers.update({'Authorization': f'Bearer {self.api_key}'}) -``` \ No newline at end of file +``` diff --git a/docs/my-website/docs/observability/langfuse_otel_integration.md b/docs/my-website/docs/observability/langfuse_otel_integration.md index b4c9a2bd1ad..79ad2f6f75d 100644 --- a/docs/my-website/docs/observability/langfuse_otel_integration.md +++ b/docs/my-website/docs/observability/langfuse_otel_integration.md @@ -83,6 +83,9 @@ os.environ["LANGFUSE_OTEL_HOST"] = "https://cloud.langfuse.com" # EU region # Or use self-hosted instance # os.environ["LANGFUSE_OTEL_HOST"] = "https://my-langfuse.company.com" +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# os.environ["OTEL_IGNORE_CONTEXT_PROPAGATION"] = "true" + litellm.callbacks = ["langfuse_otel"] ``` @@ -124,6 +127,9 @@ export LANGFUSE_PUBLIC_KEY="pk-lf-..." export LANGFUSE_SECRET_KEY="sk-lf-..." export LANGFUSE_OTEL_HOST="https://us.cloud.langfuse.com" # Default US region # export LANGFUSE_OTEL_HOST="https://otel.my-langfuse.company.com" # custom OTEL endpoint + +# Optional: Ignore otel context propagation to prevent parent-child relationships with spans from other providers +# export OTEL_IGNORE_CONTEXT_PROPAGATION="true" ``` 2. Setup config.yaml diff --git a/docs/my-website/docs/providers/azure/azure.md b/docs/my-website/docs/providers/azure/azure.md index 12ddc1bd98e..682f263c108 100644 --- a/docs/my-website/docs/providers/azure/azure.md +++ b/docs/my-website/docs/providers/azure/azure.md @@ -1032,7 +1032,7 @@ print("list_batches_response=", list_batches_response) -### [Health Check Azure Batch models](./proxy/health.md#batch-models-azure-only) +### [Health Check Azure Batch models](../../proxy/health.md#batch-models-azure-only) ### [BETA] Loadbalance Multiple Azure Deployments diff --git a/docs/my-website/docs/providers/azure/azure_anthropic.md b/docs/my-website/docs/providers/azure/azure_anthropic.md index 4c722b30397..e7cd8fffbf0 100644 --- a/docs/my-website/docs/providers/azure/azure_anthropic.md +++ b/docs/my-website/docs/providers/azure/azure_anthropic.md @@ -372,7 +372,6 @@ response = completion( ## Related Documentation -- [Anthropic Provider Documentation](./anthropic.md) - For standard Anthropic API usage +- [Anthropic Provider Documentation](../anthropic.md) - For standard Anthropic API usage - [Azure OpenAI Documentation](./azure.md) - For Azure OpenAI models -- [Azure Authentication Guide](../secret_managers/azure_key_vault.md) - For Azure AD token setup - +- [Azure Authentication Guide](../../secret_managers/azure_key_vault.md) - For Azure AD token setup diff --git a/docs/my-website/docs/providers/openai.md b/docs/my-website/docs/providers/openai.md index 80931ad8217..2907cdf9f47 100644 --- a/docs/my-website/docs/providers/openai.md +++ b/docs/my-website/docs/providers/openai.md @@ -1153,4 +1153,4 @@ response = completion( LiteLLM supports OpenAI's video generation models including Sora. -For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/video_generation.md) +For detailed documentation on video generation, see [OpenAI Video Generation →](./openai/videos.md) diff --git a/docs/my-website/docs/proxy/call_hooks.md b/docs/my-website/docs/proxy/call_hooks.md index 17354725fd5..5935a29c50b 100644 --- a/docs/my-website/docs/proxy/call_hooks.md +++ b/docs/my-website/docs/proxy/call_hooks.md @@ -7,7 +7,7 @@ import Image from '@theme/IdealImage'; - Enforce 'user' param for all openai endpoint calls :::tip -**Understanding Callback Hooks?** Check out our [Callback Management Guide](../observability/callback_management.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. +**Understanding Callback Hooks?** Check out our [Callback Guide](../observability/callbacks.md) to understand the differences between proxy-specific hooks like `async_pre_call_hook` and general logging hooks like `async_log_success_event`. ::: ## Which Hook Should I Use? diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index a0e404e3a18..f5b611a85a6 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -364,7 +364,7 @@ router_settings: | optional_pre_call_checks | List[str] | List of pre-call checks to add to the router. Supported: `router_budget_limiting`, `prompt_caching`, `responses_api_deployment_check`, `encrypted_content_affinity`, `deployment_affinity`, `session_affinity`, `forward_client_headers_by_model_group` | | deployment_affinity_ttl_seconds | int | TTL (seconds) for user-key → deployment affinity mapping when `deployment_affinity` is enabled (configured at Router init / proxy startup). Defaults to `3600` (1 hour). | | ignore_invalid_deployments | boolean | If true, ignores invalid deployments. Default for proxy is True - to prevent invalid models from blocking other models from being loaded. | -| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search.md) | +| search_tools | List[SearchToolTypedDict] | List of search tool configurations for Search API integration. Each tool specifies a search_tool_name and litellm_params with search_provider, api_key, api_base, etc. [Further Docs](../search/index.md) | | guardrail_list | List[GuardrailTypedDict] | List of guardrail configurations for guardrail load balancing. Enables load balancing across multiple guardrail deployments with the same guardrail_name. [Further Docs](./guardrails/guardrail_load_balancing.md) | diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 9ac487ae101..58a56604751 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -1,6 +1,6 @@ - import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; # Getting Started Tutorial @@ -78,13 +78,13 @@ You can skip the curl install and run `litellm --setup` directly after `pip inst ## Pre-Requisites -- Install LiteLLM Docker Image **OR** LiteLLM CLI (pip package) +Choose your install method. **Docker Compose** users complete their full setup inside the tab and are done. **Docker** and **pip** users continue with the steps below the tabs. -``` +```bash docker pull docker.litellm.ai/berriai/litellm:main-latest ``` @@ -102,7 +102,25 @@ $ pip install 'litellm[proxy]' -Use this docker compose to spin up the proxy with a postgres database running locally. +Docker Compose bundles LiteLLM with a Postgres database. Follow the steps below — the proxy will be fully running by the end. + +### Step 1 — Pull the LiteLLM database image + +LiteLLM provides a dedicated `litellm-database` image for proxy deployments that connect to Postgres. + +```bash +docker pull ghcr.io/berriai/litellm-database:main-latest +``` + +See all available tags on the [GitHub Container Registry](https://github.com/BerriAI/litellm/pkgs/container/litellm-database). + +--- + +### Step 2 — Set up a database + +Complete all three config files **before** running `docker compose up`. The proxy server will not start correctly if any of these are missing. + +#### 2.1 — Get `docker-compose.yml` and create `.env` ```bash # Get the docker compose file @@ -111,26 +129,154 @@ curl -O https://raw.githubusercontent.com/BerriAI/litellm/main/docker-compose.ym # Add the master key - you can change this after setup echo 'LITELLM_MASTER_KEY="sk-1234"' > .env -# Add the litellm salt key - you cannot change this after adding a model -# It is used to encrypt / decrypt your LLM API Key credentials -# We recommend - https://1password.com/password-generator/ -# password generator to get a random hash for litellm salt key +# Add the litellm salt key — cannot be changed after adding a model +# Used to encrypt/decrypt your LLM API key credentials +# Generate a strong random value: https://1password.com/password-generator/ echo 'LITELLM_SALT_KEY="sk-1234"' >> .env -# Start +# Add your model credentials +echo 'AZURE_API_BASE="https://openai-***********/"' >> .env +echo 'AZURE_API_KEY="your-azure-api-key"' >> .env +``` + +#### 2.2 — Create `config.yaml` + +The default `docker-compose.yml` starts a Postgres container at `db:5432`. Your `config.yaml` must include `database_url` pointing to it: + +```yaml +model_list: + - model_name: gpt-4o + litellm_params: + model: azure/my_azure_deployment + api_base: os.environ/AZURE_API_BASE + api_key: os.environ/AZURE_API_KEY + api_version: "2025-01-01-preview" + +general_settings: + master_key: sk-1234 # 🔑 your proxy admin key (must start with sk-) + database_url: "postgresql://llmproxy:dbpassword9090@db:5432/litellm" +``` + +:::tip +`database_url` enables virtual keys, spend tracking, and the UI. Replace it with your [Supabase](https://supabase.com/) or [Neon](https://neon.tech/) connection string if you prefer a managed database. +::: + +#### 2.3 — Create `prometheus.yml` + +This file **must exist as a file** before `docker compose up`. If it is missing, Docker auto-creates it as an empty directory and the Prometheus container fails to start. + +```yaml +global: + scrape_interval: 15s + evaluation_interval: 15s + +scrape_configs: + - job_name: "litellm" + static_configs: + - targets: ["litellm:4000"] +``` + +Also verify that the `config.yaml` volume mount and `--config` flag are **not commented out** in `docker-compose.yml`: + +```yaml +services: + litellm: + volumes: + - ./config.yaml:/app/config.yaml # ✅ must be uncommented + command: + - "--config=/app/config.yaml" # ✅ must be uncommented +``` + +:::warning +All three files (`.env`, `config.yaml`, `prometheus.yml`) must be present before running `docker compose up`. See [Troubleshooting](#troubleshooting) if you run into issues. +::: + +--- + +### Step 3 — Start the proxy server and test it + +After `config.yaml`, `prometheus.yml`, and `.env` are complete, start the proxy: + +```bash docker compose up ``` +Once running, test it with a curl request: + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ + -H 'Content-Type: application/json' \ + -H 'Authorization: Bearer sk-1234' \ + -d '{ + "model": "gpt-4o", + "messages": [{"role": "user", "content": "Hello!"}] + }' +``` + +**Expected response:** + +```json +{ + "id": "chatcmpl-abcd", + "created": 1773817678, + "model": "gpt-4o", + "object": "chat.completion", + "system_fingerprint": "fp_6b1ef07cda", + "choices": [ + { + "finish_reason": "stop", + "index": 0, + "message": { + "content": "Hello! How can I assist you today?", + "role": "assistant", + "annotations": [] + } + } + ], + "usage": { + "completion_tokens": 9, + "prompt_tokens": 9, + "total_tokens": 18, + "completion_tokens_details": { + "accepted_prediction_tokens": 0, + "audio_tokens": 0, + "reasoning_tokens": 0, + "rejected_prediction_tokens": 0 + }, + "prompt_tokens_details": { + "audio_tokens": 0, + "cached_tokens": 0 + } + }, + "service_tier": "default" +} +``` + +--- + +### Optional — Navigate to the LiteLLM UI and generate a virtual key + +Open [http://localhost:4000/ui](http://localhost:4000/ui) in your browser and log in with your master key (`sk-1234`). + +Navigate to **Virtual Keys** and click **+ Create New Key**: + +LiteLLM UI — Create Virtual Key + +Virtual keys let you track spend, set rate limits, and control model access per user or team. + + -## 1. Add a model +:::note Docker Compose users +Your setup is complete — the steps below are for **Docker** and **pip** users only. +::: -Control LiteLLM Proxy with a config.yaml file. +--- -Setup your config.yaml with your azure model. +## Step 1 — Add a model -Note: When using the proxy with a database, you can also **just add models via UI** (UI is available on `/ui` route). +Control LiteLLM Proxy with a `config.yaml` file. Create one with your Azure model: ```yaml model_list: @@ -154,8 +300,6 @@ You can read more about how model resolution works in the [Model Configuration]( - **`api_base`** (`str`) - The API base for your azure deployment. - **`api_version`** (`str`) - The API Version to use when calling Azure's OpenAI API. Get the latest Inference API version [here](https://learn.microsoft.com/en-us/azure/ai-services/openai/api-version-deprecation?source=recommendations#latest-preview-api-releases). ---- - --- @@ -203,19 +347,19 @@ $ litellm --config /app/config.yaml --detailed_debug +Confirm your config was loaded correctly — you should see this in the logs: -Confirm your config.yaml got mounted correctly - -```bash +``` Loaded config YAML (api_key and environment_variables are not shown): { -"model_list": [ -{ -"model_name ... + "model_list": [ + { + "model_name": ... ``` ### 2.2 Make Call +LiteLLM Proxy is 100% OpenAI-compatible. Test your model via `/chat/completions`: ```bash curl -X POST 'http://0.0.0.0:4000/chat/completions' \ @@ -309,15 +453,17 @@ curl -X POST 'http://0.0.0.0:4000/chat/completions' \ - [Other/Non-Chat Completion Endpoints](../embedding/supported_embedding.md) - [Pass-through for VertexAI, Bedrock, etc.](../pass_through/vertex_ai.md) -## 3. Generate a virtual key +## Optional: Generate a virtual key -Track Spend, and control model access via virtual keys for the proxy +Track spend and control model access via virtual keys for the proxy. -### 3.1 Set up a Database +### Prerequisite — Set up a database -**Requirements** -- Need a postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), etc) +:::note Docker Compose users +Your Postgres container is already running — skip ahead to [Create Key w/ RPM Limit](#create-key-w-rpm-limit) below. +::: +**Docker / pip users** — you need a Postgres database (e.g. [Supabase](https://supabase.com/), [Neon](https://neon.tech/), or self-hosted). Add `general_settings` to your `config.yaml`: ```yaml model_list: @@ -333,7 +479,9 @@ general_settings: database_url: "postgresql://:@:/" # 👈 KEY CHANGE ``` -Save config.yaml as `litellm_config.yaml` (used in 3.2). +Save config.yaml as `litellm_config.yaml` before continuing. + +You must finish this setup before starting the proxy server. --- @@ -359,7 +507,7 @@ See All General Settings [here](http://localhost:3000/docs/proxy/configs#all-set `database_url: "postgresql://..."` - Set `DATABASE_URL=postgresql://:@:/` in your env -### 3.2 Start Proxy +### Start Proxy ```bash docker run \ @@ -367,12 +515,11 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - docker.litellm.ai/berriai/litellm:main-latest \ + ghcr.io/berriai/litellm-database:main-latest \ --config /app/config.yaml --detailed_debug ``` - -### 3.3 Create Key w/ RPM Limit +### Create Key w/ RPM Limit Create a key with `rpm_limit: 1`. This will only allow 1 request per minute for calls to proxy with this key. @@ -395,9 +542,9 @@ curl -L -X POST 'http://0.0.0.0:4000/key/generate' \ } ``` -### 3.4 Test it! +### Test it! -**Use your virtual key from step 3.3** +**Use the virtual key you just created.** 1st call - Expect to work! @@ -611,6 +758,24 @@ model_list: ## Troubleshooting +### `prometheus.yml` mount error — "not a directory" + +If you see: + +```bash +Error: cannot create subdirectories in ".../prometheus.yml": not a directory +``` + +Docker created `prometheus.yml` as an **empty directory** instead of a file. This happens when the file is missing at `docker compose up` time. + +Fix it: +Then create the file (see [Step 2.3 — Create `prometheus.yml`](#23--create-prometheusyml)) and run `docker compose up` again. +```bash +rm -rf prometheus.yml +``` + +Then create the file (see [Step 2.4](#step-24--create-prometheusyml)) and run `docker compose up` again. + ### Non-root docker image? If you need to run the docker image as a non-root user, use [this](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root). @@ -710,6 +875,3 @@ LiteLLM Proxy uses the [LiteLLM Python SDK](https://docs.litellm.ai/docs/routing - Our emails ✉️ ishaan@berri.ai / krrish@berri.ai [![Chat on WhatsApp](https://img.shields.io/static/v1?label=Chat%20on&message=WhatsApp&color=success&logo=WhatsApp&style=flat-square)](https://wa.link/huol9n) [![Chat on Discord](https://img.shields.io/static/v1?label=Chat%20on&message=Discord&color=blue&logo=Discord&style=flat-square)](https://discord.gg/wuPM9dRgDw) - - - diff --git a/docs/my-website/docs/proxy/endpoint_activity.md b/docs/my-website/docs/proxy/endpoint_activity.md index a66c0f7a5e5..d06727ce4b4 100644 --- a/docs/my-website/docs/proxy/endpoint_activity.md +++ b/docs/my-website/docs/proxy/endpoint_activity.md @@ -114,4 +114,4 @@ Understand spend distribution across endpoints: - [Customer Usage](./customer_usage.md) - Track spend and usage for individual customers - [Cost Tracking](./cost_tracking.md) - Comprehensive cost tracking and analytics -- [Spend Logs](./spend_logs.md) - Detailed request-level spend logs +- [Spend Logs](./cost_tracking.md#-spend-logs-api---individual-transaction-logs) - Detailed request-level spend logs diff --git a/docs/my-website/docs/proxy/keys_teams_router_settings.md b/docs/my-website/docs/proxy/keys_teams_router_settings.md index ec59e8f271b..6a1744ca951 100644 --- a/docs/my-website/docs/proxy/keys_teams_router_settings.md +++ b/docs/my-website/docs/proxy/keys_teams_router_settings.md @@ -146,5 +146,5 @@ Test new router settings on specific keys or teams before applying globally: - [Router Settings Reference](./config_settings.md#router_settings---reference) - Complete reference of all router settings - [Load Balancing](./load_balancing.md) - Learn about routing strategies and load balancing - [Reliability](./reliability.md) - Configure fallbacks, retries, and error handling -- [Keys](./keys.md) - Manage API keys and their settings -- [Teams](./teams.md) - Organize keys into teams +- [Keys](./virtual_keys.md) - Manage API keys and their settings +- [Teams](./multi_tenant_architecture.md) - Organize keys into teams diff --git a/docs/my-website/docs/proxy/model_compare_ui.md b/docs/my-website/docs/proxy/model_compare_ui.md index bd6f5414224..ee0376ed2fa 100644 --- a/docs/my-website/docs/proxy/model_compare_ui.md +++ b/docs/my-website/docs/proxy/model_compare_ui.md @@ -187,7 +187,7 @@ Use tags and multiple comparisons to run structured A/B tests: ## Related Features -- [Playground Chat UI](./playground.md) - Single model testing interface +- [Playground Chat UI](./ui.md) - Single model testing interface - [Model Management](./model_management.md) - Configure and manage models -- [Guardrails](./guardrails.md) - Set up safety filters +- [Guardrails](./guardrails/quick_start.md) - Set up safety filters - [AI Hub](./ai_hub.md) - Share models and agents with your organization diff --git a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md index e1645082d97..0373d20c879 100644 --- a/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md +++ b/docs/my-website/docs/proxy/sync_anthropic_beta_headers.md @@ -125,4 +125,4 @@ curl -X DELETE "https://your-proxy-url/schedule/anthropic_beta_headers_reload" \ ## Related - [Model Cost Map Sync](./sync_models_github.md) - Auto-sync model pricing data -- [Anthropic Beta Headers](../completion/anthropic.md#beta-features) - Using Anthropic beta features +- [Anthropic Beta Headers](../providers/anthropic.md) - Using Anthropic beta features diff --git a/docs/my-website/docs/proxy/ui_store_model_db_setting.md b/docs/my-website/docs/proxy/ui_store_model_db_setting.md index 5f860137d0f..4b0bc690f9a 100644 --- a/docs/my-website/docs/proxy/ui_store_model_db_setting.md +++ b/docs/my-website/docs/proxy/ui_store_model_db_setting.md @@ -87,6 +87,6 @@ Change the setting from the UI and have it take effect immediately—perfect for ## Related Documentation -- [Admin UI Overview](./ui_overview.md) – General guide to the LiteLLM Admin UI -- [Models and Endpoints](./models_and_endpoints.md) – Managing models and API endpoints +- [Admin UI Overview](./ui.md) – General guide to the LiteLLM Admin UI +- [Models and Endpoints](./model_management.md) – Managing models and API endpoints - [Config Settings](./config_settings.md) – `store_model_in_db` in `general_settings` diff --git a/docs/my-website/docs/proxy/user_onboarding.md b/docs/my-website/docs/proxy/user_onboarding.md index baa241d6cdf..ecbdc11db43 100644 --- a/docs/my-website/docs/proxy/user_onboarding.md +++ b/docs/my-website/docs/proxy/user_onboarding.md @@ -79,4 +79,4 @@ curl -X POST http://localhost:4000/v1/chat/completions \ ## See Also - [Proxy Quick Start](./quick_start.md) - [User Management](./users.md) -- [Key Management](./key_management.md) +- [Key Management](./virtual_keys.md) diff --git a/docs/my-website/docs/tutorials/index.md b/docs/my-website/docs/tutorials/index.md new file mode 100644 index 00000000000..7f80cc760ee --- /dev/null +++ b/docs/my-website/docs/tutorials/index.md @@ -0,0 +1,98 @@ +--- +title: Tutorials +sidebar_label: Overview +--- + +import NavigationCards from '@site/src/components/NavigationCards'; + +**Tutorials** are step-by-step walkthroughs for integrating LiteLLM with external tools, frameworks, and services — or building complete end-to-end workflows. + +> Need help choosing the right path before you start? See [Learn →](/docs/learn) + +--- + +## Getting Started + + + +--- + +## Integrations + + + +--- + +## Proxy + + + +--- + +## Observability & Evaluation + + diff --git a/docs/my-website/docs/tutorials/installation.md b/docs/my-website/docs/tutorials/installation.md index ecaed0bec9d..cf39c55bee6 100644 --- a/docs/my-website/docs/tutorials/installation.md +++ b/docs/my-website/docs/tutorials/installation.md @@ -1,7 +1,3 @@ ---- -displayed_sidebar: tutorialSidebar ---- - # Set up environment Let's get the necessary keys to set up our demo environment. @@ -11,7 +7,5 @@ Every LLM provider needs API keys (e.g. `OPENAI_API_KEY`). You can get API keys Let's get them for our demo! **OpenAI**: https://platform.openai.com/account/api-keys -**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) +**Cohere**: https://dashboard.cohere.com/welcome/login?redirect_uri=%2Fapi-keys (no credit card required) **AI21**: https://studio.ai21.com/account/api-key (no credit card required) - - diff --git a/docs/my-website/docs/vertex_batch_passthrough.md b/docs/my-website/docs/vertex_batch_passthrough.md index 3203d7d792a..17ffc1e6dbc 100644 --- a/docs/my-website/docs/vertex_batch_passthrough.md +++ b/docs/my-website/docs/vertex_batch_passthrough.md @@ -155,6 +155,6 @@ Common error scenarios and their solutions: ## Related Documentation -- [Vertex AI Provider Documentation](./vertex.md) -- [General Batches API Documentation](../batches.md) -- [Cost Tracking and Monitoring](../observability/telemetry.md) +- [Vertex AI Provider Documentation](./providers/vertex.md) +- [General Batches API Documentation](./batches.md) +- [Cost Tracking and Monitoring](./observability/telemetry.md) diff --git a/docs/my-website/docusaurus.config.js b/docs/my-website/docusaurus.config.js index 32d5d800b71..9e1fae6a34f 100644 --- a/docs/my-website/docusaurus.config.js +++ b/docs/my-website/docusaurus.config.js @@ -2,9 +2,9 @@ // Note: type annotations allow type checking and IDEs autocompletion // @ts-ignore -const lightCodeTheme = require('prism-react-renderer/themes/github'); +const lightCodeTheme = require('prism-react-renderer/themes/vsLight'); // @ts-ignore -const darkCodeTheme = require('prism-react-renderer/themes/dracula'); +const darkCodeTheme = require('prism-react-renderer/themes/nightOwl'); const inkeepConfig = { baseSettings: { @@ -87,18 +87,88 @@ const config = { }, ], [ - '@docusaurus/plugin-content-blog', + '@docusaurus/plugin-content-docs', { - id: 'release_notes', + id: 'release-notes', path: './release_notes', routeBasePath: 'release_notes', - blogTitle: 'Release Notes', - blogSidebarTitle: 'Releases', - blogSidebarCount: 'ALL', - postsPerPage: 'ALL', - showReadingTime: false, - sortPosts: 'descending', - include: ['**/*.{md,mdx}'], + sidebarPath: require.resolve('./sidebars-release-notes.js'), + async sidebarItemsGenerator({defaultSidebarItemsGenerator, docs, ...args}) { + const items = await defaultSidebarItemsGenerator({docs, ...args}); + + // Build map of doc id -> year from frontmatter date + const docYearMap = {}; + for (const doc of docs) { + const date = doc.frontMatter && doc.frontMatter.date; + if (date) { + const year = new Date(date).getFullYear(); + docYearMap[doc.id] = year; + } + } + + function parseVersion(str) { + const match = (str || '').match(/v?(\d+)\.(\d+)\.(\d+)/); + if (!match) return [0, 0, 0]; + return [parseInt(match[1]), parseInt(match[2]), parseInt(match[3])]; + } + function compareVersionsDesc(a, b) { + const [aMaj, aMin, aPatch] = parseVersion(a.label || a.id || ''); + const [bMaj, bMin, bPatch] = parseVersion(b.label || b.id || ''); + if (bMaj !== aMaj) return bMaj - aMaj; + if (bMin !== aMin) return bMin - aMin; + return bPatch - aPatch; + } + + // Flatten and transform doc items (filter index, shorten labels) + function flattenDocs(list) { + const result = []; + for (const item of list) { + if (item.type === 'doc' && item.id === 'index') continue; + if (item.type === 'doc') { + const label = item.id.replace(/\/index$/, ''); + result.push({...item, label}); + } else if (item.type === 'category') { + if (item.link && item.link.type === 'doc' && item.link.id !== 'index') { + const id = item.link.id; + const label = id.replace(/\/index$/, ''); + result.push({type: 'doc', id, label}); + } else { + result.push(...flattenDocs(item.items)); + } + } + } + return result; + } + + const docItems = flattenDocs(items); + + // Group by year + const byYear = {}; + for (const item of docItems) { + const year = docYearMap[item.id] || 'Other'; + if (!byYear[year]) byYear[year] = []; + byYear[year].push(item); + } + + // Sort each year's items by version descending + for (const year of Object.keys(byYear)) { + byYear[year].sort(compareVersionsDesc); + } + + // Build categories sorted by year descending + const years = Object.keys(byYear).sort((a, b) => { + // Object.keys() returns strings; avoid numeric subtraction type errors. + const na = Number.parseInt(a, 10); + const nb = Number.parseInt(b, 10); + return nb - na; + }); + return years.map(year => ({ + type: 'category', + label: String(year), + collapsed: year !== String(years[0]), + items: byYear[year], + })); + }, }, ], [ @@ -130,6 +200,20 @@ const config = { }; }, }), + // Ensure gtag exists before the GA script loads. + () => ({ + name: 'gtag-shim', + injectHtmlTags() { + return { + headTags: [ + { + tagName: 'script', + innerHTML: `window.dataLayer=window.dataLayer||[];function gtag(){dataLayer.push(arguments);}if(!window.gtag){window.gtag=gtag;}`, + }, + ], + }; + }, + }), ], presets: [ @@ -137,10 +221,13 @@ const config = { 'classic', /** @type {import('@docusaurus/preset-classic').Options} */ ({ - gtag: { - trackingID: 'G-K7K215ZVNC', - anonymizeIP: true, - }, + gtag: + process.env.NODE_ENV === 'production' + ? { + trackingID: 'G-K7K215ZVNC', + anonymizeIP: true, + } + : undefined, docs: { sidebarPath: require.resolve('./sidebars.js'), }, @@ -181,34 +268,39 @@ const config = { label: 'Docs', }, { + type: 'docSidebar', + sidebarId: 'learnSidebar', + position: 'left', + label: 'Learn', + }, + { + type: 'docSidebar', sidebarId: 'integrationsSidebar', position: 'left', label: 'Integrations', - to: "docs/integrations" }, { - sidebarId: 'tutorialSidebar', position: 'left', label: 'Enterprise', to: "docs/enterprise" }, - { to: '/release_notes', label: 'Release Notes', position: 'left' }, { to: '/blog', label: 'Blog', position: 'left' }, - { - href: 'https://models.litellm.ai/', - label: '💸 LLM Model Cost Map', - position: 'right', - }, { href: 'https://github.com/BerriAI/litellm', - label: 'GitHub', position: 'right', + className: 'header-github-link', + 'aria-label': 'GitHub repository', }, { href: 'https://www.litellm.ai/support', - label: 'Slack/Discord', position: 'right', - } + className: 'header-discord-link', + 'aria-label': 'Discord / Slack community', + }, + { + type: 'search', + position: 'right', + }, ], }, footer: { diff --git a/docs/my-website/img/hero.png b/docs/my-website/img/hero.png new file mode 100644 index 00000000000..9f77a28d718 Binary files /dev/null and b/docs/my-website/img/hero.png differ diff --git a/docs/my-website/release_notes/index.md b/docs/my-website/release_notes/index.md new file mode 100644 index 00000000000..e2b7edf3222 --- /dev/null +++ b/docs/my-website/release_notes/index.md @@ -0,0 +1,52 @@ +--- +title: Release Notes +sidebar_label: Overview +slug: / +--- + +# Release Notes + +LiteLLM ships new releases regularly with new provider support, performance improvements, and enterprise features. Use the sidebar to browse all releases. + +## Latest Release + +### [v1.82.3 — Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models](/release_notes/v1.82.3/v1-82-3) + +_March 16, 2026_ + +116 new models including Nebius AI, gpt-5.4, Gemini 3.x, and FLUX Kontext. + +--- + +## Recent Releases + +| Version | Date | Highlights | +| ----------------------------------- | ------------ | ---------------------------------------------------------- | +| [v1.82.0](/release_notes/v1.82.0/v1-82-0) | Feb 28, 2026 | Realtime Guardrails, Projects Management, and 10+ Performance Optimizations | +| [v1.81.14](/release_notes/v1.81.14/v1-81-14) | Feb 21, 2026 | New Gateway Level Guardrails & Compliance Playground | +| [v1.81.12](/release_notes/v1.81.12/v1-81-12) | Feb 14, 2026 | Guardrail Policy Templates & Action Builder | +| [v1.81.9](/release_notes/v1.81.9/v1-81-9) | Feb 7, 2026 | Control which MCP Servers are exposed on the Internet | +| [v1.81.6](/release_notes/v1.81.6/v1-81-6) | Jan 31, 2026 | Logs v2 with Tool Call Tracing | +| [v1.81.3](/release_notes/v1.81.3-stable/v1-81-3) | Jan 26, 2026 | Performance — 25% CPU Usage Reduction | +| [v1.81.0](/release_notes/v1.81.0/v1-81-0) | Jan 18, 2026 | Claude Code — Web Search Across All Providers | +| [v1.80.15](/release_notes/v1.80.15/v1-80-15) | Jan 10, 2026 | Manus API Support | +| [v1.80.8](/release_notes/v1.80.8-stable/v1-80-8) | Dec 6, 2025 | Introducing A2A Agent Gateway | +| [v1.80.5](/release_notes/v1.80.5-stable/v1-80-5) | Nov 22, 2025 | Gemini 3.0 Support | +| [v1.80.0](/release_notes/v1.80.0-stable/v1-80-0) | Nov 15, 2025 | Introducing Agent Hub: Register, Publish, and Share Agents | +| [v1.79.3](/release_notes/v1.79.3-stable/v1-79-3) | Nov 8, 2025 | Built-in Guardrails on AI Gateway | +| [v1.79.0](/release_notes/v1.79.0-stable/v1-79-0) | Oct 26, 2025 | Search APIs | +| [v1.78.5](/release_notes/v1.78.5-stable/v1-78-5) | Oct 18, 2025 | Native OCR Support | +| [v1.78.0](/release_notes/v1.78.0-stable/v1-78-0) | Oct 11, 2025 | MCP Gateway: Control Tool Access by Team, Key | +| [v1.77.7](/release_notes/v1.77.7-stable/v1-77-7) | Oct 4, 2025 | 2.9x Lower Median Latency | +| [v1.77.5](/release_notes/v1.77.5-stable/v1-77-5) | Sep 29, 2025 | MCP OAuth 2.0 Support | +| [v1.77.3](/release_notes/v1.77.3-stable/v1-77-3) | Sep 21, 2025 | Priority Based Rate Limiting | + +--- + +## Stay Updated + +- **GitHub**: Watch the [BerriAI/litellm](https://github.com/BerriAI/litellm) repository for release notifications +- **Discord**: Join our [community](https://discord.com/invite/wuPM9dRgDw) for announcements +- **Twitter**: Follow [@LiteLLM](https://twitter.com/LiteLLM) + +Use the sidebar to browse the full release history. diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md index 1ac713fc2d5..d34b2c7b335 100644 --- a/docs/my-website/release_notes/v1.63.14/index.md +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -62,10 +62,10 @@ Here's a Demo Instance to test changes: - Infer aws region from bedrock application profile id - (`arn:aws:bedrock:us-east-1:...`) - Ollama - support calling via `/v1/completions` [Get Started](../../docs/providers/ollama#using-ollama-fim-on-v1completions) - Bedrock - support `us.deepseek.r1-v1:0` model name [Docs](../../docs/providers/bedrock#supported-aws-bedrock-models) -- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter.md) +- OpenRouter - `OPENROUTER_API_BASE` env var support [Docs](../../docs/providers/openrouter) - Azure - add audio model parameter support - [Docs](../../docs/providers/azure#azure-audio-model) - OpenAI - PDF File support [Docs](../../docs/completion/document_understanding#openai-file-message-type) -- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api.md#streaming) +- OpenAI - o1-pro Responses API streaming support [Docs](../../docs/response_api#streaming) - [BETA] MCP - Use MCP Tools with LiteLLM SDK [Docs](../../docs/mcp) 2. **Bug Fixes** @@ -102,7 +102,7 @@ Here's a Demo Instance to test changes: - fix logging to just log the LLM I/O [PR](https://github.com/BerriAI/litellm/pull/9353) - Dynamic API Key/Space param support [Get Started](../../docs/observability/arize_integration#pass-arize-spacekey-per-request) - StandardLoggingPayload - Log litellm_model_name in payload. Allows knowing what the model sent to API provider was [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) -- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management.md) +- Prompt Management - Allow building custom prompt management integration [Get Started](../../docs/proxy/custom_prompt_management) ## Performance / Reliability improvements @@ -128,4 +128,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.63.11-stable...v1.63.14.rc) diff --git a/docs/my-website/release_notes/v1.63.2-stable/index.md b/docs/my-website/release_notes/v1.63.2-stable/index.md index a248aa94342..18233f25c21 100644 --- a/docs/my-website/release_notes/v1.63.2-stable/index.md +++ b/docs/my-website/release_notes/v1.63.2-stable/index.md @@ -64,15 +64,15 @@ Here's a Demo Instance to test changes: 9. Bedrock - handle thinking blocks in assistant message. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) 10. Anthropic - Return `signature` on streaming. [Get Started](https://docs.litellm.ai/docs/providers/bedrock#usage---thinking--reasoning-content) - Note: We've also migrated from `signature_delta` to `signature`. [Read more](https://docs.litellm.ai/release_notes/v1.63.0) -11. Support format param for specifying image type. [Get Started](../../docs/completion/vision.md#explicitly-specify-image-type) -12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified.md) +11. Support format param for specifying image type. [Get Started](../../docs/completion/vision#explicitly-specify-image-type) +12. Anthropic - `/v1/messages` endpoint - `thinking` param support. [Get Started](../../docs/anthropic_unified) - Note: this refactors the [BETA] unified `/v1/messages` endpoint, to just work for the Anthropic API. 13. Vertex AI - handle $id in response schema when calling vertex ai. [Get Started](https://docs.litellm.ai/docs/providers/vertex#json-schema) ## Spend Tracking Improvements 1. Batches API - Fix cost calculation to run on retrieve_batch. [Get Started](https://docs.litellm.ai/docs/batches) -2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec.md#standardlogginghiddenparams) +2. Batches API - Log batch models in spend logs / standard logging payload. [Get Started](../../docs/proxy/logging_spec#standardlogginghiddenparams) ## Management Endpoints / UI @@ -109,4 +109,4 @@ Here's a Demo Instance to test changes: ## Complete Git Diff -[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) \ No newline at end of file +[Here's the complete git diff](https://github.com/BerriAI/litellm/compare/v1.61.20-stable...v1.63.2-stable) diff --git a/docs/my-website/release_notes/v1.80.15/index.md b/docs/my-website/release_notes/v1.80.15/index.md index 4037a0d9b5d..15b49822965 100644 --- a/docs/my-website/release_notes/v1.80.15/index.md +++ b/docs/my-website/release_notes/v1.80.15/index.md @@ -53,7 +53,7 @@ pip install litellm==1.80.15 - **MCP Global Mode** - [Configure MCP servers globally with visibility controls](../../docs/mcp) - **Interactions API Bridge** - [Use all LiteLLM providers with the Interactions API](../../docs/interactions) - **RAG Query Endpoint** - [New RAG Search/Query endpoint for retrieval-augmented generation](../../docs/search/index) -- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity.md) +- **UI Usage - Endpoint Activity** - [Users can now see Endpoint Activity Metrics in the UI](../../docs/proxy/endpoint_activity) - **50% Overhead Reduction** - LiteLLM now sends 2.5× more requests to LLM providers @@ -640,4 +640,3 @@ Users can now see Endpoint Activity Metrics in the UI. **[View complete changelog on GitHub](https://github.com/BerriAI/litellm/compare/v1.80.11.rc.1...v1.80.15-stable.1)** - diff --git a/docs/my-website/release_notes/v1.81.0/index.md b/docs/my-website/release_notes/v1.81.0/index.md index e61d7d2d593..5953c572a7a 100644 --- a/docs/my-website/release_notes/v1.81.0/index.md +++ b/docs/my-website/release_notes/v1.81.0/index.md @@ -48,7 +48,7 @@ pip install litellm==1.81.0 - **Claude Code** - Support for using web search across Bedrock, Vertex AI, and all LiteLLM providers - **Major Change** - [50MB limit on image URL downloads](#major-change---chatcompletions-image-url-download-size-limit) to improve reliability - **Performance** - [25% CPU Usage Reduction](#performance---25-cpu-usage-reduction) by removing premature model.dump() calls from the hot path -- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams.md) with spend and budget information at the time of deletion +- **Deleted Keys Audit Table on UI** - [View deleted keys and teams for audit purposes](../../docs/proxy/deleted_keys_teams) with spend and budget information at the time of deletion --- @@ -166,7 +166,7 @@ LiteLLM now reduces CPU usage by removing premature `model.dump()` calls from th -LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams.md). +LiteLLM now provides a comprehensive audit table for deleted API keys and teams directly in the UI. This feature allows you to easily track the spend of deleted keys, view their associated team information, and maintain accurate financial records for auditing and compliance purposes. The table displays key details including key aliases, team associations, and spend information captured at the time of deletion. For more information on how to use this feature, see the [Deleted Keys & Teams documentation](../../docs/proxy/deleted_keys_teams). --- diff --git a/docs/my-website/release_notes/v1.81.12.md b/docs/my-website/release_notes/v1.81.12/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.12.md rename to docs/my-website/release_notes/v1.81.12/index.md index 0b7c1e146ab..1bbea5b82bc 100644 --- a/docs/my-website/release_notes/v1.81.12.md +++ b/docs/my-website/release_notes/v1.81.12/index.md @@ -62,13 +62,13 @@ This release fixes out-of-memory (OOM) risks from unbounded `asyncio.Queue()` us This release adds a visual action builder for guardrail policies with conditional execution support. You can now chain guardrails into multi-step pipelines — if a simple guardrail fails, route to an advanced one instead of immediately blocking. Each step has configurable ON PASS and ON FAIL actions (Next Step, Block, or Allow), and you can test the full pipeline with a sample message before saving. -![Guardrail Action Builder](../img/release_notes/guard_actions.png) +![Guardrail Action Builder](../../img/release_notes/guard_actions.png) ### Access Groups Access Groups simplify defining resource access across your organization. One group can grant access to models, MCP servers, and agents—simply attach it to a key or team. Create groups in the Admin UI, define which resources each group includes, then assign the group when creating keys or teams. Updates to a group apply automatically to all attached keys and teams. - + ## New Providers and Endpoints diff --git a/docs/my-website/release_notes/v1.81.14.md b/docs/my-website/release_notes/v1.81.14/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.14.md rename to docs/my-website/release_notes/v1.81.14/index.md index c342bc47ee9..92c22bc0ea3 100644 --- a/docs/my-website/release_notes/v1.81.14.md +++ b/docs/my-website/release_notes/v1.81.14/index.md @@ -56,7 +56,7 @@ pip install litellm==1.81.14 AI Platform Admins can now browse built-in and partner guardrails from the Guardrail Garden. Guardrails are organized by use case — blocking financial advice, filtering insults, detecting competitor mentions, and more — so you can find the right one and deploy it in a few clicks. -![Guardrail Garden](../img/release_notes/guardrail_garden.png) +![Guardrail Garden](../../img/release_notes/guardrail_garden.png) ### 3 New Built-in Guardrails @@ -72,7 +72,7 @@ These guardrails are built for production and on our benchmarks had a 100% Recal Previously, the `store_model_in_db` setting could only be configured in `proxy_config.yaml` under `general_settings`, requiring a proxy restart to take effect. Now you can enable or disable this setting directly from the Admin UI without any restarts. This is especially useful for cloud deployments where you don't have direct access to config files or want to avoid downtime. Enable `store_model_in_db` to move model definitions from your YAML into the database—reducing config complexity, improving scalability, and enabling dynamic model management across multiple proxy instances. -![Store model in DB Setting](../img/ui_store_model_in_db.png) +![Store model in DB Setting](../../img/ui_store_model_in_db.png) #### Eval results @@ -91,14 +91,14 @@ We benchmarked our new built-in guardrails against labeled datasets before shipp The Compliance Playground lets you test any guardrail against our pre-built eval datasets or your own custom datasets, so you can see precision, recall, and false positive rate before rolling it out to production. -![Compliance Playground](../img/release_notes/compliance_playground.png) +![Compliance Playground](../../img/release_notes/compliance_playground.png) --- ## Performance & Reliability — Up to 13% Lower Latency - + This release cuts latency across all percentiles through 20+ micro-optimizations across logging, cost calculation, routing, and connection management. See [benchmarking](../../docs/benchmarks) for more info about how to benchmark yourself. diff --git a/docs/my-website/release_notes/v1.81.6.md b/docs/my-website/release_notes/v1.81.6/index.md similarity index 100% rename from docs/my-website/release_notes/v1.81.6.md rename to docs/my-website/release_notes/v1.81.6/index.md diff --git a/docs/my-website/release_notes/v1.81.9.md b/docs/my-website/release_notes/v1.81.9/index.md similarity index 99% rename from docs/my-website/release_notes/v1.81.9.md rename to docs/my-website/release_notes/v1.81.9/index.md index 80be4179b46..d11b52e892c 100644 --- a/docs/my-website/release_notes/v1.81.9.md +++ b/docs/my-website/release_notes/v1.81.9/index.md @@ -81,7 +81,7 @@ This release makes it safe to expose MCP servers on the public internet by addin [Get started](../../docs/mcp_public_internet) @@ -92,7 +92,7 @@ Set a soft budget on any team to receive email alerts when spending crosses the [Get started](../../docs/proxy/ui_team_soft_budget_alerts) diff --git a/docs/my-website/release_notes/v1.82.0.md b/docs/my-website/release_notes/v1.82.0/index.md similarity index 100% rename from docs/my-website/release_notes/v1.82.0.md rename to docs/my-website/release_notes/v1.82.0/index.md diff --git a/docs/my-website/release_notes/v1.82.3/index.md b/docs/my-website/release_notes/v1.82.3/index.md new file mode 100644 index 00000000000..15df33fdb80 --- /dev/null +++ b/docs/my-website/release_notes/v1.82.3/index.md @@ -0,0 +1,374 @@ +--- +title: "v1.82.3 - Nebius AI, gpt-5.4, Gemini 3.x, FLUX Kontext, and 116 New Models" +slug: "v1-82-3" +date: 2026-03-16T00:00:00 +authors: + - name: Krrish Dholakia + title: CEO, LiteLLM + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: CTO, LiteLLM + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg +hide_table_of_contents: false +--- + +## Deploy this version + +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + + + + +``` showLineNumbers title="docker run litellm" +docker run \ +-e STORE_MODEL_IN_DB=True \ +-p 4000:4000 \ +ghcr.io/berriai/litellm:main-1.82.3-stable +``` + + + + +``` showLineNumbers title="pip install litellm" +pip install litellm==1.82.3 +``` + + + + +## Key Highlights + +- **Nebius AI — new provider** — [30 models across DeepSeek, Qwen, Llama, Mistral, NVIDIA, and BAAI available via Nebius AI cloud](../../docs/providers/nebius) - [PR #22614](https://github.com/BerriAI/litellm/pull/22614) +- **OpenAI gpt-5.4 / gpt-5.4-pro — day 0** — Full pricing and routing support for `gpt-5.4` (1M context, $2.50/$15.00) and `gpt-5.4-pro` ($30.00/$180.00) on OpenAI and Azure +- **Gemini 3.x models** — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-image-preview`, and `gemini-embedding-2-preview` added to cost map for Google AI and Vertex AI +- **FLUX Kontext image editing** — `flux-kontext-pro` and `flux-kontext-max` added to Black Forest Labs, alongside `flux-pro-1.0-fill` and `flux-pro-1.0-expand` for inpainting and outpainting +- **116 new models, 132 deprecated models cleaned up** — Major model map refresh including Mistral Magistral, Dashscope Qwen3 VL, xAI Grok via Azure AI, ZAI GLM-5, Serper Search; removal of OpenAI GPT-3.5/GPT-4 legacy variants, Gemini 1.5, and Vertex AI PaLM2 +- **SageMaker Nova provider** — [New `sagemaker_nova` provider for Amazon Nova models on SageMaker](../../docs/providers/aws_sagemaker) - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +- **Secret redaction in logs** — API keys, tokens, and credentials automatically scrubbed from all proxy log output. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +- **Streaming stability fix** — Critical fix for `RuntimeError: Cannot send a request, as the client has been closed.` crashes after ~1 hour in production - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) + +--- + +## New Providers and Endpoints + +### New Providers (5 new providers) + +| Provider | Supported LiteLLM Endpoints | Description | +| -------- | --------------------------- | ----------- | +| [Nebius AI](../../docs/providers/nebius) (`nebius/`) | `/chat/completions`, `/embeddings` | EU-based AI cloud with 30+ open models — DeepSeek, Qwen3, Llama 3.1/3.3, NVIDIA Nemotron, BAAI embeddings | +| [ZAI](../../docs/providers/zai) (`zai/`) | `/chat/completions` | ZhipuAI GLM-5 models via ZAI cloud | +| [Black Forest Labs](../../docs/providers/black_forest_labs) (`black_forest_labs/`) | `/images/generations`, `/images/edits` | FLUX image generation and editing — Kontext Pro/Max, Pro 1.0 Fill/Expand | +| [Serper](../../docs/providers/serper) (`serper/`) | `/search` | Web search via Serper API | +| [SageMaker Nova](../../docs/providers/aws_sagemaker) (`sagemaker_nova/`) | `/chat/completions` | Amazon Nova models via SageMaker endpoint | + +--- + +## New Models / Updated Models + +#### New Model Support (116 new models) + +| Provider | Model | Context Window | Input ($/1M tokens) | Output ($/1M tokens) | Features | +| -------- | ----- | -------------- | ------------------- | -------------------- | -------- | +| OpenAI | `gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| OpenAI | `gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| OpenAI | `gpt-5.3-chat-latest` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4` | 1.05M | $2.50 | $15.00 | chat, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.4-pro` | 1.05M | $30.00 | $180.00 | responses, vision, tools, reasoning | +| Azure OpenAI | `azure/gpt-5.3-chat` | 128K | $1.75 | $14.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3-flash-preview` | 1M | $0.50 | $3.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-pro-preview` | 1M | $2.00 | $12.00 | chat, vision, tools, reasoning | +| Google Gemini | `gemini/gemini-3.1-flash-image-preview` | 65K | $0.25 | $1.50 | image generation, vision | +| Google Gemini | `gemini/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Gemini | `gemini/gemini-3-pro-image-preview` | - | - | - | image generation | +| Google Gemini | `gemini/gemini-embedding-2-preview` | 8K | $0.20 | - | embeddings | +| Google Vertex AI | `vertex_ai/gemini-3-flash-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-pro-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-3.1-flash-lite-preview` | - | - | - | chat | +| Google Vertex AI | `vertex_ai/gemini-embedding-2-preview` | - | $0.20 | - | embeddings | +| Mistral | `mistral/magistral-medium-1-2-2509` | 40K | $2.00 | $5.00 | chat, tools, reasoning | +| Mistral | `mistral/magistral-small-1-2-2509` | 40K | $0.50 | $1.50 | chat, tools, reasoning | +| Mistral | `mistral/mistral-large-2512` | 262K | $0.50 | $1.50 | chat, vision, tools | +| Mistral | `mistral/mistral-medium-3-1-2508` | - | - | - | chat | +| Mistral | `mistral/mistral-small-3-2-2506` | - | - | - | chat | +| Mistral | `mistral/ministral-3-3b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-8b-2512` | - | - | - | chat | +| Mistral | `mistral/ministral-3-14b-2512` | - | - | - | chat | +| Black Forest Labs | `black_forest_labs/flux-kontext-pro` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-kontext-max` | - | - | - | image editing | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-fill` | - | - | - | image editing (inpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.0-expand` | - | - | - | image editing (outpaint) | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro-1.1-ultra` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-dev` | - | - | - | image generation | +| Black Forest Labs | `black_forest_labs/flux-pro` | - | - | - | image generation | +| Azure AI | `azure_ai/grok-4-1-fast-non-reasoning` | 131K | $0.20 | $0.50 | chat, tools | +| Azure AI | `azure_ai/grok-4-1-fast-reasoning` | 131K | $0.20 | $0.50 | chat, tools, reasoning | +| Azure AI | `azure_ai/mistral-document-ai-2512` | - | - | - | OCR | +| Dashscope | `dashscope/qwen3-next-80b-a3b-instruct` | 262K | $0.15 | $1.20 | chat | +| Dashscope | `dashscope/qwen3-next-80b-a3b-thinking` | 262K | $0.15 | $1.20 | chat, reasoning | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-instruct` | 131K | $0.40 | $1.60 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-235b-a22b-thinking` | 131K | $0.40 | $4.00 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-32b-instruct` | 131K | $0.16 | $0.64 | chat, vision | +| Dashscope | `dashscope/qwen3-vl-32b-thinking` | 131K | $0.16 | $2.87 | chat, vision, reasoning | +| Dashscope | `dashscope/qwen3-vl-plus` | 260K | - | - | chat, vision | +| Dashscope | `dashscope/qwen3.5-plus` | 992K | - | - | chat | +| Dashscope | `dashscope/qwen3-max-2026-01-23` | 258K | - | - | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1` | 128K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-0528` | 164K | $0.80 | $2.40 | chat, reasoning | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-V3-0324` | 128K | $0.50 | $1.50 | chat | +| Nebius AI | `nebius/deepseek-ai/DeepSeek-R1-Distill-Llama-70B` | 128K | $0.25 | $0.75 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-235B-A22B` | 262K | $0.20 | $0.60 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-32B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-30B-A3B` | 32K | $0.10 | $0.30 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-14B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/Qwen3-4B` | 32K | $0.08 | $0.24 | chat | +| Nebius AI | `nebius/Qwen/QwQ-32B` | 32K | $0.15 | $0.45 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-72B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-32B-Instruct` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-72B-Instruct` | 131K | $0.13 | $0.40 | chat, vision | +| Nebius AI | `nebius/Qwen/Qwen2-VL-7B-Instruct` | 131K | $0.02 | $0.06 | chat, vision | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-405B-Instruct` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Meta-Llama-3.1-8B-Instruct` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/meta-llama/Llama-3.3-70B-Instruct` | 128K | $0.13 | $0.40 | chat | +| Nebius AI | `nebius/meta-llama/Llama-Guard-3-8B` | 128K | $0.02 | $0.06 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.1-Nemotron-Ultra-253B-v1` | 128K | $0.60 | $1.80 | chat | +| Nebius AI | `nebius/nvidia/Llama-3.3-Nemotron-Super-49B-v1` | 131K | $0.10 | $0.40 | chat | +| Nebius AI | `nebius/NousResearch/Hermes-3-Llama-3.1-405B` | 128K | $1.00 | $3.00 | chat | +| Nebius AI | `nebius/google/gemma-3-27b-it` | 128K | $0.06 | $0.20 | chat | +| Nebius AI | `nebius/mistralai/Mistral-Nemo-Instruct-2407` | 128K | $0.04 | $0.12 | chat | +| Nebius AI | `nebius/Qwen/Qwen2.5-Coder-7B` | 32K | $0.01 | $0.03 | chat | +| Nebius AI | `nebius/BAAI/bge-en-icl` | 32K | $0.01 | - | embeddings | +| Nebius AI | `nebius/BAAI/bge-multilingual-gemma2` | 8K | $0.01 | - | embeddings | +| Nebius AI | `nebius/intfloat/e5-mistral-7b-instruct` | 32K | $0.01 | - | embeddings | +| AWS Bedrock | `mistral.devstral-2-123b` | 256K | $0.40 | $2.00 | chat, tools | +| AWS Bedrock | `zai.glm-4.7-flash` | 200K | $0.07 | $0.40 | chat, tools, reasoning | +| ZAI | `zai/glm-5` | 200K | $1.00 | $3.20 | chat, tools, reasoning | +| ZAI | `zai/glm-5-code` | 200K | $1.20 | $5.00 | chat, tools, reasoning | +| OpenRouter | `openrouter/anthropic/claude-sonnet-4.6` | - | - | - | chat | +| OpenRouter | `openrouter/google/gemini-3.1-pro-preview` | - | - | - | chat | +| OpenRouter | `openrouter/openai/gpt-5.1-codex-max` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3-coder-plus` | - | - | - | chat | +| OpenRouter | `openrouter/qwen/qwen3.5-*` (5 models) | - | - | - | chat | +| OpenRouter | `openrouter/z-ai/glm-5` | - | - | - | chat | +| Together AI | `together_ai/Qwen/Qwen3.5-397B-A17B` | - | - | - | chat | +| Perplexity | `perplexity/pplx-embed-v1-0.6b` | 32K | $0.00 | - | embeddings | +| Perplexity | `perplexity/pplx-embed-v1-4b` | 32K | $0.03 | - | embeddings | +| Serper | `serper/search` | - | - | - | search | + +#### Updated Models + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `cache_read_input_token_cost` and `cache_creation_input_token_cost` to Bedrock-hosted Anthropic models (`claude-3-opus`, `claude-3-sonnet`, `claude-3-haiku`, and APAC/EU variants) — prompt caching is now tracked for cost estimation + - Rename `apac.anthropic.claude-sonnet-4-6` → `au.anthropic.claude-sonnet-4-6` to reflect correct regional identifier + +- **[Azure OpenAI](../../docs/providers/azure)** + - Add `supports_none_reasoning_effort` to all `gpt-5.1-chat`, `gpt-5.1-codex`, and `gpt-5.4` variants (global, EU, standard deployments) — allows passing `reasoning_effort: null` to disable reasoning + +- **[Azure OpenAI](../../docs/providers/azure)** — Removed deprecated models + - Remove `azure/gpt-35-turbo-0301` (deprecated 2025-02-13) + - Remove `azure/gpt-35-turbo-0613` (deprecated 2025-02-13) + +#### Features + +- **[OpenAI](../../docs/providers/openai)** + - Day 0 support for `gpt-5.4` and `gpt-5.4-pro` on OpenAI and Azure + +- **[Google Gemini](../../docs/providers/gemini)** + - Add Gemini 3.x model cost map entries — `gemini-3-flash-preview`, `gemini-3.1-pro-preview`, `gemini-3.1-flash-lite-preview`, `gemini-3-pro-image-preview`, `gemini-embedding-2-preview` + - Add Gemini 2.0 Flash and Flash Lite to cost map (re-added with updated pricing) + +- **[Google Vertex AI](../../docs/providers/vertex)** + - Add `gemini-3-flash-preview`, `gemini-3.1-flash-lite-preview`, `gemini-flash-experimental`, and `gemini-embedding-2-preview` to Vertex AI model cost map + +- **[Mistral](../../docs/providers/mistral)** + - Add Magistral reasoning models (`magistral-medium-1-2-2509`, `magistral-small-1-2-2509`) + - Add `mistral-large-2512`, `mistral-medium-3-1-2508`, `mistral-small-3-2-2506`, `ministral-3-*` variants + +- **[Dashscope / Qwen](../../docs/providers/dashscope)** + - Add Qwen3 VL multimodal models (`qwen3-vl-235b`, `qwen3-vl-32b` — instruct and thinking variants) + - Add `qwen3-next-80b-a3b` (instruct + thinking), `qwen3.5-plus`, `qwen3-max-2026-01-23` + +- **[Black Forest Labs](../../docs/providers/black_forest_labs)** + - Add FLUX Kontext image editing models (`flux-kontext-pro`, `flux-kontext-max`) + - Add FLUX Pro 1.0 Fill (inpainting) and Expand (outpainting) + - Add `flux-pro-1.1`, `flux-pro-1.1-ultra`, `flux-dev`, `flux-pro` + +- **[Azure AI](../../docs/providers/azure_ai)** + - Add xAI Grok models via Azure AI Foundry (`grok-4-1-fast-non-reasoning`, `grok-4-1-fast-reasoning`) + - Add Mistral Document AI (`mistral-document-ai-2512`) — OCR mode + +- **[AWS Bedrock](../../docs/providers/bedrock)** + - Add `mistral.devstral-2-123b` (256K context, tools) + - Add `zai.glm-4.7-flash` via Bedrock Converse (200K context, tools, reasoning) + +- **[SageMaker](../../docs/providers/aws_sagemaker)** + - Add `sagemaker_nova` provider for Amazon Nova models on SageMaker - [PR #21542](https://github.com/BerriAI/litellm/pull/21542) + +#### Deprecated / Removed Models + +**OpenAI** — Legacy models removed from cost map: +- `gpt-3.5-turbo-0301`, `gpt-3.5-turbo-0613`, `gpt-3.5-turbo-16k-0613` +- `gpt-4-0314`, `gpt-4-32k`, `gpt-4-32k-0314`, `gpt-4-32k-0613`, `gpt-4-1106-vision-preview`, `gpt-4-vision-preview` +- `gpt-4.5-preview`, `gpt-4.5-preview-2025-02-27` +- `gpt-4o-audio-preview-2024-10-01`, `gpt-4o-realtime-preview-2024-10-01` +- `o1-mini`, `o1-mini-2024-09-12`, `o1-preview`, `o1-preview-2024-09-12` + +**Google Gemini** — Gemini 1.5 and legacy 2.0 variants removed: +- All `gemini-1.5-*` variants (flash, flash-8b, pro, and dated versions) +- `gemini-2.0-flash-exp`, `gemini-2.0-pro-exp-02-05`, `gemini-2.5-flash-preview-04-17`, `gemini-2.5-flash-preview-05-20` + +**Google Vertex AI** — PaLM 2 / legacy models removed: +- All `chat-bison`, `text-bison`, `codechat-bison`, `code-bison`, `code-gecko` variants +- Gemini 1.0 Pro, 1.5 Flash/Pro, 2.0 Flash experimental, and preview variants + +**Perplexity** — Legacy Llama-sonar models removed: +- `llama-3.1-sonar-huge-128k-online`, `llama-3.1-sonar-large/small-128k-chat/online` + +--- + +## LLM API Endpoints + +#### Features + +- **[Responses API](../../docs/response_api)** + - Handle `response.failed`, `response.incomplete`, and `response.cancelled` terminal event types in background streaming — previously only `response.completed` was handled - [PR #23492](https://github.com/BerriAI/litellm/pull/23492) + +#### Bug Fixes + +- **[Anthropic](../../docs/providers/anthropic)** + - Preserve native tool format (web_search, bash, tool_search, etc.) when guardrails convert tools for the Anthropic Messages API - [PR #23526](https://github.com/BerriAI/litellm/pull/23526) + +- **[Moonshot / Kimi](../../docs/providers/openai_compatible)** + - Auto-fill `reasoning_content` for Moonshot Kimi reasoning models - [PR #23580](https://github.com/BerriAI/litellm/pull/23580) + +- **[HuggingFace](../../docs/providers/huggingface)** + - Forward `extra_headers` to HuggingFace embedding API - [PR #23525](https://github.com/BerriAI/litellm/pull/23525) + +- **General** + - Normalize `content_filtered` finish reason across providers - [PR #23564](https://github.com/BerriAI/litellm/pull/23564) + - Fix custom cost tracking on deployments for `/v1/messages` and `/v1/responses` - [PR #23647](https://github.com/BerriAI/litellm/pull/23647) + - Fix per-request custom pricing when `router_model_id` has no pricing data — now falls back to model name + +--- + +## Management Endpoints / UI + +#### Features + +- **Virtual Keys** + - Add Organization dropdown to Create/Edit Key form — `organization_id` is now a first-class field in Key Ownership - [PR #23595](https://github.com/BerriAI/litellm/pull/23595) + - Allow setting `organization_id` on `/key/update` — keys can be assigned or moved to a different organization after creation - [PR #23557](https://github.com/BerriAI/litellm/pull/23557) + +- **Internal Users** + - Add/Remove Team Membership directly from the Internal Users info page — includes searchable dropdown and role selector; no longer requires navigating to each team - [PR #23638](https://github.com/BerriAI/litellm/pull/23638) + +- **Default Team Settings** + - Modernize page to antd (consistent with rest of app) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: default team params (budget, duration, tpm, rpm, permissions) now correctly applied on `/team/new` - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: settings persist across proxy restarts (`default_team_params` added to `LITELLM_SETTINGS_SAFE_DB_OVERRIDES`) - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + - Fix: resolved race condition in `_update_litellm_setting` where `get_config()` could overwrite freshly saved values - [PR #23614](https://github.com/BerriAI/litellm/pull/23614) + +- **Usage** + - Auto-paginate daily spend data — all entity views (teams, orgs, customers, tags, agents, users) fetch pages progressively with charts updating after each page - [PR #23622](https://github.com/BerriAI/litellm/pull/23622) + +- **Models / Cost** + - Azure Model Router cost breakdown in UI — show per-sub-model `additional_costs` from `hidden_params` in `CostBreakdownViewer` - [PR #23550](https://github.com/BerriAI/litellm/pull/23550) + +- **User Management** + - New `/user/info/v2` endpoint — scoped, paginated replacement for the existing god endpoint that caused memory and stability issues on large installs - [PR #23437](https://github.com/BerriAI/litellm/pull/23437) + +#### Bugs + +- Fix Tag list endpoint returning 500 due to invalid Prisma `group_by` kwargs - [PR #23606](https://github.com/BerriAI/litellm/pull/23606) +- Fix Team Admin getting 403 on `/user/filter/ui` when `scope_user_search_to_org` is enabled - [PR #23671](https://github.com/BerriAI/litellm/pull/23671) +- Fix Public Model Hub not showing config-defined models after save - [PR #23501](https://github.com/BerriAI/litellm/pull/23501) +- Fix fallback popup model dropdown z-index issue - [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +- Fix double-counting bug in org/team key limit checks on `/key/update` + +--- + +## AI Integrations + +### Logging + +- **[Vantage](https://vantage.sh)** + - Add Vantage integration for FOCUS 1.2 CSV export — export LiteLLM proxy spend data as FinOps Open Cost & Usage Specification reports, with time-windowed filenames to prevent overwrites - [PR #23333](https://github.com/BerriAI/litellm/pull/23333) + +- **General** + - Fix silent metrics race condition causing metric collision across experiments - [PR #23542](https://github.com/BerriAI/litellm/pull/23542) + +### Guardrails + +No major guardrail changes in this release. + +### Prompt Management + +No major prompt management changes in this release. + +### Secret Managers + +No major secret manager changes in this release. + +--- + +## Performance / Loadbalancing / Reliability improvements + +- **Fix streaming crashes after ~1 hour** — `LLMClientCache._remove_key()` no longer calls `close()`/`aclose()` on evicted HTTP/SDK clients. In-flight requests were crashing with `RuntimeError: Cannot send a request, as the client has been closed.` after the 1-hour TTL expired. Cleanup now happens only at shutdown via `close_litellm_async_clients()` - [PR #22926](https://github.com/BerriAI/litellm/pull/22926) +- **Fix OOM / Prisma connection loss** on large installs — unbounded managed-object poll was exhausting Prisma connections after ~60–70 minutes on instances with 336K+ queued response rows - [PR #23472](https://github.com/BerriAI/litellm/pull/23472) +- **Centralize logging kwarg updates** — root cause fix migrating all logging updates to a single function, eliminating kwarg inconsistencies across logging paths - [PR #23659](https://github.com/BerriAI/litellm/pull/23659) +- **Fix tiktoken cache for non-root offline containers** — tiktoken cache now works correctly in offline environments running as non-root users - [PR #23498](https://github.com/BerriAI/litellm/pull/23498) +- **Add CodSpeed continuous performance benchmarks** — automated performance regression tracking on CI - [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Security + +- **Secret redaction in proxy logs** — Adds a `SecretRedactionFilter` to all LiteLLM loggers that scrubs API keys, tokens, and credentials from log messages, format args, exception tracebacks, and extra fields. Enabled by default; opt out with `LITELLM_DISABLE_REDACT_SECRETS=true` - [PR #23668](https://github.com/BerriAI/litellm/pull/23668), [PR #23667](https://github.com/BerriAI/litellm/pull/23667) +- **Bump PyJWT to `^2.12.0`** — addresses security vulnerability in `^2.10.1` - [PR #23678](https://github.com/BerriAI/litellm/pull/23678) +- **Bump `tar` to 7.5.11 and `tornado` to 6.5.5** — addresses CVEs in transitive dependencies - [PR #23602](https://github.com/BerriAI/litellm/pull/23602) + +--- + +## Database / Proxy Operations + +- **Fix Prisma migrate deploy on pre-existing instances** — resolved multiple bugs in migration recovery logic: missing return in the P3018 idempotent error handler and unhandled exceptions in `_roll_back_migration` that caused silent failures even after successful recovery - [PR #23655](https://github.com/BerriAI/litellm/pull/23655) +- **Make DB migration failure exit opt-in** — proxy no longer exits on `prisma migrate deploy` failure by default; enable with `--enforce_prisma_migration_check` - [PR #23675](https://github.com/BerriAI/litellm/pull/23675) + +--- + +## New Contributors + +* @ryanh-ai made their first contribution in [PR #21542](https://github.com/BerriAI/litellm/pull/21542) +* @ryan-crabbe made their first contribution in [PR #23668](https://github.com/BerriAI/litellm/pull/23668) +* @Jah-yee made their first contribution in [PR #23525](https://github.com/BerriAI/litellm/pull/23525) +* @gambletan made their first contribution in [PR #23516](https://github.com/BerriAI/litellm/pull/23516) +* @awais786 made their first contribution in [PR #23183](https://github.com/BerriAI/litellm/pull/23183) +* @pradyyadav made their first contribution in [PR #23580](https://github.com/BerriAI/litellm/pull/23580) +* @xianzongxie-stripe made their first contribution in [PR #23492](https://github.com/BerriAI/litellm/pull/23492) +* @Harshit28j made their first contribution in [PR #23333](https://github.com/BerriAI/litellm/pull/23333) +* @codspeed-hq[bot] made their first contribution in [PR #23676](https://github.com/BerriAI/litellm/pull/23676) + +--- + +## Diff Summary + +## 03/16/2026 +* New Providers: 5 +* New Models / Updated Models: 116 new, 132 removed +* LLM API Endpoints: 5 +* Management Endpoints / UI: 11 +* AI Integrations: 2 +* Performance / Reliability: 5 +* Security: 3 +* Database / Proxy Operations: 2 + +--- + +## Full Changelog +[v1.82.0-stable...v1.82.3-stable](https://github.com/BerriAI/litellm/compare/v1.82.0-stable...v1.82.3-stable) diff --git a/docs/my-website/sidebars-release-notes.js b/docs/my-website/sidebars-release-notes.js new file mode 100644 index 00000000000..6ed29003ce1 --- /dev/null +++ b/docs/my-website/sidebars-release-notes.js @@ -0,0 +1,14 @@ +// @ts-check + +/** @type {import('@docusaurus/plugin-content-docs').SidebarsConfig} */ +const sidebars = { + releaseNotesSidebar: [ + { type: 'doc', id: 'index', label: 'Release Notes' }, + { + type: 'autogenerated', + dirName: '.', + }, + ], +}; + +module.exports = sidebars; diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 9d95ce38f7a..7db72da276a 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -20,6 +20,7 @@ const sidebars = { { type: "category", label: "Observability", + link: { type: "doc", id: "integrations/observability_index" }, items: [ { type: "category", @@ -29,80 +30,55 @@ const sidebars = { type: "autogenerated", dirName: "contribute_integration" } - ] + ], }, { type: "autogenerated", dirName: "observability" - } - ], - }, - { - type: "category", - label: "Guardrails", - items: [ - "proxy/guardrails/quick_start", - "proxy/guardrails/team_based_guardrails", - "proxy/guardrails/guardrail_load_balancing", - "proxy/guardrails/test_playground", - "proxy/guardrails/litellm_content_filter", - "proxy/guardrails/realtime_guardrails", - { - type: "category", - label: "Providers", - items: [ - ...[ - "proxy/guardrails/qualifire", - "proxy/guardrails/aim_security", - "proxy/guardrails/onyx_security", - "proxy/guardrails/aporia_api", - "proxy/guardrails/azure_content_guardrail", - "proxy/guardrails/bedrock", - "proxy/guardrails/crowdstrike_aidr", - "proxy/guardrails/enkryptai", - "proxy/guardrails/ibm_guardrails", - "proxy/guardrails/grayswan", - "proxy/guardrails/hiddenlayer", - "proxy/guardrails/lasso_security", - "proxy/guardrails/guardrails_ai", - "proxy/guardrails/lakera_ai", - "proxy/guardrails/model_armor", - "proxy/guardrails/noma_security", - "proxy/guardrails/dynamoai", - "proxy/guardrails/openai_moderation", - "proxy/guardrails/pangea", - "proxy/guardrails/pillar_security", - "proxy/guardrails/pii_masking_v2", - "proxy/guardrails/panw_prisma_airs", - "proxy/guardrails/secret_detection", - "proxy/guardrails/custom_guardrail", - "proxy/guardrails/custom_code_guardrail", - "proxy/guardrails/prompt_injection", - "proxy/guardrails/tool_permission", - "proxy/guardrails/zscaler_ai_guard", - "proxy/guardrails/javelin" - ].sort(), - ], - }, - { - type: "category", - label: "Contributing to Guardrails", - items: [ - "adding_provider/generic_guardrail_api", - "adding_provider/simple_guardrail_tutorial", - "adding_provider/adding_guardrail_support", - ] }, ], }, { type: "category", - label: "Policies", + label: "Guardrail Providers", + link: { + type: "generated-index", + title: "Guardrail Providers", + description: "Add safety and content filtering to LLM calls", + slug: "/guardrail_providers" + }, items: [ - "proxy/guardrails/guardrail_policies", - "proxy/guardrails/policy_flow_builder", - "proxy/guardrails/policy_templates", - "proxy/guardrails/policy_tags", + ...[ + "proxy/guardrails/qualifire", + "proxy/guardrails/aim_security", + "proxy/guardrails/onyx_security", + "proxy/guardrails/aporia_api", + "proxy/guardrails/azure_content_guardrail", + "proxy/guardrails/bedrock", + "proxy/guardrails/crowdstrike_aidr", + "proxy/guardrails/enkryptai", + "proxy/guardrails/ibm_guardrails", + "proxy/guardrails/grayswan", + "proxy/guardrails/hiddenlayer", + "proxy/guardrails/lasso_security", + "proxy/guardrails/guardrails_ai", + "proxy/guardrails/lakera_ai", + "proxy/guardrails/model_armor", + "proxy/guardrails/noma_security", + "proxy/guardrails/dynamoai", + "proxy/guardrails/openai_moderation", + "proxy/guardrails/pangea", + "proxy/guardrails/pillar_security", + "proxy/guardrails/pii_masking_v2", + "proxy/guardrails/panw_prisma_airs", + "proxy/guardrails/secret_detection", + "proxy/guardrails/custom_guardrail", + "proxy/guardrails/custom_code_guardrail", + "proxy/guardrails/prompt_injection", + "proxy/guardrails/tool_permission", + "proxy/guardrails/zscaler_ai_guard", + "proxy/guardrails/javelin" + ].sort(), ], }, { @@ -115,11 +91,6 @@ const sidebars = { "proxy/pyroscope_profiling" ] }, - { - type: "doc", - id: "integrations/websearch_interception", - label: "Web Search Integration" - }, { type: "category", label: "[Beta] Prompt Management", @@ -140,7 +111,7 @@ const sidebars = { }, { type: "category", - label: "AI Tools (OpenWebUI, Claude Code, etc.)", + label: "AI Tools", link: { type: "generated-index", title: "AI Tools", @@ -167,14 +138,13 @@ const sidebars = { }, "tutorials/opencode_integration", "tutorials/openclaw_integration", - "tutorials/cost_tracking_coding", "tutorials/cursor_integration", "tutorials/github_copilot_integration", "tutorials/litellm_gemini_cli", - "tutorials/google_genai_sdk", "tutorials/litellm_qwen_code_cli", "tutorials/openai_codex", - "tutorials/retool_assist" + "tutorials/retool_assist", + "tutorials/cost_tracking_coding" ] }, { @@ -191,7 +161,11 @@ const sidebars = { "tutorials/claude_agent_sdk", "tutorials/copilotkit_sdk", "tutorials/google_adk", + "tutorials/google_genai_sdk", "tutorials/livekit_xai_realtime", + "integrations/letta", + { type: "doc", id: "tutorials/instructor", label: "Instructor with LiteLLM" }, + { type: "doc", id: "langchain/langchain", label: "LangChain with LiteLLM" }, "projects/openai-agents" ] }, @@ -212,7 +186,20 @@ const sidebars = { ], // But you can create a sidebar manually tutorialSidebar: [ - { type: "doc", id: "index", label: "Getting Started" }, + // ════════════════════════════════════════════════════════════ + // GET STARTED + // ════════════════════════════════════════════════════════════ + { + type: "category", + label: "Get Started", + collapsible: false, + collapsed: false, + items: [ + { type: "doc", id: "index", label: "Quickstart" }, + { type: "link", label: "Models & Pricing", href: "https://models.litellm.ai" }, + { type: "link", label: "Changelog", href: "/release_notes" }, + ], + }, { type: "category", @@ -280,11 +267,6 @@ const sidebars = { }, "completion/token_usage", "exception_mapping", - { - type: "category", - label: "LangChain, LlamaIndex, Instructor", - items: ["langchain/langchain", "tutorials/instructor"], - } ], }, { @@ -297,16 +279,46 @@ const sidebars = { slug: "/simple_proxy", }, items: [ - "proxy/docker_quick_start", + { type: "doc", id: "proxy/docker_quick_start", label: "Getting Started Tutorial" }, { - type: "link", - label: "A2A Agent Gateway", - href: "https://docs.litellm.ai/docs/a2a", - }, - { - type: "link", - label: "MCP Gateway", - href: "https://docs.litellm.ai/docs/mcp", + type: "category", + label: "Agent & MCP Gateway", + items: [ + { + type: "category", + label: "A2A Agent Gateway", + items: [ + "a2a", + "a2a_invoking_agents", + "a2a_agent_headers", + "a2a_cost_tracking", + "a2a_agent_permissions", + "a2a_iteration_budgets", + ], + }, + { + type: "category", + label: "MCP Gateway", + items: [ + "mcp", + "mcp_usage", + "mcp_openapi", + "mcp_oauth", + "mcp_aws_sigv4", + "mcp_zero_trust", + "mcp_public_internet", + "mcp_semantic_filter", + "mcp_control", + "mcp_cost", + "mcp_guardrail", + { + type: "link", + label: "MCP Troubleshooting Guide", + href: "/docs/mcp_troubleshoot" + }, + ], + }, + ], }, { "type": "category", @@ -365,7 +377,11 @@ const sidebars = { type: "category", label: "Teams & Organizations", items: [ - "proxy/access_control", + { + type: "link", + label: "Role-based Access Controls (RBAC) →", + href: "/docs/proxy/access_control" + }, "proxy/self_serve", "proxy/public_teams", "proxy/ui_project_management", @@ -448,14 +464,40 @@ const sidebars = { }, "proxy/caching", { - type: "link", + type: "category", label: "Guardrails", - href: "https://docs.litellm.ai/docs/proxy/guardrails/quick_start", + items: [ + "proxy/guardrails/quick_start", + "proxy/guardrails/team_based_guardrails", + "proxy/guardrails/guardrail_load_balancing", + "proxy/guardrails/test_playground", + "proxy/guardrails/litellm_content_filter", + "proxy/guardrails/realtime_guardrails", + { + type: "link", + label: "Providers →", + href: "/docs/guardrail_providers", + }, + { + type: "category", + label: "Contributing to Guardrails", + items: [ + "adding_provider/generic_guardrail_api", + "adding_provider/simple_guardrail_tutorial", + "adding_provider/adding_guardrail_support", + ] + }, + ], }, { - type: "link", + type: "category", label: "Policies", - href: "https://docs.litellm.ai/docs/proxy/guardrails/guardrail_policies", + items: [ + "proxy/guardrails/guardrail_policies", + "proxy/guardrails/policy_flow_builder", + "proxy/guardrails/policy_templates", + "proxy/guardrails/policy_tags", + ], }, { type: "category", @@ -551,16 +593,9 @@ const sidebars = { }, items: [ { - type: "category", + type: "link", label: "/a2a - A2A Agent Gateway", - items: [ - "a2a", - "a2a_invoking_agents", - "a2a_agent_headers", - "a2a_cost_tracking", - "a2a_agent_permissions", - "a2a_iteration_budgets" - ], + href: "/docs/a2a", }, "assistants", "audio_transcription", @@ -975,51 +1010,14 @@ const sidebars = { "providers/zai", ], }, - { - type: "category", - label: "Guides", - items: [ - "budget_manager", - "completion/computer_use", - "completion/web_search", - "completion/web_fetch", - "completion/function_call", - "completion/audio", - "completion/document_understanding", - "completion/drop_params", - "completion/image_generation_chat", - "completion/json_mode", - "completion/knowledgebase", - "providers/anthropic_tool_search", - "guides/code_interpreter", - "completion/message_trimming", - "completion/message_sanitization", - "completion/model_alias", - "completion/mock_requests", - "completion/predict_outputs", - "completion/prefix", - "completion/prompt_caching", - "completion/prompt_formatting", - "completion/reliable_completions", - "completion/stream", - "completion/provider_specific_params", - "completion/vision", - "exception_mapping", - "completion/batching", - "guides/finetuned_models", - "guides/security_settings", - "proxy/veo_video_generation", - "reasoning_content", - "extras/creating_adapters", - ] - }, + { type: "category", - label: "Routing, Loadbalancing & Fallbacks", + label: "Routing & Load Balancing", link: { type: "generated-index", - title: "Routing, Loadbalancing & Fallbacks", + title: "Routing & Load Balancing", description: "Learn how to load balance, route, and set fallbacks for your LLM requests", slug: "/routing-load-balancing", }, @@ -1047,43 +1045,6 @@ const sidebars = { "load_test_rpm", ] }, - { - type: "category", - label: "Tutorials", - items: [ - { - type: "link", - label: "AI Coding Tools (OpenWebUI, Claude Code, Gemini CLI, OpenAI Codex, etc.)", - href: "/docs/ai_tools", - }, - "tutorials/anthropic_file_usage", - "tutorials/default_team_self_serve", - "tutorials/msft_sso", - "tutorials/prompt_caching", - "tutorials/tag_management", - 'tutorials/litellm_proxy_aporia', - "tutorials/presidio_pii_masking", - "tutorials/elasticsearch_logging", - "tutorials/gemini_realtime_with_audio", - "tutorials/claude_code_beta_headers", - { - type: "category", - label: "LiteLLM Python SDK Tutorials", - items: [ - 'tutorials/azure_openai', - 'tutorials/instructor', - "tutorials/gradio_integration", - "tutorials/huggingface_codellama", - "tutorials/huggingface_tutorial", - "tutorials/TogetherAI_liteLLM", - "tutorials/finetuned_chat_gpt", - "tutorials/text_completion", - "tutorials/first_playground", - "tutorials/model_fallbacks", - ], - }, - ] - }, { type: "category", label: "Contributing", @@ -1140,12 +1101,11 @@ const sidebars = { "projects/Codium PR Agent", "projects/Prompt2Model", "projects/SalesGPT", + "projects/Softgen", "projects/Quivr", "projects/Langstream", "projects/Otter", - "projects/GPT Migrate", "projects/YiVal", - "projects/LiteLLM Proxy", "projects/llm_cord", "projects/pgai", "projects/GPTLocalhost", @@ -1183,23 +1143,309 @@ const sidebars = { "troubleshoot", ], }, + ], +}; + +const learnSidebar = { + learnSidebar: [ + // ── Landing page ────────────────────────────────────────────────── + { type: "doc", id: "learn/index", label: "Learn" }, { type: "category", - label: "Blog", + label: "Start Here", + collapsible: true, + collapsed: false, + items: [ + "learn/sdk_quickstart", + "learn/gateway_quickstart", + ], + }, + + // ── Guides ──────────────────────────────────────────────────────── + { + type: "category", + label: "Guides", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "guides/index" }, items: [ { - type: "link", - label: "Day 0 Support: Claude Sonnet 4.6", - href: "/blog/claude_sonnet_4_6", + type: "category", + label: "Core Requests", + collapsible: true, + collapsed: false, + link: { + type: "generated-index", + title: "Core Requests", + description: "Streaming, batching, structured outputs, and reasoning behavior", + slug: "/guides/core_request_response_patterns" + }, + items: [ + "completion/stream", + "completion/batching", + "completion/json_mode", + "reasoning_content", + ], + }, + { + type: "category", + label: "Tool Calling", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Tool Calling", + description: "Function calling, web tools, interception patterns, computer use, code interpreter, and tool-call hygiene", + slug: "/guides/tools_integrations" + }, + items: [ + "completion/function_call", + "completion/web_search", + { + type: "doc", + id: "integrations/websearch_interception", + label: "Web Search Interception", + }, + "completion/web_fetch", + "completion/computer_use", + "guides/code_interpreter", + "completion/message_sanitization", + ], + }, + { + type: "category", + label: "Multimodal I/O", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Multimodal I/O", + description: "Vision, audio, PDFs, image generation, and video generation", + slug: "/guides/multimodal_io" + }, + items: [ + "completion/vision", + "completion/audio", + "completion/document_understanding", + "completion/image_generation_chat", + "proxy/veo_video_generation", + ], + }, + { + type: "category", + label: "Retrieval & Knowledge", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Retrieval & Knowledge", + description: "Vector stores, file search, citations, and knowledge-base routing", + slug: "/guides/retrieval_knowledge" + }, + items: [ + "completion/knowledgebase", + ], + }, + { + type: "category", + label: "Prompts & Context", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Prompts & Context", + description: "Prompt caching, trimming, formatting, assistant prefill, and predicted outputs", + slug: "/guides/prompts_context" + }, + items: [ + "completion/prefix", + "completion/predict_outputs", + "completion/message_trimming", + "completion/prompt_caching", + "completion/prompt_formatting", + ], + }, + { + type: "category", + label: "Compatibility & Extensibility", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Compatibility & Extensibility", + description: "Provider-specific params, model aliases, fine-tuned models, and adapters", + slug: "/guides/compatibility_extensibility" + }, + items: [ + "completion/provider_specific_params", + "completion/drop_params", + "completion/model_alias", + "guides/finetuned_models", + "extras/creating_adapters", + ], + }, + { + type: "category", + label: "Reliability, Testing & Spend", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Reliability, Testing & Spend", + description: "Retries, fallbacks, mock responses, and budget controls", + slug: "/guides/reliability_testing_spend" + }, + items: [ + "completion/mock_requests", + "completion/reliable_completions", + "budget_manager", + ], + }, + { + type: "category", + label: "Security & Network", + collapsible: true, + collapsed: true, + link: { + type: "generated-index", + title: "Security & Network", + description: "SSL, custom CA bundles, HTTP proxy settings, and per-service verification", + slug: "/guides/security_network" + }, + items: [ + "guides/security_settings", + ], + }, + ], + }, + + // ── Tutorials ───────────────────────────────────────────────────── + { + type: "category", + label: "Tutorials", + collapsible: true, + collapsed: false, + link: { type: "doc", id: "tutorials/index" }, + items: [ + { + type: "category", + label: "Getting Started", + collapsed: false, + link: { + type: "generated-index", + title: "Getting Started", + description: "Installation, playground, text completion, and mock completions", + slug: "/tutorials/getting_started" + }, + items: [ + "tutorials/installation", + "tutorials/first_playground", + "tutorials/text_completion", + "tutorials/mock_completion", + ], }, { type: "link", - label: "Incident: Broken Model Cost Map", - href: "/blog/model-cost-map-incident", + label: "Agent SDKs & Frameworks", + href: "/docs/agent_sdks", + }, + { + type: "link", + label: "AI Coding Tools", + href: "/docs/ai_tools", + }, + { + type: "category", + label: "Python SDK", + collapsed: true, + link: { + type: "generated-index", + title: "Python SDK", + description: "Tutorials using only the Python SDK — no proxy server required", + slug: "/tutorials/python_sdk" + }, + items: [ + "tutorials/gradio_integration", + "tutorials/provider_specific_params", + "tutorials/model_fallbacks", + "tutorials/fallbacks", + ], + }, + { + type: "category", + label: "Provider Setup", + collapsed: true, + link: { + type: "generated-index", + title: "Provider Setup", + description: "Connect LiteLLM to Azure OpenAI, HuggingFace, TogetherAI, local models, and more", + slug: "/tutorials/provider_tutorials" + }, + items: [ + "tutorials/azure_openai", + "tutorials/TogetherAI_liteLLM", + "tutorials/huggingface_tutorial", + "tutorials/huggingface_codellama", + "tutorials/finetuned_chat_gpt", + "tutorials/oobabooga", + ], + }, + { + type: "category", + label: "Proxy: Admin & Access", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Admin & Access", + description: "User and team management, SSO, SCIM, and routing rules", + slug: "/tutorials/proxy_admin_access" + }, + items: [ + "tutorials/default_team_self_serve", + "tutorials/msft_sso", + "tutorials/scim_litellm", + "tutorials/tag_management", + ], + }, + { + type: "category", + label: "Proxy: Features & Safety", + collapsed: true, + link: { + type: "generated-index", + title: "Proxy: Features & Safety", + description: "Prompt caching, passthrough APIs, realtime, guardrails, and PII masking", + slug: "/tutorials/proxy_features_safety" + }, + items: [ + "tutorials/prompt_caching", + "tutorials/anthropic_file_usage", + "tutorials/gemini_realtime_with_audio", + "tutorials/litellm_proxy_aporia", + "tutorials/presidio_pii_masking", + ], + }, + { + type: "category", + label: "Observability & Evaluation", + collapsed: true, + link: { + type: "generated-index", + title: "Observability & Evaluation", + description: "Logging, monitoring, benchmarking, and evaluation suites", + slug: "/tutorials/observability_evaluation" + }, + items: [ + "tutorials/elasticsearch_logging", + "tutorials/compare_llms", + "tutorials/litellm_Test_Multiple_Providers", + "tutorials/eval_suites", + "tutorials/lm_evaluation_harness", + ], }, ], }, ], }; -module.exports = sidebars; +module.exports = { ...sidebars, ...learnSidebar }; diff --git a/docs/my-website/src/components/NavigationCards/index.js b/docs/my-website/src/components/NavigationCards/index.js new file mode 100644 index 00000000000..5efd89ee140 --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/index.js @@ -0,0 +1,44 @@ +import React from 'react'; +import Link from '@docusaurus/Link'; +import styles from './styles.module.css'; + +export default function NavigationCards({ items, columns = 2 }) { + return ( +
+ {items.map((item, i) => { + const isExternal = + item.to && (item.to.startsWith('http://') || item.to.startsWith('https://')); + return ( + + {item.icon && ( +
{item.icon}
+ )} +
{item.title}
+ {item.description && ( +
{item.description}
+ )} + {item.listDescription && ( +
    + {item.listDescription.map((line, j) => ( +
  • {line}
    • + ))} +
+ )} + {isExternal && ( + ↗ + )} + + ); + })} +
+ ); +} diff --git a/docs/my-website/src/components/NavigationCards/styles.module.css b/docs/my-website/src/components/NavigationCards/styles.module.css new file mode 100644 index 00000000000..64f5a42374b --- /dev/null +++ b/docs/my-website/src/components/NavigationCards/styles.module.css @@ -0,0 +1,82 @@ +.grid { + display: grid; + grid-template-columns: repeat(var(--nav-columns, 2), 1fr); + gap: 0.75rem; + margin: 1.25rem 0; +} + +@media (max-width: 768px) { + .grid { + grid-template-columns: 1fr; + } +} + +.card { + position: relative; + display: flex; + flex-direction: column; + padding: 1rem 1.1rem; + border: 1px solid var(--ifm-color-emphasis-200); + border-radius: 6px; + text-decoration: none !important; + color: inherit !important; + background: var(--ifm-background-surface-color); + transition: border-color 0.15s ease, box-shadow 0.15s ease; +} + +.card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); + text-decoration: none !important; +} + +[data-theme='dark'] .card { + background: var(--ifm-background-surface-color); + border-color: #2d3748; +} + +[data-theme='dark'] .card:hover { + border-color: var(--ifm-color-primary); + box-shadow: 0 0 0 1px var(--ifm-color-primary); +} + +.icon { + font-size: 1.4rem; + margin-bottom: 0.5rem; + line-height: 1; +} + +.title { + font-size: 14px; + font-weight: 600; + margin-bottom: 0.35rem; + color: var(--ifm-heading-color); +} + +.description { + font-size: 13px; + line-height: 1.5; + color: var(--ifm-color-emphasis-700); + margin-bottom: 0.5rem; +} + +.list { + margin: 0.35rem 0 0 0; + padding-left: 1.1rem; + list-style: disc; +} + +.list li { + font-size: 12.5px; + color: var(--ifm-color-emphasis-700); + line-height: 1.6; + margin-bottom: 0; +} + +.externalIcon { + position: absolute; + top: 0.75rem; + right: 0.75rem; + font-size: 12px; + color: var(--ifm-color-emphasis-500); +} diff --git a/docs/my-website/src/css/custom.css b/docs/my-website/src/css/custom.css index 9fa4443afc9..d0702fcc57c 100644 --- a/docs/my-website/src/css/custom.css +++ b/docs/my-website/src/css/custom.css @@ -1,10 +1,16 @@ /** - * Any CSS included here will be global. The classic template - * bundles Infima by default. Infima is a CSS framework designed to - * work well for content-centric websites. + * Global CSS overrides for LiteLLM docs. + * Infima (Docusaurus CSS framework) variables + custom styling. */ -/* You can override the default Infima variables here. */ +/* ========================================= + FONTS + ========================================= */ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600;700&display=swap'); + +/* ========================================= + ROOT — Light Mode Variables + ========================================= */ :root { --ifm-color-primary: #2e8555; --ifm-color-primary-dark: #29784c; @@ -13,11 +19,22 @@ --ifm-color-primary-light: #33925d; --ifm-color-primary-lighter: #359962; --ifm-color-primary-lightest: #3cad6e; - --ifm-code-font-size: 95%; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.1); + --ifm-code-font-size: 85%; + --ifm-menu-color: #6b7280; + --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.08); + --ifm-font-family-base: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; + --ifm-heading-font-weight: 600; + --ifm-font-size-base: 15px; + --ifm-line-height-base: 1.65; + --ifm-border-radius: 6px; + /* Wider reading column — reduces excessive whitespace on large monitors */ + --ifm-container-width: 1380px; + --ifm-container-width-xl: 1560px; } -/* For readability concerns, you should choose a lighter palette in dark mode. */ +/* ========================================= + DARK MODE Variables + ========================================= */ [data-theme='dark'] { --ifm-color-primary: #25c2a0; --ifm-color-primary-dark: #21af90; @@ -26,10 +43,710 @@ --ifm-color-primary-light: #29d5b0; --ifm-color-primary-lighter: #32d8b4; --ifm-color-primary-lightest: #4fddbf; - --docusaurus-highlighted-code-line-bg: rgba(0, 0, 0, 0.3); + --ifm-background-color: #0d1117; + --ifm-background-surface-color: #161b22; + --docusaurus-highlighted-code-line-bg: rgba(255, 255, 255, 0.07); } -/* Levo logo sizing and theme switching */ +/* ========================================= + TYPOGRAPHY + ========================================= */ +.theme-doc-markdown h1 { + font-size: 2.2rem; + letter-spacing: -0.02em; + line-height: 1.2; +} + +.theme-doc-markdown h2 { + font-size: 1.6rem; + letter-spacing: -0.01em; + line-height: 1.3; +} + +.theme-doc-markdown h3 { + font-size: 1.25rem; + line-height: 1.4; +} + +.theme-doc-markdown p, +.theme-doc-markdown ul, +.theme-doc-markdown ol { + font-size: 0.9rem; +} + +.theme-doc-markdown table { + font-size: 0.875rem; +} + +.theme-doc-markdown td { + font-size: 0.85rem; +} + +/* ========================================= + NAVBAR + ========================================= */ +[data-theme='light'] .navbar { + background-color: #ffffff; + box-shadow: 0 1px 0 0 #e5e7eb; +} + +[data-theme='dark'] .navbar { + background-color: var(--ifm-background-color); + border-bottom: 1px solid #21262d; + box-shadow: none; +} + +.navbar__link { + font-weight: 400 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; + padding-bottom: 2px; +} + +.navbar__link--active { + font-weight: 500 !important; + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +@media (max-width: 1330px) { + .navbar__link { + font-size: 13px !important; + } +} + +/* Three-column navbar: logo | center nav | right icons */ +@media (min-width: 997px) { + .navbar__inner { + display: flex !important; + align-items: center; + justify-content: space-between; + } + + .navbar__brand-col { + display: flex; + align-items: center; + flex: 0 0 auto; + margin-left: 1rem; + } + + .navbar__brand-col .navbar__brand { + font-size: 1.25rem; + } + + .navbar__brand-col .navbar__logo { + height: 2rem; + width: auto; + } + + .navbar__center-col { + display: flex; + align-items: center; + justify-content: center; + flex: 1; + } + + .navbar__right-col { + display: flex; + align-items: center; + flex: 0 0 auto; + gap: 0.25rem; + margin-right: 2rem; + } +} + +/* ========================================= + ALERTS / ADMONITIONS + ========================================= */ +.alert { + padding: 0.75rem 1rem; + font-size: 14px; + border-radius: var(--ifm-border-radius); + border-left-width: 3px; +} + +/* Light mode */ +.alert--info { + --ifm-alert-background-color: #f0f7ff !important; + --ifm-alert-border-color: #2264ab !important; +} + +.alert--success { + --ifm-alert-background-color: #f0fff8 !important; + --ifm-alert-border-color: #09bda8 !important; +} + +.alert--secondary { + --ifm-alert-background-color: #f8fafc !important; + --ifm-alert-border-color: #64748b !important; +} + +.alert--danger { + --ifm-alert-background-color: #fff0f5 !important; + --ifm-alert-border-color: #e11d48 !important; +} + +.alert--warning { + --ifm-alert-background-color: #fffbeb !important; + --ifm-alert-border-color: #d97706 !important; +} + +/* Dark mode */ +[data-theme='dark'] .alert--info { + --ifm-alert-background-color: #0c1e30 !important; + --ifm-alert-border-color: #3b82f6 !important; + color: #bfdbfe !important; +} + +[data-theme='dark'] .alert--success { + --ifm-alert-background-color: #022c22 !important; + --ifm-alert-border-color: #10b981 !important; +} + +[data-theme='dark'] .alert--secondary { + --ifm-alert-background-color: #0f172a !important; + --ifm-alert-border-color: #475569 !important; +} + +[data-theme='dark'] .alert--danger { + --ifm-alert-background-color: #2d0a14 !important; + --ifm-alert-border-color: #f43f5e !important; +} + +[data-theme='dark'] .alert--warning { + --ifm-alert-background-color: #1c1200 !important; + --ifm-alert-border-color: #f59e0b !important; +} + +/* ========================================= + COLLAPSIBLE / DETAILS + ========================================= */ +details { + color: #1d232e; + background-color: #ffffff; + border: 1px solid #e9eef2 !important; + border-radius: var(--ifm-border-radius) !important; + padding: 0.75rem !important; + margin-bottom: 1rem !important; + margin-top: 1.5rem !important; + box-shadow: 0 1px 4px 0 rgba(0, 0, 0, 0.05) !important; +} + +details summary { + font-weight: 600 !important; +} + +details [class*='collapsibleContent'] { + border-top: 1px solid #e2e8f0 !important; +} + +details [class*='collapsibleContent'] p, +details [class*='collapsibleContent'] ul { + font-size: 13px !important; + line-height: 1.75; +} + +[data-theme='dark'] details { + background-color: #1c2130 !important; + color: #e5e7eb !important; + border: 1px solid #2d3748 !important; + box-shadow: 0 1px 6px 0 rgba(0, 0, 0, 0.3) !important; +} + +[data-theme='dark'] details summary { + color: #f3f4f6 !important; +} + +[data-theme='dark'] details [class*='collapsibleContent'] { + border-top: 1px solid #2d3748 !important; +} + +/* ========================================= + TABS + ========================================= */ +.tabs-container > div { + padding: 1rem; + border: 1px solid #e2e8f0; + border-radius: var(--ifm-border-radius); +} + +/* Remove styling from nested tabs containers */ +.tabs-container .tabs-container > div { + padding: 0; + border: none; + border-radius: 0; +} + +[data-theme='dark'] .tabs-container > div { + border-color: #2d3748; +} + +ul.tabs { + border-bottom: 1px solid #e2e8f0 !important; + column-gap: 0.5rem !important; +} + +[data-theme='dark'] ul.tabs { + border-bottom-color: #2d3748 !important; +} + +li.tabs__item { + padding: 0.5rem !important; + font-weight: 500 !important; + font-size: 14px !important; + border-bottom: 2px solid transparent !important; +} + +li.tabs__item--active { + border-bottom: 2px solid var(--ifm-color-primary) !important; +} + +/* ========================================= + CODE BLOCKS + ========================================= */ +.prism-code { + border-radius: var(--ifm-border-radius); + font-size: 12.5px !important; + line-height: 1.6; +} + +[data-theme='dark'] .prism-code { + border: 1px solid #21262d; +} + +[class*='codeLineNumber']::before { + font-size: 12px !important; + display: flex !important; + align-items: center !important; + justify-content: center !important; +} + +[class*='codeBlockTitle'] { + font-size: 13px !important; + font-weight: 500 !important; + padding: 0.5rem 1rem !important; + border-bottom: 1px solid #334155 !important; +} + +.theme-code-block-highlighted-line { + background-color: rgba(0, 0, 0, 0.1) !important; +} + +[data-theme='dark'] .theme-code-block-highlighted-line { + background-color: rgba(255, 255, 255, 0.06) !important; +} + +.theme-code-block-highlighted-line > span { + background-color: transparent !important; +} + +/* ========================================= + SIDEBAR / MENU + ========================================= */ +.menu { + font-weight: 400; + padding: 0.5rem 0.25rem !important; + background-image: radial-gradient(rgba(0, 0, 0, 0.07) 1px, transparent 1px); + background-size: 24px 24px; +} + +[data-theme='dark'] .menu { + background-image: radial-gradient(rgba(255, 255, 255, 0.04) 1px, transparent 1px); + background-size: 24px 24px; +} + +.menu__link { + font-size: 14px; + padding: 0.22rem 0.75rem !important; + border-radius: 4px; +} + +.menu__link--active { + font-weight: 600 !important; + background-color: rgba(46, 133, 85, 0.08) !important; +} + +[data-theme='dark'] .menu__link--active { + background-color: rgba(37, 194, 160, 0.1) !important; +} + +/* ─── Sidebar collapse arrows — uniform size & alignment ────── */ + +/* 1. Categories WITHOUT a link prop: + The button itself holds the text + ::after arrow. + Make it flex so the arrow never wraps to a new line. */ +.menu__link--sublist-caret { + display: flex !important; + align-items: center !important; + justify-content: space-between !important; + gap: 0.5rem; + padding-right: 0.625rem !important; +} + +.menu__link--sublist-caret::after { + content: '' !important; + display: block !important; + flex-shrink: 0 !important; + width: 1.25rem !important; + height: 1.25rem !important; + min-width: 1.25rem !important; + background: var(--ifm-menu-link-sublist-icon) center / 1.25rem 1.25rem no-repeat !important; + margin: 0 !important; +} + +/* 2. Categories WITH a link prop: + A separate +