From b0a9c85a522bdd3eaa1e6b8fabbc37c0228b4c80 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Dec 2025 07:38:07 +0530 Subject: [PATCH 01/35] fix: fix ruff linting errors --- litellm/a2a_protocol/providers/base.py | 2 +- .../a2a_protocol/providers/pydantic_ai_agents/transformation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/a2a_protocol/providers/base.py b/litellm/a2a_protocol/providers/base.py index 656bc78d6d2..9931076a948 100644 --- a/litellm/a2a_protocol/providers/base.py +++ b/litellm/a2a_protocol/providers/base.py @@ -3,7 +3,7 @@ Base configuration for A2A protocol providers. """ from abc import ABC, abstractmethod -from typing import Any, AsyncIterator, Dict, Optional +from typing import Any, AsyncIterator, Dict class BaseA2AProviderConfig(ABC): diff --git a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py index 67c2154f6c9..6f46933cf9f 100644 --- a/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py +++ b/litellm/a2a_protocol/providers/pydantic_ai_agents/transformation.py @@ -6,7 +6,7 @@ This module provides fake streaming by converting non-streaming responses into s """ import asyncio -from typing import Any, AsyncIterator, Dict, Optional +from typing import Any, AsyncIterator, Dict from uuid import uuid4 import httpx From c754794bc3171d64a52dda8d2f82af963a9432da Mon Sep 17 00:00:00 2001 From: Shivam Rawat <161387515+shivamrawat1@users.noreply.github.com> Date: Mon, 15 Dec 2025 18:08:21 -0800 Subject: [PATCH 02/35] [fix] add qwen3-embedding-8b input per token price (#18018) * added embedding input token price * added embedding input token price --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 26aae425fae..01d7f076edc 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30632,7 +30632,7 @@ "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 0.0, + "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, "litellm_provider": "fireworks_ai", "mode": "embedding" diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 26aae425fae..2a7f8aa3ddf 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -30628,11 +30628,11 @@ "litellm_provider": "fireworks_ai", "mode": "embedding" }, - "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { + "fireworks_ai/accounts/fireworks/models/": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, - "input_cost_per_token": 0.0, + "input_cost_per_token": 1e-07, "output_cost_per_token": 0.0, "litellm_provider": "fireworks_ai", "mode": "embedding" From fabbd28b4578de139e1dceae8d514eccfd4ae0aa Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 18:31:35 -0800 Subject: [PATCH 03/35] Adding Milvus to Vector Store in UI --- .../src/components/vector_store_providers.tsx | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/ui/litellm-dashboard/src/components/vector_store_providers.tsx b/ui/litellm-dashboard/src/components/vector_store_providers.tsx index a6f222da67c..68c5c3745c3 100644 --- a/ui/litellm-dashboard/src/components/vector_store_providers.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_providers.tsx @@ -4,6 +4,7 @@ export enum VectorStoreProviders { VertexRagEngine = "Vertex AI RAG Engine", OpenAI = "OpenAI", Azure = "Azure OpenAI", + Milvus = "Milvus", } export const vectorStoreProviderMap: Record = { @@ -12,6 +13,7 @@ export const vectorStoreProviderMap: Record = { VertexRagEngine: "vertex_ai", OpenAI: "openai", Azure: "azure", + Milvus: "milvus", }; const asset_logos_folder = "../ui/assets/logos/"; @@ -22,6 +24,7 @@ export const vectorStoreProviderLogoMap: Record = { [VectorStoreProviders.VertexRagEngine]: `${asset_logos_folder}google.svg`, [VectorStoreProviders.OpenAI]: `${asset_logos_folder}openai_small.svg`, [VectorStoreProviders.Azure]: `${asset_logos_folder}microsoft_azure.svg`, + [VectorStoreProviders.Milvus]: `${asset_logos_folder}milvus.svg`, }; // Define field types for provider-specific configurations @@ -84,6 +87,25 @@ export const vectorStoreProviderFields: Record type: "text", }, ], + milvus: [ + { + name: "api_key", + label: "API Key", + tooltip: + "To obtain a token, you should use a colon (:) to concatenate the username and password that you use to access your Milvus instance (e.g., username:password)", + placeholder: "username:password or api key", + required: true, + type: "password", + }, + { + name: "api_base", + label: "API Base", + tooltip: "Enter your Milvus endpoint (e.g., https://your-milvus-endpoint.com/)", + placeholder: "https://your-milvus-endpoint.com/", + required: true, + type: "text", + }, + ], }; export const getVectorStoreProviderLogoAndName = (providerValue: string): { logo: string; displayName: string } => { From 0ee924f91e67381bdc5d886768c3596211684391 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 18:34:34 -0800 Subject: [PATCH 04/35] Adding svg --- ui/litellm-dashboard/public/assets/logos/milvus.svg | 1 + 1 file changed, 1 insertion(+) create mode 100644 ui/litellm-dashboard/public/assets/logos/milvus.svg diff --git a/ui/litellm-dashboard/public/assets/logos/milvus.svg b/ui/litellm-dashboard/public/assets/logos/milvus.svg new file mode 100644 index 00000000000..76154467b4b --- /dev/null +++ b/ui/litellm-dashboard/public/assets/logos/milvus.svg @@ -0,0 +1 @@ +milvus-horizontal-color \ No newline at end of file From 8e3eb331655502a00d8a1cff99da9a07ea627d0b Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 18:36:01 -0800 Subject: [PATCH 05/35] Tests --- .../VectorStoreForm.test.tsx | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx new file mode 100644 index 00000000000..7bc948c9127 --- /dev/null +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx @@ -0,0 +1,28 @@ +import { describe, it, expect, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import VectorStoreForm from "./VectorStoreForm"; +import * as networking from "../networking"; +import { CredentialItem } from "../networking"; + +vi.mock("../networking"); + +describe("VectorStoreForm", () => { + it("should render the form when visible", () => { + const mockOnCancel = vi.fn(); + const mockOnSuccess = vi.fn(); + const mockAccessToken = "test-token"; + const mockCredentials: CredentialItem[] = []; + + render( + , + ); + + expect(screen.getByText("Add New Vector Store")).toBeInTheDocument(); + }); +}); From be4f11eaccea5191e6bea84f74f976abb399253d Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 18:38:41 -0800 Subject: [PATCH 06/35] Fixing build --- .../vector_store_management/VectorStoreForm.test.tsx | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx index 7bc948c9127..97d18e33640 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.test.tsx @@ -1,8 +1,7 @@ -import { describe, it, expect, vi } from "vitest"; import { render, screen } from "@testing-library/react"; -import VectorStoreForm from "./VectorStoreForm"; -import * as networking from "../networking"; +import { describe, expect, it, vi } from "vitest"; import { CredentialItem } from "../networking"; +import VectorStoreForm from "./VectorStoreForm"; vi.mock("../networking"); From edfb4148dcf280146ed9c9359c823c35504b68a4 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 16 Dec 2025 08:13:54 +0530 Subject: [PATCH 07/35] Add workflow to create daily staging branches (#18020) Co-authored-by: Cursor Agent --- .../workflows/create_daily_staging_branch.yml | 43 +++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 .github/workflows/create_daily_staging_branch.yml diff --git a/.github/workflows/create_daily_staging_branch.yml b/.github/workflows/create_daily_staging_branch.yml new file mode 100644 index 00000000000..a97cf6f9740 --- /dev/null +++ b/.github/workflows/create_daily_staging_branch.yml @@ -0,0 +1,43 @@ +name: Create Daily Staging Branch + +on: + schedule: + - cron: '0 0 * * *' # Runs daily at midnight UTC + workflow_dispatch: # Allow manual trigger + +jobs: + create-staging-branch: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Create daily staging branch + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Configure Git user + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + # Generate branch name with MM_DD_YYYY format + BRANCH_NAME="litellm_staging_$(date +'%m_%d_%Y')" + echo "Creating branch: $BRANCH_NAME" + + # Fetch all branches + git fetch --all + + # Check if the branch already exists + if git show-ref --verify --quiet refs/remotes/origin/$BRANCH_NAME; then + echo "Branch $BRANCH_NAME already exists. Skipping creation." + else + echo "Creating new branch: $BRANCH_NAME" + # Create the new branch from main + git checkout -b $BRANCH_NAME origin/main + # Push the new branch + git push origin $BRANCH_NAME + echo "Successfully created and pushed branch: $BRANCH_NAME" + fi From df9d3abf49bb76c7a2f94ef52bfd1cb257e9d6d0 Mon Sep 17 00:00:00 2001 From: Dmitrii Komarov Date: Tue, 16 Dec 2025 03:50:09 +0100 Subject: [PATCH 08/35] Propagate token usage when generating images with Gemini (#17987) --- .../vertex_gemini_transformation.py | 26 ++++++++++++++++++- ...rtex_ai_image_generation_transformation.py | 23 +++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py index b9747652362..619bd006300 100644 --- a/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py +++ b/litellm/llms/vertex_ai/image_generation/vertex_gemini_transformation.py @@ -13,7 +13,7 @@ from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ImageObject, ImageResponse +from litellm.types.utils import ImageObject, ImageResponse, ImageUsage, ImageUsageInputTokensDetails if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -234,6 +234,27 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): return request_body + def _transform_image_usage(self, usage: dict) -> ImageUsage: + input_tokens_details = ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ) + tokens_details = usage.get("promptTokensDetails", []) + for details in tokens_details: + if isinstance(details, dict) and (modality := details.get("modality")): + token_count = details.get("tokenCount", 0) + if modality == "TEXT": + input_tokens_details.text_tokens += token_count + elif modality == "IMAGE": + input_tokens_details.image_tokens += token_count + + return ImageUsage( + input_tokens=usage.get("promptTokenCount", 0), + input_tokens_details=input_tokens_details, + output_tokens=usage.get("candidatesTokenCount", 0), + total_tokens=usage.get("totalTokenCount", 0), + ) + def transform_image_generation_response( self, model: str, @@ -276,6 +297,9 @@ class VertexAIGeminiImageGenerationConfig(BaseImageGenerationConfig, VertexLLM): b64_json=inline_data["data"], url=None, )) + + if usage_metadata := response_data.get("usageMetadata", None): + model_response.usage = self._transform_image_usage(usage_metadata) return model_response diff --git a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py index 7cba03c38c8..b91438b3cac 100644 --- a/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/image_generation/test_vertex_ai_image_generation_transformation.py @@ -141,7 +141,22 @@ class TestVertexAIGeminiImageGenerationConfig: ] } } - ] + ], + "usageMetadata": { + "promptTokenCount": 93, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 54, + }, + { + "modality": "IMAGE", + "tokenCount": 39, + } + ], + "candidatesTokenCount": 17, + "totalTokenCount": 110, + } } mock_response.headers = {} @@ -162,6 +177,12 @@ class TestVertexAIGeminiImageGenerationConfig: assert len(result.data) == 1 assert result.data[0].b64_json == "base64_encoded_image_data" assert result.data[0].url is None + assert result.usage.input_tokens == 93 + assert result.usage.input_tokens_details.text_tokens == 54 + assert result.usage.input_tokens_details.image_tokens == 39 + assert result.usage.output_tokens == 17 + assert result.usage.total_tokens == 110 + def test_transform_image_generation_response_multiple_images(self): """Test response transformation with multiple images""" From e83f08c24da1fff7ad5334c433eedf3dfe228f45 Mon Sep 17 00:00:00 2001 From: Doni Crosby Date: Mon, 15 Dec 2025 21:54:16 -0500 Subject: [PATCH 09/35] feat(venice.ai): add support for Venice.ai API via providers.json (#17962) --- litellm/llms/openai_like/providers.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/litellm/llms/openai_like/providers.json b/litellm/llms/openai_like/providers.json index a6c19222619..2d801506d5f 100644 --- a/litellm/llms/openai_like/providers.json +++ b/litellm/llms/openai_like/providers.json @@ -14,5 +14,9 @@ "helicone": { "base_url": "https://ai-gateway.helicone.ai/", "api_key_env": "HELICONE_API_KEY" + }, + "veniceai": { + "base_url": "https://api.venice.ai/api/v1", + "api_key_env": "VENICE_AI_API_KEY" } } From 7c2478b70ef5be2e0596a600b3886421c6dbcc90 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Dec 2025 08:35:39 +0530 Subject: [PATCH 10/35] docs: replace ghcr link with docker.litellm.ai --- deploy/charts/litellm-helm/README.md | 2 +- docs/my-website/docs/index.md | 2 +- docs/my-website/docs/observability/datadog.md | 2 +- docs/my-website/docs/proxy/configs.md | 4 +- docs/my-website/docs/proxy/deploy.md | 50 +++++++++---------- .../docs/proxy/docker_quick_start.md | 6 +-- .../docs/proxy/guardrails/pangea.md | 2 +- .../docs/proxy/shared_health_check.md | 2 +- .../secret_managers/custom_secret_manager.md | 2 +- .../docs/tutorials/elasticsearch_logging.md | 2 +- .../my-website/docs/tutorials/openai_codex.md | 2 +- .../release_notes/v1.55.8-stable/index.md | 2 +- .../my-website/release_notes/v1.57.3/index.md | 2 +- .../release_notes/v1.63.11-stable/index.md | 2 +- .../release_notes/v1.63.14/index.md | 2 +- .../release_notes/v1.65.4-stable/index.md | 2 +- .../release_notes/v1.66.0-stable/index.md | 2 +- .../release_notes/v1.67.4-stable/index.md | 2 +- .../release_notes/v1.68.0-stable/index.md | 2 +- .../release_notes/v1.69.0-stable/index.md | 2 +- .../release_notes/v1.70.1-stable/index.md | 2 +- .../release_notes/v1.71.1-stable/index.md | 2 +- .../release_notes/v1.72.0-stable/index.md | 2 +- .../release_notes/v1.72.2-stable/index.md | 2 +- .../release_notes/v1.72.6-stable/index.md | 2 +- .../release_notes/v1.73.0-stable/index.md | 2 +- .../release_notes/v1.73.6-stable/index.md | 2 +- .../release_notes/v1.74.0-stable/index.md | 2 +- .../release_notes/v1.74.15-stable/index.md | 2 +- .../release_notes/v1.74.3-stable/index.md | 2 +- .../my-website/release_notes/v1.74.7/index.md | 2 +- .../release_notes/v1.74.9-stable/index.md | 2 +- .../release_notes/v1.75.5-stable/index.md | 2 +- .../my-website/release_notes/v1.75.8/index.md | 2 +- .../release_notes/v1.76.1-stable/index.md | 2 +- .../release_notes/v1.76.3-stable/index.md | 2 +- .../release_notes/v1.77.2-stable/index.md | 2 +- .../release_notes/v1.77.3-stable/index.md | 2 +- .../release_notes/v1.77.5-stable/index.md | 2 +- .../release_notes/v1.77.7-stable/index.md | 2 +- .../release_notes/v1.78.0-stable/index.md | 2 +- .../release_notes/v1.78.5-stable/index.md | 2 +- .../release_notes/v1.79.0-stable/index.md | 2 +- .../release_notes/v1.79.1-stable/index.md | 2 +- .../release_notes/v1.79.3-stable/index.md | 2 +- .../release_notes/v1.80.0-stable/index.md | 2 +- .../release_notes/v1.80.10-stable/index.md | 2 +- .../release_notes/v1.80.5-stable/index.md | 2 +- .../release_notes/v1.80.8-stable/index.md | 2 +- docs/my-website/src/pages/index.md | 2 +- 50 files changed, 77 insertions(+), 77 deletions(-) diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 6fdc423a177..2fa856843f3 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -29,7 +29,7 @@ If `db.useStackgresOperator` is used (not yet implemented): | `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | | `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | | `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | +| `image.repository` | LiteLLM Proxy image repository | `docker.litellm.ai/berriai/litellm` | | `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | | `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | | `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | diff --git a/docs/my-website/docs/index.md b/docs/my-website/docs/index.md index f393b300f73..ba605e316d3 100644 --- a/docs/my-website/docs/index.md +++ b/docs/my-website/docs/index.md @@ -657,7 +657,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index b2901650ea6..7cf91ced34c 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -181,7 +181,7 @@ docker run \ -e USE_DDTRACE=true \ -e USE_DDPROFILER=true \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/proxy/configs.md b/docs/my-website/docs/proxy/configs.md index 77ab3158f74..ba4ca190aa9 100644 --- a/docs/my-website/docs/proxy/configs.md +++ b/docs/my-website/docs/proxy/configs.md @@ -655,7 +655,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -e LITELLM_CONFIG_BUCKET_TYPE="gcs" \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-latest --detailed_debug + docker.litellm.ai/berriai/litellm-database:main-latest --detailed_debug ``` @@ -676,7 +676,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_NAME= \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-latest + docker.litellm.ai/berriai/litellm-database:main-latest ``` diff --git a/docs/my-website/docs/proxy/deploy.md b/docs/my-website/docs/proxy/deploy.md index 0f0e5f678d3..abdb9aa3298 100644 --- a/docs/my-website/docs/proxy/deploy.md +++ b/docs/my-website/docs/proxy/deploy.md @@ -57,7 +57,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-stable \ + docker.litellm.ai/berriai/litellm:main-stable \ --config /app/config.yaml --detailed_debug ``` @@ -87,12 +87,12 @@ See all supported CLI args [here](https://docs.litellm.ai/docs/proxy/cli): Here's how you can run the docker image and pass your config to `litellm` ```shell -docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml +docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml ``` Here's how you can run the docker image and start litellm on port 8002 with `num_workers=8` ```shell -docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8 +docker run docker.litellm.ai/berriai/litellm:main-stable --port 8002 --num_workers 8 ``` @@ -100,7 +100,7 @@ docker run ghcr.io/berriai/litellm:main-stable --port 8002 --num_workers 8 ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-stable +FROM docker.litellm.ai/berriai/litellm:main-stable # Set the working directory to /app WORKDIR /app @@ -242,7 +242,7 @@ spec: spec: containers: - name: litellm - image: ghcr.io/berriai/litellm:main-stable # it is recommended to fix a version generally + image: docker.litellm.ai/berriai/litellm:main-stable # it is recommended to fix a version generally args: - "--config" - "/app/proxy_server_config.yaml" @@ -279,9 +279,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart #### Step 1. Pull the litellm helm chart ```bash -helm pull oci://ghcr.io/berriai/litellm-helm +helm pull oci://docker.litellm.ai/berriai/litellm-helm -# Pulled: ghcr.io/berriai/litellm-helm:0.1.2 +# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2 # Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a ``` @@ -340,7 +340,7 @@ Requirements: We maintain a [separate Dockerfile](https://github.com/BerriAI/litellm/pkgs/container/litellm-database) for reducing build time when running LiteLLM proxy with a connected Postgres Database ```shell -docker pull ghcr.io/berriai/litellm-database:main-stable +docker pull docker.litellm.ai/berriai/litellm-database:main-stable ``` ```shell @@ -351,7 +351,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable \ + docker.litellm.ai/berriai/litellm-database:main-stable \ --config /app/config.yaml --detailed_debug ``` @@ -379,7 +379,7 @@ spec: spec: containers: - name: litellm-container - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable imagePullPolicy: Always env: - name: AZURE_API_KEY @@ -516,9 +516,9 @@ Use this when you want to use litellm helm chart as a dependency for other chart #### Step 1. Pull the litellm helm chart ```bash -helm pull oci://ghcr.io/berriai/litellm-helm +helm pull oci://docker.litellm.ai/berriai/litellm-helm -# Pulled: ghcr.io/berriai/litellm-helm:0.1.2 +# Pulled: docker.litellm.ai/berriai/litellm-helm:0.1.2 # Digest: sha256:7d3ded1c99c1597f9ad4dc49d84327cf1db6e0faa0eeea0c614be5526ae94e2a ``` @@ -575,7 +575,7 @@ router_settings: Start docker container with config ```shell -docker run ghcr.io/berriai/litellm:main-stable --config your_config.yaml +docker run docker.litellm.ai/berriai/litellm:main-stable --config your_config.yaml ``` ### Deploy with Database + Redis @@ -610,7 +610,7 @@ Start `litellm-database`docker container with config docker run --name litellm-proxy \ -e DATABASE_URL=postgresql://:@:/ \ -p 4000:4000 \ -ghcr.io/berriai/litellm-database:main-stable --config your_config.yaml +docker.litellm.ai/berriai/litellm-database:main-stable --config your_config.yaml ``` ### (Non Root) - without Internet Connection @@ -620,7 +620,7 @@ By default `prisma generate` downloads [prisma's engine binaries](https://www.pr Use this docker image to deploy litellm with pre-generated prisma binaries. ```bash -docker pull ghcr.io/berriai/litellm-non_root:main-stable +docker pull docker.litellm.ai/berriai/litellm-non_root:main-stable ``` [Published Docker Image link](https://github.com/BerriAI/litellm/pkgs/container/litellm-non_root) @@ -639,7 +639,7 @@ Use this, If you need to set ssl certificates for your on prem litellm proxy Pass `ssl_keyfile_path` (Path to the SSL keyfile) and `ssl_certfile_path` (Path to the SSL certfile) when starting litellm proxy ```shell -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --ssl_keyfile_path ssl_test/keyfile.key \ --ssl_certfile_path ssl_test/certfile.crt ``` @@ -654,7 +654,7 @@ Step 1. Build your custom docker image with hypercorn ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-stable +FROM docker.litellm.ai/berriai/litellm:main-stable # Set the working directory to /app WORKDIR /app @@ -702,7 +702,7 @@ Usage Example: In this example, we set the keepalive timeout to 75 seconds. ```shell showLineNumbers title="docker run" -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --keepalive_timeout 75 ``` @@ -711,7 +711,7 @@ In this example, we set the keepalive timeout to 75 seconds. ```shell showLineNumbers title="Environment Variable" export KEEPALIVE_TIMEOUT=75 -docker run ghcr.io/berriai/litellm:main-stable +docker run docker.litellm.ai/berriai/litellm:main-stable ``` @@ -722,7 +722,7 @@ Use this to mitigate memory growth by recycling workers after a fixed number of Usage Examples: ```shell showLineNumbers title="docker run (CLI flag)" -docker run ghcr.io/berriai/litellm:main-stable \ +docker run docker.litellm.ai/berriai/litellm:main-stable \ --max_requests_before_restart 10000 ``` @@ -730,7 +730,7 @@ Or set via environment variable: ```shell showLineNumbers title="Environment Variable" export MAX_REQUESTS_BEFORE_RESTART=10000 -docker run ghcr.io/berriai/litellm:main-stable +docker run docker.litellm.ai/berriai/litellm:main-stable ``` @@ -759,7 +759,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -e LITELLM_CONFIG_BUCKET_TYPE="gcs" \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable --detailed_debug + docker.litellm.ai/berriai/litellm-database:main-stable --detailed_debug ``` @@ -780,7 +780,7 @@ docker run --name litellm-proxy \ -e LITELLM_CONFIG_BUCKET_NAME= \ -e LITELLM_CONFIG_BUCKET_OBJECT_KEY="> \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable + docker.litellm.ai/berriai/litellm-database:main-stable ``` @@ -907,7 +907,7 @@ Run the following command, replacing `` with the value you copied docker run --name litellm-proxy \ -e DATABASE_URL= \ -p 4000:4000 \ - ghcr.io/berriai/litellm-database:main-stable + docker.litellm.ai/berriai/litellm-database:main-stable ``` #### 4. Access the Application: @@ -986,7 +986,7 @@ services: context: . args: target: runtime - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable ports: - "4000:4000" # Map the container port to the host, change the host port if necessary volumes: diff --git a/docs/my-website/docs/proxy/docker_quick_start.md b/docs/my-website/docs/proxy/docker_quick_start.md index 35d9923e92c..efdc73de43e 100644 --- a/docs/my-website/docs/proxy/docker_quick_start.md +++ b/docs/my-website/docs/proxy/docker_quick_start.md @@ -20,7 +20,7 @@ End-to-End tutorial for LiteLLM Proxy to: ``` -docker pull ghcr.io/berriai/litellm:main-latest +docker pull docker.litellm.ai/berriai/litellm:main-latest ``` [**See all docker images**](https://github.com/orgs/BerriAI/packages) @@ -119,7 +119,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug # RUNNING on http://0.0.0.0:4000 @@ -302,7 +302,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` diff --git a/docs/my-website/docs/proxy/guardrails/pangea.md b/docs/my-website/docs/proxy/guardrails/pangea.md index 180b9100d6b..3de5ddfa530 100644 --- a/docs/my-website/docs/proxy/guardrails/pangea.md +++ b/docs/my-website/docs/proxy/guardrails/pangea.md @@ -67,7 +67,7 @@ docker run --rm \ -e PANGEA_AI_GUARD_TOKEN=$PANGEA_AI_GUARD_TOKEN \ -e OPENAI_API_KEY=$OPENAI_API_KEY \ -v $(pwd)/config.yaml:/app/config.yaml \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml ``` diff --git a/docs/my-website/docs/proxy/shared_health_check.md b/docs/my-website/docs/proxy/shared_health_check.md index d4b70116309..c9c975c7911 100644 --- a/docs/my-website/docs/proxy/shared_health_check.md +++ b/docs/my-website/docs/proxy/shared_health_check.md @@ -269,7 +269,7 @@ spec: spec: containers: - name: litellm-proxy - image: ghcr.io/berriai/litellm:latest + image: docker.litellm.ai/berriai/litellm:latest env: - name: USE_SHARED_HEALTH_CHECK value: "true" diff --git a/docs/my-website/docs/secret_managers/custom_secret_manager.md b/docs/my-website/docs/secret_managers/custom_secret_manager.md index c51eeeb0727..a6a91a0336d 100644 --- a/docs/my-website/docs/secret_managers/custom_secret_manager.md +++ b/docs/my-website/docs/secret_managers/custom_secret_manager.md @@ -76,7 +76,7 @@ docker run -d \ --name litellm-proxy \ -v $(pwd)/config.yaml:/app/config.yaml \ -v $(pwd)/my_secret_manager.py:/app/my_secret_manager.py \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml \ --port 4000 \ --detailed_debug diff --git a/docs/my-website/docs/tutorials/elasticsearch_logging.md b/docs/my-website/docs/tutorials/elasticsearch_logging.md index eabd47f095d..85a9f1452d7 100644 --- a/docs/my-website/docs/tutorials/elasticsearch_logging.md +++ b/docs/my-website/docs/tutorials/elasticsearch_logging.md @@ -221,7 +221,7 @@ services: - elasticsearch litellm: - image: ghcr.io/berriai/litellm:main-latest + image: docker.litellm.ai/berriai/litellm:main-latest ports: - "4000:4000" environment: diff --git a/docs/my-website/docs/tutorials/openai_codex.md b/docs/my-website/docs/tutorials/openai_codex.md index 41416f85159..563d6559ca5 100644 --- a/docs/my-website/docs/tutorials/openai_codex.md +++ b/docs/my-website/docs/tutorials/openai_codex.md @@ -53,7 +53,7 @@ yarn global add @openai/codex docker run \ -v $(pwd)/litellm_config.yaml:/app/config.yaml \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml ``` diff --git a/docs/my-website/release_notes/v1.55.8-stable/index.md b/docs/my-website/release_notes/v1.55.8-stable/index.md index 38c78eb5372..bf239e0889d 100644 --- a/docs/my-website/release_notes/v1.55.8-stable/index.md +++ b/docs/my-website/release_notes/v1.55.8-stable/index.md @@ -53,7 +53,7 @@ Send LLM usage (spend, tokens) data to [Azure Data Lake](https://learn.microsoft docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable +docker.litellm.ai/berriai/litellm:litellm_stable_release_branch-v1.55.8-stable ``` ## Get Daily Updates diff --git a/docs/my-website/release_notes/v1.57.3/index.md b/docs/my-website/release_notes/v1.57.3/index.md index ab1154a0a8c..bbffa990b32 100644 --- a/docs/my-website/release_notes/v1.57.3/index.md +++ b/docs/my-website/release_notes/v1.57.3/index.md @@ -39,7 +39,7 @@ Instead of `apt-get` use `apk`, the base litellm image will no longer have `apt- **You are only impacted if you use `apt-get` in your Dockerfile** ```shell # Use the provided base image -FROM ghcr.io/berriai/litellm:main-latest +FROM docker.litellm.ai/berriai/litellm:main-latest # Set the working directory WORKDIR /app diff --git a/docs/my-website/release_notes/v1.63.11-stable/index.md b/docs/my-website/release_notes/v1.63.11-stable/index.md index 882747a07b3..3273f9a8e06 100644 --- a/docs/my-website/release_notes/v1.63.11-stable/index.md +++ b/docs/my-website/release_notes/v1.63.11-stable/index.md @@ -36,7 +36,7 @@ This release is primarily focused on: docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.63.11-stable +docker.litellm.ai/berriai/litellm:main-v1.63.11-stable ``` ## Demo Instance diff --git a/docs/my-website/release_notes/v1.63.14/index.md b/docs/my-website/release_notes/v1.63.14/index.md index ff2630468c5..1ac713fc2d5 100644 --- a/docs/my-website/release_notes/v1.63.14/index.md +++ b/docs/my-website/release_notes/v1.63.14/index.md @@ -32,7 +32,7 @@ This release brings: docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.63.14-stable.patch1 +docker.litellm.ai/berriai/litellm:main-v1.63.14-stable.patch1 ``` ## Demo Instance diff --git a/docs/my-website/release_notes/v1.65.4-stable/index.md b/docs/my-website/release_notes/v1.65.4-stable/index.md index 872024a47ab..80d703e1116 100644 --- a/docs/my-website/release_notes/v1.65.4-stable/index.md +++ b/docs/my-website/release_notes/v1.65.4-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.65.4-stable +docker.litellm.ai/berriai/litellm:main-v1.65.4-stable ``` diff --git a/docs/my-website/release_notes/v1.66.0-stable/index.md b/docs/my-website/release_notes/v1.66.0-stable/index.md index 939322e0317..693cd7fc5ac 100644 --- a/docs/my-website/release_notes/v1.66.0-stable/index.md +++ b/docs/my-website/release_notes/v1.66.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.66.0-stable +docker.litellm.ai/berriai/litellm:main-v1.66.0-stable ``` diff --git a/docs/my-website/release_notes/v1.67.4-stable/index.md b/docs/my-website/release_notes/v1.67.4-stable/index.md index 93a27155d2b..f61c99f7d02 100644 --- a/docs/my-website/release_notes/v1.67.4-stable/index.md +++ b/docs/my-website/release_notes/v1.67.4-stable/index.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.67.4-stable +docker.litellm.ai/berriai/litellm:main-v1.67.4-stable ``` diff --git a/docs/my-website/release_notes/v1.68.0-stable/index.md b/docs/my-website/release_notes/v1.68.0-stable/index.md index 4d456d9c853..f3e7fa27427 100644 --- a/docs/my-website/release_notes/v1.68.0-stable/index.md +++ b/docs/my-website/release_notes/v1.68.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.68.0-stable +docker.litellm.ai/berriai/litellm:main-v1.68.0-stable ``` diff --git a/docs/my-website/release_notes/v1.69.0-stable/index.md b/docs/my-website/release_notes/v1.69.0-stable/index.md index 3f8ce7a29c4..f3f094e5403 100644 --- a/docs/my-website/release_notes/v1.69.0-stable/index.md +++ b/docs/my-website/release_notes/v1.69.0-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.69.0-stable +docker.litellm.ai/berriai/litellm:main-v1.69.0-stable ``` diff --git a/docs/my-website/release_notes/v1.70.1-stable/index.md b/docs/my-website/release_notes/v1.70.1-stable/index.md index c55ac8b9c61..5d4bde0f6a0 100644 --- a/docs/my-website/release_notes/v1.70.1-stable/index.md +++ b/docs/my-website/release_notes/v1.70.1-stable/index.md @@ -30,7 +30,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.70.1-stable +docker.litellm.ai/berriai/litellm:main-v1.70.1-stable ``` diff --git a/docs/my-website/release_notes/v1.71.1-stable/index.md b/docs/my-website/release_notes/v1.71.1-stable/index.md index 2d21d49171b..bd37183455d 100644 --- a/docs/my-website/release_notes/v1.71.1-stable/index.md +++ b/docs/my-website/release_notes/v1.71.1-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.71.1-stable +docker.litellm.ai/berriai/litellm:main-v1.71.1-stable ``` diff --git a/docs/my-website/release_notes/v1.72.0-stable/index.md b/docs/my-website/release_notes/v1.72.0-stable/index.md index 47bc19e8aa8..fe235cf07b1 100644 --- a/docs/my-website/release_notes/v1.72.0-stable/index.md +++ b/docs/my-website/release_notes/v1.72.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.0-stable +docker.litellm.ai/berriai/litellm:main-v1.72.0-stable ``` diff --git a/docs/my-website/release_notes/v1.72.2-stable/index.md b/docs/my-website/release_notes/v1.72.2-stable/index.md index 023180f9758..36d01c131c7 100644 --- a/docs/my-website/release_notes/v1.72.2-stable/index.md +++ b/docs/my-website/release_notes/v1.72.2-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.2-stable +docker.litellm.ai/berriai/litellm:main-v1.72.2-stable ``` diff --git a/docs/my-website/release_notes/v1.72.6-stable/index.md b/docs/my-website/release_notes/v1.72.6-stable/index.md index 5603548364f..a20488e2318 100644 --- a/docs/my-website/release_notes/v1.72.6-stable/index.md +++ b/docs/my-website/release_notes/v1.72.6-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run -e STORE_MODEL_IN_DB=True -p 4000:4000 -ghcr.io/berriai/litellm:main-v1.72.6-stable +docker.litellm.ai/berriai/litellm:main-v1.72.6-stable ``` diff --git a/docs/my-website/release_notes/v1.73.0-stable/index.md b/docs/my-website/release_notes/v1.73.0-stable/index.md index 307fecc36dd..802c5ac028b 100644 --- a/docs/my-website/release_notes/v1.73.0-stable/index.md +++ b/docs/my-website/release_notes/v1.73.0-stable/index.md @@ -37,7 +37,7 @@ The `non-root` docker image has a known issue around the UI not loading. If you docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.73.0-stable +docker.litellm.ai/berriai/litellm:v1.73.0-stable ``` diff --git a/docs/my-website/release_notes/v1.73.6-stable/index.md b/docs/my-website/release_notes/v1.73.6-stable/index.md index b03380f9b2b..da748c5c99f 100644 --- a/docs/my-website/release_notes/v1.73.6-stable/index.md +++ b/docs/my-website/release_notes/v1.73.6-stable/index.md @@ -29,7 +29,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.73.6-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.73.6-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.74.0-stable/index.md b/docs/my-website/release_notes/v1.74.0-stable/index.md index e49c2b4f620..ee39c0a26a8 100644 --- a/docs/my-website/release_notes/v1.74.0-stable/index.md +++ b/docs/my-website/release_notes/v1.74.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.0-stable +docker.litellm.ai/berriai/litellm:v1.74.0-stable ``` diff --git a/docs/my-website/release_notes/v1.74.15-stable/index.md b/docs/my-website/release_notes/v1.74.15-stable/index.md index 9807a00b7e7..c0facf8afb0 100644 --- a/docs/my-website/release_notes/v1.74.15-stable/index.md +++ b/docs/my-website/release_notes/v1.74.15-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.15-stable +docker.litellm.ai/berriai/litellm:v1.74.15-stable ``` diff --git a/docs/my-website/release_notes/v1.74.3-stable/index.md b/docs/my-website/release_notes/v1.74.3-stable/index.md index 167d81e52af..05386172e71 100644 --- a/docs/my-website/release_notes/v1.74.3-stable/index.md +++ b/docs/my-website/release_notes/v1.74.3-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.3-stable +docker.litellm.ai/berriai/litellm:v1.74.3-stable ``` diff --git a/docs/my-website/release_notes/v1.74.7/index.md b/docs/my-website/release_notes/v1.74.7/index.md index 7d7a568e13f..10fbd21b498 100644 --- a/docs/my-website/release_notes/v1.74.7/index.md +++ b/docs/my-website/release_notes/v1.74.7/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.7-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.74.7-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.74.9-stable/index.md b/docs/my-website/release_notes/v1.74.9-stable/index.md index 3f100745dfe..9feed6d62e6 100644 --- a/docs/my-website/release_notes/v1.74.9-stable/index.md +++ b/docs/my-website/release_notes/v1.74.9-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.74.9-stable.patch.1 +docker.litellm.ai/berriai/litellm:v1.74.9-stable.patch.1 ``` diff --git a/docs/my-website/release_notes/v1.75.5-stable/index.md b/docs/my-website/release_notes/v1.75.5-stable/index.md index 7035d285057..043f1267fc8 100644 --- a/docs/my-website/release_notes/v1.75.5-stable/index.md +++ b/docs/my-website/release_notes/v1.75.5-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.75.5-stable +docker.litellm.ai/berriai/litellm:v1.75.5-stable ``` diff --git a/docs/my-website/release_notes/v1.75.8/index.md b/docs/my-website/release_notes/v1.75.8/index.md index d7d4f37c4ee..3db1fe4b2cd 100644 --- a/docs/my-website/release_notes/v1.75.8/index.md +++ b/docs/my-website/release_notes/v1.75.8/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.75.8-stable +docker.litellm.ai/berriai/litellm:v1.75.8-stable ``` diff --git a/docs/my-website/release_notes/v1.76.1-stable/index.md b/docs/my-website/release_notes/v1.76.1-stable/index.md index 4437b7f5799..f458dfde6d4 100644 --- a/docs/my-website/release_notes/v1.76.1-stable/index.md +++ b/docs/my-website/release_notes/v1.76.1-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.76.1 +docker.litellm.ai/berriai/litellm:v1.76.1 ``` diff --git a/docs/my-website/release_notes/v1.76.3-stable/index.md b/docs/my-website/release_notes/v1.76.3-stable/index.md index 6b40e4f5b35..9763a57975b 100644 --- a/docs/my-website/release_notes/v1.76.3-stable/index.md +++ b/docs/my-website/release_notes/v1.76.3-stable/index.md @@ -35,7 +35,7 @@ This release has a known issue where startup is leading to Out of Memory errors docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.76.3 +docker.litellm.ai/berriai/litellm:v1.76.3 ``` diff --git a/docs/my-website/release_notes/v1.77.2-stable/index.md b/docs/my-website/release_notes/v1.77.2-stable/index.md index fdd80693d05..4f732a1604d 100644 --- a/docs/my-website/release_notes/v1.77.2-stable/index.md +++ b/docs/my-website/release_notes/v1.77.2-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:main-v1.77.2-stable +docker.litellm.ai/berriai/litellm:main-v1.77.2-stable ``` diff --git a/docs/my-website/release_notes/v1.77.3-stable/index.md b/docs/my-website/release_notes/v1.77.3-stable/index.md index c7c17e5baee..11b82c4c834 100644 --- a/docs/my-website/release_notes/v1.77.3-stable/index.md +++ b/docs/my-website/release_notes/v1.77.3-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.3-stable +docker.litellm.ai/berriai/litellm:v1.77.3-stable ``` diff --git a/docs/my-website/release_notes/v1.77.5-stable/index.md b/docs/my-website/release_notes/v1.77.5-stable/index.md index 6843800ee6d..8e59ea92cc2 100644 --- a/docs/my-website/release_notes/v1.77.5-stable/index.md +++ b/docs/my-website/release_notes/v1.77.5-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.5-stable +docker.litellm.ai/berriai/litellm:v1.77.5-stable ``` diff --git a/docs/my-website/release_notes/v1.77.7-stable/index.md b/docs/my-website/release_notes/v1.77.7-stable/index.md index 62d9a2eee4f..b4df447f334 100644 --- a/docs/my-website/release_notes/v1.77.7-stable/index.md +++ b/docs/my-website/release_notes/v1.77.7-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.77.7.rc.1 +docker.litellm.ai/berriai/litellm:v1.77.7.rc.1 ``` diff --git a/docs/my-website/release_notes/v1.78.0-stable/index.md b/docs/my-website/release_notes/v1.78.0-stable/index.md index 7f6c5ba1e08..8322f0479c5 100644 --- a/docs/my-website/release_notes/v1.78.0-stable/index.md +++ b/docs/my-website/release_notes/v1.78.0-stable/index.md @@ -28,7 +28,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.78.0-stable +docker.litellm.ai/berriai/litellm:v1.78.0-stable ``` diff --git a/docs/my-website/release_notes/v1.78.5-stable/index.md b/docs/my-website/release_notes/v1.78.5-stable/index.md index af1fd359fa2..2bcdfab472c 100644 --- a/docs/my-website/release_notes/v1.78.5-stable/index.md +++ b/docs/my-website/release_notes/v1.78.5-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.78.5-stable +docker.litellm.ai/berriai/litellm:v1.78.5-stable ``` diff --git a/docs/my-website/release_notes/v1.79.0-stable/index.md b/docs/my-website/release_notes/v1.79.0-stable/index.md index 8327f4b6178..4bb7094a3fc 100644 --- a/docs/my-website/release_notes/v1.79.0-stable/index.md +++ b/docs/my-website/release_notes/v1.79.0-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.0-stable +docker.litellm.ai/berriai/litellm:v1.79.0-stable ``` diff --git a/docs/my-website/release_notes/v1.79.1-stable/index.md b/docs/my-website/release_notes/v1.79.1-stable/index.md index ea8cfeae740..19fc7f9f3ff 100644 --- a/docs/my-website/release_notes/v1.79.1-stable/index.md +++ b/docs/my-website/release_notes/v1.79.1-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.1-stable +docker.litellm.ai/berriai/litellm:v1.79.1-stable ``` diff --git a/docs/my-website/release_notes/v1.79.3-stable/index.md b/docs/my-website/release_notes/v1.79.3-stable/index.md index c4f3ba1e017..542f88787e0 100644 --- a/docs/my-website/release_notes/v1.79.3-stable/index.md +++ b/docs/my-website/release_notes/v1.79.3-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.79.3-stable +docker.litellm.ai/berriai/litellm:v1.79.3-stable ``` diff --git a/docs/my-website/release_notes/v1.80.0-stable/index.md b/docs/my-website/release_notes/v1.80.0-stable/index.md index 17fcf6646ed..d0cf28a5c58 100644 --- a/docs/my-website/release_notes/v1.80.0-stable/index.md +++ b/docs/my-website/release_notes/v1.80.0-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.0-stable +docker.litellm.ai/berriai/litellm:v1.80.0-stable ``` diff --git a/docs/my-website/release_notes/v1.80.10-stable/index.md b/docs/my-website/release_notes/v1.80.10-stable/index.md index 1b0a9866fae..2290c06de53 100644 --- a/docs/my-website/release_notes/v1.80.10-stable/index.md +++ b/docs/my-website/release_notes/v1.80.10-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.10.rc.1 +docker.litellm.ai/berriai/litellm:v1.80.10.rc.1 ``` diff --git a/docs/my-website/release_notes/v1.80.5-stable/index.md b/docs/my-website/release_notes/v1.80.5-stable/index.md index 598fa47f223..9c769f8996f 100644 --- a/docs/my-website/release_notes/v1.80.5-stable/index.md +++ b/docs/my-website/release_notes/v1.80.5-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.5-stable +docker.litellm.ai/berriai/litellm:v1.80.5-stable ``` diff --git a/docs/my-website/release_notes/v1.80.8-stable/index.md b/docs/my-website/release_notes/v1.80.8-stable/index.md index 29075a9594f..106c594968f 100644 --- a/docs/my-website/release_notes/v1.80.8-stable/index.md +++ b/docs/my-website/release_notes/v1.80.8-stable/index.md @@ -27,7 +27,7 @@ import TabItem from '@theme/TabItem'; docker run \ -e STORE_MODEL_IN_DB=True \ -p 4000:4000 \ -ghcr.io/berriai/litellm:v1.80.8-stable +docker.litellm.ai/berriai/litellm:v1.80.8-stable ``` diff --git a/docs/my-website/src/pages/index.md b/docs/my-website/src/pages/index.md index 1dc2995c5fe..91215b33c5d 100644 --- a/docs/my-website/src/pages/index.md +++ b/docs/my-website/src/pages/index.md @@ -604,7 +604,7 @@ docker run \ -e AZURE_API_KEY=d6*********** \ -e AZURE_API_BASE=https://openai-***********/ \ -p 4000:4000 \ - ghcr.io/berriai/litellm:main-latest \ + docker.litellm.ai/berriai/litellm:main-latest \ --config /app/config.yaml --detailed_debug ``` From a0754f1c88acf5f64cc69e46b103340f08c05c60 Mon Sep 17 00:00:00 2001 From: kothamah <104782493+kothamah@users.noreply.github.com> Date: Mon, 15 Dec 2025 22:11:13 -0500 Subject: [PATCH 11/35] Litellm bedrock guardrails block precedence over masking (#17968) * prioritized bedrock guardrail blocking by removing early return based on masking flags When mask_request_content: true or mask_response_content: true, the method immediately returning False. The Result: Even when Bedrock Guardrails returned action: "BLOCKED" for dangerous content, LiteLLM would not raise an exception and allowing the content through the response. So removed that early condition which will return true for the blocked actions based on guardrails. * Added test case for bedrock guardrail block content precedence --- .../guardrail_hooks/bedrock_guardrails.py | 10 +-- .../test_bedrock_guardrails.py | 88 +++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index fb14ccce50c..62c997659bd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -605,13 +605,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): """ Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions. - If `self.mask_request_content` or `self.mask_response_content` is set to `True`, then use the output from the guardrail to mask the request or response content. + If `self.mask_request_content` or `self.mask_response_content` is set to `True`, + then use the output from the guardrail to mask the request or response content. + + However, even with masking enabled, content with action="BLOCKED" should still + raise an exception, only content with action="ANONYMIZED" should be masked. """ - # if user opted into masking, return False. since we'll use the masked output from the guardrail - if self.mask_request_content or self.mask_response_content: - return False - # if no intervention, return False if response.get("action") != "GUARDRAIL_INTERVENED": return False diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 69b0bb27b4b..84d320a0a27 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1101,3 +1101,91 @@ async def test_bedrock_apply_guardrail_with_only_tool_calls_response(): # Verify that the Bedrock API was NOT called since there's no text to process mock_api_request.assert_not_called() print("✅ apply_guardrail with tool_calls test passed - no API call made") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_blocked_content_with_masking_enabled(): + """Test that BLOCKED content raises exception even when masking is enabled + + This test verifies the bug fix where previously mask_request_content=True or + mask_response_content=True would bypass all BLOCKED content checks. Now it + properly distinguishes between BLOCKED (raise exception) and ANONYMIZED (apply masking). + """ + + # Create guardrail with masking enabled + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + mask_request_content=True, # Masking enabled + mask_response_content=True, # Masking enabled + ) + + # Mock Bedrock response with BLOCKED content (hate speech) + blocked_response = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "contentPolicy": { + "filters": [ + { + "type": "HATE", + "confidence": "HIGH", + "action": "BLOCKED", # Should raise exception + } + ] + }, + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "NAME", + "match": "John Doe", + "action": "ANONYMIZED", # Should be masked + } + ] + }, + } + ], + "outputs": [{"text": "Content blocked due to policy violation"}], + } + + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = blocked_response + + # Mock credentials + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Test message with PII and hate speech"}, + ], + } + + # Mock AWS-related methods + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), patch.object( + guardrail, "_prepare_request", return_value=MagicMock() + ): + mock_post.return_value = mock_bedrock_response + + # Should raise HTTPException for BLOCKED content + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=request_data.get("messages"), + request_data=request_data, + ) + + # Verify exception details + assert exc_info.value.status_code == 400 + assert "Violated guardrail policy" in str(exc_info.value.detail) + + print("✅ BLOCKED content with masking enabled raises exception correctly") + From 0abe5cdce98e4540d402c5607429b4d8182a1f83 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 16 Dec 2025 08:41:36 +0530 Subject: [PATCH 12/35] Revert "Litellm bedrock guardrails block precedence over masking (#17968)" (#18022) This reverts commit a0754f1c88acf5f64cc69e46b103340f08c05c60. --- .../guardrail_hooks/bedrock_guardrails.py | 10 +-- .../test_bedrock_guardrails.py | 88 ------------------- 2 files changed, 5 insertions(+), 93 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index 62c997659bd..fb14ccce50c 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -605,13 +605,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): """ Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions. - If `self.mask_request_content` or `self.mask_response_content` is set to `True`, - then use the output from the guardrail to mask the request or response content. - - However, even with masking enabled, content with action="BLOCKED" should still - raise an exception, only content with action="ANONYMIZED" should be masked. + If `self.mask_request_content` or `self.mask_response_content` is set to `True`, then use the output from the guardrail to mask the request or response content. """ + # if user opted into masking, return False. since we'll use the masked output from the guardrail + if self.mask_request_content or self.mask_response_content: + return False + # if no intervention, return False if response.get("action") != "GUARDRAIL_INTERVENED": return False diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 84d320a0a27..69b0bb27b4b 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1101,91 +1101,3 @@ async def test_bedrock_apply_guardrail_with_only_tool_calls_response(): # Verify that the Bedrock API was NOT called since there's no text to process mock_api_request.assert_not_called() print("✅ apply_guardrail with tool_calls test passed - no API call made") - - -@pytest.mark.asyncio -async def test_bedrock_guardrail_blocked_content_with_masking_enabled(): - """Test that BLOCKED content raises exception even when masking is enabled - - This test verifies the bug fix where previously mask_request_content=True or - mask_response_content=True would bypass all BLOCKED content checks. Now it - properly distinguishes between BLOCKED (raise exception) and ANONYMIZED (apply masking). - """ - - # Create guardrail with masking enabled - guardrail = BedrockGuardrail( - guardrailIdentifier="test-guardrail", - guardrailVersion="DRAFT", - mask_request_content=True, # Masking enabled - mask_response_content=True, # Masking enabled - ) - - # Mock Bedrock response with BLOCKED content (hate speech) - blocked_response = { - "action": "GUARDRAIL_INTERVENED", - "assessments": [ - { - "contentPolicy": { - "filters": [ - { - "type": "HATE", - "confidence": "HIGH", - "action": "BLOCKED", # Should raise exception - } - ] - }, - "sensitiveInformationPolicy": { - "piiEntities": [ - { - "type": "NAME", - "match": "John Doe", - "action": "ANONYMIZED", # Should be masked - } - ] - }, - } - ], - "outputs": [{"text": "Content blocked due to policy violation"}], - } - - mock_bedrock_response = MagicMock() - mock_bedrock_response.status_code = 200 - mock_bedrock_response.json.return_value = blocked_response - - # Mock credentials - mock_credentials = MagicMock() - mock_credentials.access_key = "test-access-key" - mock_credentials.secret_key = "test-secret-key" - mock_credentials.token = None - - request_data = { - "model": "gpt-4o", - "messages": [ - {"role": "user", "content": "Test message with PII and hate speech"}, - ], - } - - # Mock AWS-related methods - with patch.object( - guardrail.async_handler, "post", new_callable=AsyncMock - ) as mock_post, patch.object( - guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") - ), patch.object( - guardrail, "_prepare_request", return_value=MagicMock() - ): - mock_post.return_value = mock_bedrock_response - - # Should raise HTTPException for BLOCKED content - with pytest.raises(HTTPException) as exc_info: - await guardrail.make_bedrock_api_request( - source="INPUT", - messages=request_data.get("messages"), - request_data=request_data, - ) - - # Verify exception details - assert exc_info.value.status_code == 400 - assert "Violated guardrail policy" in str(exc_info.value.detail) - - print("✅ BLOCKED content with masking enabled raises exception correctly") - From f58b76aee8f97fe223d6ba40d47159a8d04185ed Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 16 Dec 2025 08:42:10 +0530 Subject: [PATCH 13/35] =?UTF-8?q?Revert=20"Revert=20"Litellm=20bedrock=20g?= =?UTF-8?q?uardrails=20block=20precedence=20over=20masking=20(#17=E2=80=A6?= =?UTF-8?q?"=20(#18023)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 0abe5cdce98e4540d402c5607429b4d8182a1f83. --- .../guardrail_hooks/bedrock_guardrails.py | 10 +-- .../test_bedrock_guardrails.py | 88 +++++++++++++++++++ 2 files changed, 93 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py index fb14ccce50c..62c997659bd 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py +++ b/litellm/proxy/guardrails/guardrail_hooks/bedrock_guardrails.py @@ -605,13 +605,13 @@ class BedrockGuardrail(CustomGuardrail, BaseAWSLLM): """ Only raise exception for "BLOCKED" actions, not for "ANONYMIZED" actions. - If `self.mask_request_content` or `self.mask_response_content` is set to `True`, then use the output from the guardrail to mask the request or response content. + If `self.mask_request_content` or `self.mask_response_content` is set to `True`, + then use the output from the guardrail to mask the request or response content. + + However, even with masking enabled, content with action="BLOCKED" should still + raise an exception, only content with action="ANONYMIZED" should be masked. """ - # if user opted into masking, return False. since we'll use the masked output from the guardrail - if self.mask_request_content or self.mask_response_content: - return False - # if no intervention, return False if response.get("action") != "GUARDRAIL_INTERVENED": return False diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 69b0bb27b4b..84d320a0a27 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -1101,3 +1101,91 @@ async def test_bedrock_apply_guardrail_with_only_tool_calls_response(): # Verify that the Bedrock API was NOT called since there's no text to process mock_api_request.assert_not_called() print("✅ apply_guardrail with tool_calls test passed - no API call made") + + +@pytest.mark.asyncio +async def test_bedrock_guardrail_blocked_content_with_masking_enabled(): + """Test that BLOCKED content raises exception even when masking is enabled + + This test verifies the bug fix where previously mask_request_content=True or + mask_response_content=True would bypass all BLOCKED content checks. Now it + properly distinguishes between BLOCKED (raise exception) and ANONYMIZED (apply masking). + """ + + # Create guardrail with masking enabled + guardrail = BedrockGuardrail( + guardrailIdentifier="test-guardrail", + guardrailVersion="DRAFT", + mask_request_content=True, # Masking enabled + mask_response_content=True, # Masking enabled + ) + + # Mock Bedrock response with BLOCKED content (hate speech) + blocked_response = { + "action": "GUARDRAIL_INTERVENED", + "assessments": [ + { + "contentPolicy": { + "filters": [ + { + "type": "HATE", + "confidence": "HIGH", + "action": "BLOCKED", # Should raise exception + } + ] + }, + "sensitiveInformationPolicy": { + "piiEntities": [ + { + "type": "NAME", + "match": "John Doe", + "action": "ANONYMIZED", # Should be masked + } + ] + }, + } + ], + "outputs": [{"text": "Content blocked due to policy violation"}], + } + + mock_bedrock_response = MagicMock() + mock_bedrock_response.status_code = 200 + mock_bedrock_response.json.return_value = blocked_response + + # Mock credentials + mock_credentials = MagicMock() + mock_credentials.access_key = "test-access-key" + mock_credentials.secret_key = "test-secret-key" + mock_credentials.token = None + + request_data = { + "model": "gpt-4o", + "messages": [ + {"role": "user", "content": "Test message with PII and hate speech"}, + ], + } + + # Mock AWS-related methods + with patch.object( + guardrail.async_handler, "post", new_callable=AsyncMock + ) as mock_post, patch.object( + guardrail, "_load_credentials", return_value=(mock_credentials, "us-east-1") + ), patch.object( + guardrail, "_prepare_request", return_value=MagicMock() + ): + mock_post.return_value = mock_bedrock_response + + # Should raise HTTPException for BLOCKED content + with pytest.raises(HTTPException) as exc_info: + await guardrail.make_bedrock_api_request( + source="INPUT", + messages=request_data.get("messages"), + request_data=request_data, + ) + + # Verify exception details + assert exc_info.value.status_code == 400 + assert "Violated guardrail policy" in str(exc_info.value.detail) + + print("✅ BLOCKED content with masking enabled raises exception correctly") + From 32c07113cf8a49c238144144cf07f0e2f8005e24 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 15 Dec 2025 19:14:49 -0800 Subject: [PATCH 14/35] [Feat] New Provider - VertexAI Agent Engine (#18014) * init A2AProviderConfigManager * move file * move file * add pydnatic ai folder * init providers * test_pydantic_ai_non_streaming * fix import * INIT pydantic * use_a2a_form_fields * test_vertex_agent_engine_streaming * add agent_engine * init transform for agent engine * init agent engine * VertexAgentEngineSSEStreamIterator * sample * ui add new fields * fix vertex_credentials * working SSE iterator * TestVertexAgentEngineTransformRequest * fix code QA check * Potential fix for code scanning alert no. 3923: Clear-text logging of sensitive information Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --------- Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- .../llms/vertex_ai/agent_engine/__init__.py | 13 + .../vertex_ai/agent_engine/sse_iterator.py | 90 ++++ .../vertex_ai/agent_engine/transformation.py | 508 ++++++++++++++++++ litellm/llms/vertex_ai/common_utils.py | 7 +- litellm/main.py | 31 ++ ...odel_prices_and_context_window_backup.json | 2 +- .../public_endpoints/agent_create_fields.json | 23 + .../provider_create_fields.json | 4 +- tests/agent_tests/local_vertex_agent.py | 151 ++++++ .../agent_tests/test_a2a_completion_bridge.py | 76 +++ .../agent_engine/test_transformation.py | 128 +++++ 11 files changed, 1029 insertions(+), 4 deletions(-) create mode 100644 litellm/llms/vertex_ai/agent_engine/__init__.py create mode 100644 litellm/llms/vertex_ai/agent_engine/sse_iterator.py create mode 100644 litellm/llms/vertex_ai/agent_engine/transformation.py create mode 100644 tests/agent_tests/local_vertex_agent.py create mode 100644 tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py diff --git a/litellm/llms/vertex_ai/agent_engine/__init__.py b/litellm/llms/vertex_ai/agent_engine/__init__.py new file mode 100644 index 00000000000..de891f85602 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/__init__.py @@ -0,0 +1,13 @@ +""" +Vertex AI Agent Engine (Reasoning Engines) Provider + +Supports Vertex AI Reasoning Engines via the :query and :streamQuery endpoints. +""" + +from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + VertexAgentEngineError, +) + +__all__ = ["VertexAgentEngineConfig", "VertexAgentEngineError"] + diff --git a/litellm/llms/vertex_ai/agent_engine/sse_iterator.py b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py new file mode 100644 index 00000000000..06fb55e1848 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/sse_iterator.py @@ -0,0 +1,90 @@ +""" +SSE Stream Iterator for Vertex AI Agent Engine. + +Handles Server-Sent Events (SSE) streaming responses from Vertex AI Reasoning Engines. +""" + +from typing import Any, Union + +from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator +from litellm.types.llms.openai import ChatCompletionUsageBlock +from litellm.types.utils import ( + Delta, + GenericStreamingChunk, + ModelResponseStream, + StreamingChoices, +) + + +class VertexAgentEngineResponseIterator(BaseModelResponseIterator): + """ + Iterator for Vertex Agent Engine SSE streaming responses. + + Uses BaseModelResponseIterator which handles sync/async iteration. + We just need to implement chunk_parser to parse Vertex Agent Engine response format. + """ + + def __init__(self, streaming_response: Any, sync_stream: bool) -> None: + super().__init__(streaming_response=streaming_response, sync_stream=sync_stream) + + def chunk_parser( + self, chunk: dict + ) -> Union[GenericStreamingChunk, ModelResponseStream]: + """ + Parse a Vertex Agent Engine response chunk into ModelResponseStream. + + Vertex Agent Engine response format: + { + "content": { + "parts": [{"text": "..."}], + "role": "model" + }, + "finish_reason": "STOP", + "usage_metadata": { + "prompt_token_count": 100, + "candidates_token_count": 50, + "total_token_count": 150 + } + } + """ + # Extract text from content.parts + text = None + content = chunk.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if isinstance(part, dict) and "text" in part: + text = part["text"] + break + + # Extract finish_reason + finish_reason = None + raw_finish_reason = chunk.get("finish_reason") + if raw_finish_reason == "STOP": + finish_reason = "stop" + elif raw_finish_reason: + finish_reason = raw_finish_reason.lower() + + # Extract usage from usage_metadata + usage = None + usage_metadata = chunk.get("usage_metadata", {}) + if usage_metadata: + usage = ChatCompletionUsageBlock( + prompt_tokens=usage_metadata.get("prompt_token_count", 0), + completion_tokens=usage_metadata.get("candidates_token_count", 0), + total_tokens=usage_metadata.get("total_token_count", 0), + ) + + # Return ModelResponseStream (OpenAI-compatible chunk) + return ModelResponseStream( + choices=[ + StreamingChoices( + finish_reason=finish_reason, + index=0, + delta=Delta( + content=text, + role="assistant" if text else None, + ), + ) + ], + usage=usage, + ) diff --git a/litellm/llms/vertex_ai/agent_engine/transformation.py b/litellm/llms/vertex_ai/agent_engine/transformation.py new file mode 100644 index 00000000000..4c07e8455e3 --- /dev/null +++ b/litellm/llms/vertex_ai/agent_engine/transformation.py @@ -0,0 +1,508 @@ +""" +Transformation for Vertex AI Agent Engine (Reasoning Engines) + +Handles the transformation between LiteLLM's OpenAI-compatible format and +Vertex AI Reasoning Engine's API format. + +API Reference: +- :query endpoint - for session management (create, get, list, delete) +- :streamQuery endpoint - for actual queries (stream_query method) +""" + +import json +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast + +import httpx + +from litellm._logging import verbose_logger +from litellm._uuid import uuid +from litellm.litellm_core_utils.prompt_templates.common_utils import ( + convert_content_list_to_str, +) +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( + VertexAgentEngineResponseIterator, +) +from litellm.llms.vertex_ai.vertex_llm_base import VertexBase +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler + from litellm.utils import CustomStreamWrapper + + LiteLLMLoggingObj = _LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + HTTPHandler = Any + AsyncHTTPHandler = Any + CustomStreamWrapper = Any + + +class VertexAgentEngineError(BaseLLMException): + """Exception for Vertex Agent Engine errors.""" + + def __init__(self, status_code: int, message: str): + self.status_code = status_code + self.message = message + super().__init__(message=message, status_code=status_code) + + +class VertexAgentEngineConfig(BaseConfig, VertexBase): + """ + Configuration for Vertex AI Agent Engine (Reasoning Engines). + + Model format: vertex_ai/agent_engine/ + Where resource_id is the numeric ID of the reasoning engine. + """ + + def __init__(self, **kwargs): + BaseConfig.__init__(self, **kwargs) + VertexBase.__init__(self) + + def get_supported_openai_params(self, model: str) -> List[str]: + """Vertex Agent Engine has limited OpenAI compatible params.""" + return ["user"] + + def map_openai_params( + self, + non_default_params: dict, + optional_params: dict, + model: str, + drop_params: bool, + ) -> dict: + """Map OpenAI params to Agent Engine params.""" + # Map 'user' to 'user_id' for session management + if "user" in non_default_params: + optional_params["user_id"] = non_default_params["user"] + return optional_params + + def _parse_model_string(self, model: str) -> Tuple[str, str]: + """ + Parse model string to extract resource ID. + + Model format: agent_engine/// + Or: agent_engine/ (uses default project/location) + + Returns: (resource_path, engine_id) + """ + # Remove 'agent_engine/' prefix if present + if model.startswith("agent_engine/"): + model = model[len("agent_engine/") :] + + # Check if it's a full resource path + if model.startswith("projects/"): + # Full path: projects/123/locations/us-central1/reasoningEngines/456 + return model, model.split("/")[-1] + + # Just the engine ID + return model, model + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the request. + + For Vertex Agent Engine: + - Non-streaming: :query endpoint (for session management) + - Streaming: :streamQuery endpoint (for actual queries) + """ + resource_path, engine_id = self._parse_model_string(model) + + # Get project and location from litellm_params or environment + vertex_project = self.safe_get_vertex_ai_project(litellm_params) + vertex_location = self.safe_get_vertex_ai_location(litellm_params) or "us-central1" + + # Build the full resource path if only engine_id was provided + if not resource_path.startswith("projects/"): + if not vertex_project: + raise ValueError( + "vertex_project is required for Vertex Agent Engine. " + "Set via litellm_params['vertex_project'] or VERTEXAI_PROJECT env var." + ) + resource_path = f"projects/{vertex_project}/locations/{vertex_location}/reasoningEngines/{engine_id}" + + # Build the base URL + base_url = f"https://{vertex_location}-aiplatform.googleapis.com" + + # Always use :streamQuery endpoint for actual queries + # The :query endpoint only supports session management methods + # (create_session, get_session, list_sessions, delete_session, etc.) + endpoint = f"{base_url}/v1beta1/{resource_path}:streamQuery" + + verbose_logger.debug(f"Vertex Agent Engine URL: {endpoint}") + return endpoint + + def _get_auth_headers( + self, + optional_params: dict, + litellm_params: dict, + ) -> Dict[str, str]: + """Get authentication headers using Google Cloud credentials.""" + vertex_credentials = self.safe_get_vertex_ai_credentials(litellm_params) + vertex_project = self.safe_get_vertex_ai_project(litellm_params) + + # Get access token using VertexBase + access_token, project_id = self.get_access_token( + credentials=vertex_credentials, + project_id=vertex_project, + ) + + verbose_logger.debug(f"Vertex Agent Engine: Authenticated for project {project_id}") + + return { + "Authorization": f"Bearer {access_token}", + "Content-Type": "application/json", + } + + def _get_user_id(self, optional_params: dict) -> str: + """Get or generate user ID for session management.""" + user_id = optional_params.get("user_id") or optional_params.get("user") + if user_id: + return user_id + # Generate a user ID + return f"litellm-user-{str(uuid.uuid4())[:8]}" + + def _get_session_id(self, optional_params: dict) -> Optional[str]: + """Get session ID if provided.""" + return optional_params.get("session_id") + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform the request to Vertex Agent Engine format. + + The API expects: + { + "class_method": "stream_query", + "input": { + "message": "...", + "user_id": "...", + "session_id": "..." (optional) + } + } + """ + # Use the last message content as the prompt + prompt = convert_content_list_to_str(messages[-1]) + + # Get user_id and session_id + user_id = self._get_user_id(optional_params) + session_id = self._get_session_id(optional_params) + + # Build the input + input_data: Dict[str, Any] = { + "message": prompt, + "user_id": user_id, + } + + if session_id: + input_data["session_id"] = session_id + + # Build the request payload + # Note: stream_query is used for both streaming and non-streaming + # The difference is the endpoint (:streamQuery vs :query) + payload = { + "class_method": "stream_query", + "input": input_data, + } + + verbose_logger.debug(f"Vertex Agent Engine payload: {payload}") + return payload + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """Validate environment and set up authentication headers.""" + auth_headers = self._get_auth_headers(optional_params, litellm_params) + headers.update(auth_headers) + return headers + + def _extract_text_from_response(self, response_data: dict) -> str: + """Extract text content from the response.""" + # Try to get from content.parts + content = response_data.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if "text" in part: + return part["text"] + + # Try actions.state_delta + actions = response_data.get("actions", {}) + state_delta = actions.get("state_delta", {}) + for key, value in state_delta.items(): + if isinstance(value, str) and value: + return value + + return "" + + def _calculate_usage( + self, model: str, messages: List[AllMessageValues], content: str + ) -> Optional[Usage]: + """Calculate token usage using LiteLLM's token counter.""" + try: + from litellm.utils import token_counter + + prompt_tokens = token_counter(model="gpt-3.5-turbo", messages=messages) + completion_tokens = token_counter( + model="gpt-3.5-turbo", text=content, count_response_tokens=True + ) + total_tokens = prompt_tokens + completion_tokens + + return Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ) + except Exception as e: + verbose_logger.warning(f"Failed to calculate token usage: {str(e)}") + return None + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: LiteLLMLoggingObj, + request_data: dict, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + encoding: Any, + api_key: Optional[str] = None, + json_mode: Optional[bool] = None, + ) -> ModelResponse: + """ + Transform Vertex Agent Engine response to LiteLLM ModelResponse format. + + The response is a streaming SSE format even for non-streaming requests. + We need to collect all the chunks and extract the final response. + """ + try: + content_type = raw_response.headers.get("content-type", "").lower() + verbose_logger.debug(f"Vertex Agent Engine response Content-Type: {content_type}") + + # Parse the SSE response + response_text = raw_response.text + verbose_logger.debug(f"Response (first 500 chars): {response_text[:500]}") + + # Extract content from SSE stream + content = "" + for line in response_text.strip().split("\n"): + line = line.strip() + if not line: + continue + + try: + data = json.loads(line) + if isinstance(data, dict): + text = self._extract_text_from_response(data) + if text: + content = text # Use the last non-empty text + except json.JSONDecodeError: + continue + + # Create the message + message = Message(content=content, role="assistant") + + # Create choices + choice = Choices(finish_reason="stop", index=0, message=message) + + # Update model response + model_response.choices = [choice] + model_response.model = model + + # Calculate usage + calculated_usage = self._calculate_usage(model, messages, content) + if calculated_usage: + setattr(model_response, "usage", calculated_usage) + + return model_response + + except Exception as e: + verbose_logger.error(f"Error processing Vertex Agent Engine response: {str(e)}") + raise VertexAgentEngineError( + message=f"Error processing response: {str(e)}", + status_code=raw_response.status_code, + ) + + def get_streaming_response( + self, + model: str, + raw_response: httpx.Response, + ) -> VertexAgentEngineResponseIterator: + """Return a streaming iterator for SSE responses.""" + return VertexAgentEngineResponseIterator( + streaming_response=raw_response.iter_lines(), + sync_stream=True, + ) + + def get_sync_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional[Union[HTTPHandler, "AsyncHTTPHandler"]] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """Get a CustomStreamWrapper for synchronous streaming.""" + from litellm.llms.custom_httpx.http_handler import ( + HTTPHandler, + _get_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, HTTPHandler): + client = _get_httpx_client(params={}) + + # Avoid logging sensitive api_base directly + verbose_logger.debug("Making sync streaming request to Vertex AI endpoint.") + + # Make streaming request + response = client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise VertexAgentEngineError( + status_code=response.status_code, message=str(response.read()) + ) + + # Create iterator for SSE stream + completion_stream = self.get_streaming_response(model=model, raw_response=response) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + async def get_async_custom_stream_wrapper( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj, + api_base: str, + headers: dict, + data: dict, + messages: list, + client: Optional["AsyncHTTPHandler"] = None, + json_mode: Optional[bool] = None, + signed_json_body: Optional[bytes] = None, + ) -> "CustomStreamWrapper": + """Get a CustomStreamWrapper for asynchronous streaming.""" + from litellm.llms.custom_httpx.http_handler import ( + AsyncHTTPHandler, + get_async_httpx_client, + ) + from litellm.utils import CustomStreamWrapper + + if client is None or not isinstance(client, AsyncHTTPHandler): + client = get_async_httpx_client( + llm_provider=cast(Any, "vertex_ai"), params={} + ) + + # Avoid logging sensitive api_base directly + verbose_logger.debug("Making async streaming request to Vertex AI endpoint.") + + # Make async streaming request + response = await client.post( + api_base, + headers=headers, + data=json.dumps(data), + stream=True, + logging_obj=logging_obj, + ) + + if response.status_code != 200: + raise VertexAgentEngineError( + status_code=response.status_code, message=str(await response.aread()) + ) + + # Create iterator for SSE stream (async) + completion_stream = VertexAgentEngineResponseIterator( + streaming_response=response.aiter_lines(), + sync_stream=False, + ) + + streaming_response = CustomStreamWrapper( + completion_stream=completion_stream, + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + ) + + # LOGGING + logging_obj.post_call( + input=messages, + api_key="", + original_response="first stream response received", + additional_args={"complete_input_dict": data}, + ) + + return streaming_response + + @property + def has_custom_stream_wrapper(self) -> bool: + """Indicates that this config has custom streaming support.""" + return True + + @property + def supports_stream_param_in_request_body(self) -> bool: + """Agent Engine does not allow passing `stream` in the request body.""" + return False + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return VertexAgentEngineError(status_code=status_code, message=error_message) + + def should_fake_stream( + self, + model: Optional[str], + stream: Optional[bool], + custom_llm_provider: Optional[str] = None, + ) -> bool: + """Agent Engine always returns SSE streams, so we use real streaming.""" + return False + diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 3cfa55c0606..6bb11430f20 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -5,7 +5,6 @@ from typing import Any, Dict, List, Literal, Optional, Set, Tuple, Union, get_ty import httpx import litellm -from litellm.utils import supports_response_schema, supports_system_messages from litellm._logging import verbose_logger from litellm.constants import DEFAULT_MAX_RECURSE_DEPTH from litellm.litellm_core_utils.prompt_templates.common_utils import unpack_defs @@ -14,6 +13,7 @@ from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.types.llms.openai import AllMessageValues from litellm.types.llms.vertex_ai import PartType, Schema from litellm.types.utils import TokenCountResponse +from litellm.utils import supports_response_schema, supports_system_messages class VertexAIError(BaseLLMException): @@ -36,6 +36,7 @@ class VertexAIModelRoute(str, Enum): MODEL_GARDEN = "model_garden" NON_GEMINI = "non_gemini" OPENAI_COMPATIBLE = "openai" + AGENT_ENGINE = "agent_engine" VERTEX_AI_MODEL_ROUTES = [f"{route.value}/" for route in VertexAIModelRoute] @@ -76,6 +77,10 @@ def get_vertex_ai_model_route( if litellm_params and litellm_params.get("base_model") is not None: if "gemini" in litellm_params["base_model"]: return VertexAIModelRoute.GEMINI + + # Check for agent_engine models (Reasoning Engines) + if "agent_engine/" in model: + return VertexAIModelRoute.AGENT_ENGINE # Check if numeric endpoint ID with custom api_base (PSC endpoint) # Route to GEMINI (HTTP path) to support PSC endpoints properly diff --git a/litellm/main.py b/litellm/main.py index b08ffd16e3d..4176c96d348 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -3242,6 +3242,37 @@ def completion( # type: ignore # noqa: PLR0915 timeout=timeout, client=client, ) + elif model_route == VertexAIModelRoute.AGENT_ENGINE: + # Vertex AI Agent Engine (Reasoning Engines) + from litellm.llms.vertex_ai.agent_engine.transformation import ( + VertexAgentEngineConfig, + ) + + vertex_agent_engine_config = VertexAgentEngineConfig() + + # Update litellm_params with vertex credentials + litellm_params["vertex_project"] = vertex_ai_project + litellm_params["vertex_location"] = vertex_ai_location + litellm_params["vertex_credentials"] = vertex_credentials + + model_response = base_llm_http_handler.completion( + model=model, + stream=stream, + messages=messages, + model_response=model_response, + optional_params=new_params, + litellm_params=litellm_params, # type: ignore + encoding=encoding, + api_key=None, + api_base=api_base, + logging_obj=logging, + acompletion=acompletion, + timeout=timeout, + client=client, + custom_llm_provider="vertex_ai", + provider_config=vertex_agent_engine_config, + headers=headers or {}, + ) else: # VertexAIModelRoute.NON_GEMINI model_response = vertex_ai_non_gemini.completion( model=model, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 01d7f076edc..2a7f8aa3ddf 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -30628,7 +30628,7 @@ "litellm_provider": "fireworks_ai", "mode": "embedding" }, - "fireworks_ai/accounts/fireworks/models/qwen3-embedding-8b": { + "fireworks_ai/accounts/fireworks/models/": { "max_tokens": 40960, "max_input_tokens": 40960, "max_output_tokens": 40960, diff --git a/litellm/proxy/public_endpoints/agent_create_fields.json b/litellm/proxy/public_endpoints/agent_create_fields.json index c559b61a76f..931c9a43498 100644 --- a/litellm/proxy/public_endpoints/agent_create_fields.json +++ b/litellm/proxy/public_endpoints/agent_create_fields.json @@ -166,6 +166,29 @@ "litellm_params_template": { "custom_llm_provider": "pydantic_ai_agents" } + }, + { + "agent_type": "vertex_agent_engine", + "agent_type_display_name": "Vertex AI Agent Engine", + "description": "Connect to Google Cloud Vertex AI Reasoning Engines", + "logo_url": "/ui/assets/logos/google.svg", + "inherit_credentials_from_provider": "Vertex_AI", + "model_template": "vertex_ai/agent_engine/{reasoning_engine_id}", + "credential_fields": [ + { + "key": "reasoning_engine_id", + "label": "Reasoning Engine Resource ID", + "placeholder": "projects/123456789/locations/us-central1/reasoningEngines/987654321", + "tooltip": "The full resource ID of your Vertex AI Reasoning Engine. Find this in Google Cloud Console under Vertex AI > Agent Builder > Your Agent.", + "required": true, + "field_type": "text", + "default_value": null, + "include_in_litellm_params": false + } + ], + "litellm_params_template": { + "custom_llm_provider": "vertex_ai" + } } ] diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 629760a7dd2..68264a576fe 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -2689,8 +2689,8 @@ "key": "vertex_credentials", "label": "Vertex Credentials", "placeholder": null, - "tooltip": null, - "required": true, + "tooltip": "Optional - Upload your GCP service account JSON file. If not provided, uses default GCP credentials (ADC).", + "required": false, "field_type": "upload", "options": null, "default_value": null diff --git a/tests/agent_tests/local_vertex_agent.py b/tests/agent_tests/local_vertex_agent.py new file mode 100644 index 00000000000..3cc9f868612 --- /dev/null +++ b/tests/agent_tests/local_vertex_agent.py @@ -0,0 +1,151 @@ +""" +Test script for Vertex AI Reasoning Engine. + +This script demonstrates how to: +1. Authenticate with Google Cloud +2. Send queries to a Vertex AI Reasoning Engine using the :query endpoint + +Usage: + python local_vertex_agent.py + +Requirements: + pip install httpx google-auth +""" + +import asyncio +import json +from uuid import uuid4 + +from google.auth import default +from google.auth.transport.requests import Request +import httpx + +# Configuration - update these for your agent +PROJECT_ID = "gen-lang-client-0682925754" # Your GCP project ID +LOCATION = "us-central1" # Your agent's location + +# For Reasoning Engines, use just the numeric ID at the end +REASONING_ENGINE_ID = "8263861224643493888" + +# The project number from the resource name +PROJECT_NUMBER = "1060139831167" + + +async def main(): + """Main function to test Vertex AI Reasoning Engine.""" + + # Step 1: Authenticate with Google Cloud + print("Step 1: Authenticating with Google Cloud...") + credentials, project = default(scopes=['https://www.googleapis.com/auth/cloud-platform']) + credentials.refresh(Request()) + print(f"Authenticated! Project: {project}") + print(f"Token (first 20 chars): {credentials.token[:20]}...") + + # Step 2: Build the endpoint URL + base_url = f"https://{LOCATION}-aiplatform.googleapis.com" + resource_path = f"projects/{PROJECT_NUMBER}/locations/{LOCATION}/reasoningEngines/{REASONING_ENGINE_ID}" + + # The Reasoning Engine uses :query endpoint with specific format + query_url = f"{base_url}/v1beta1/{resource_path}:query" + stream_url = f"{base_url}/v1beta1/{resource_path}:streamQuery" + + print(f"\nQuery URL: {query_url}") + print(f"Stream URL: {stream_url}") + + # Step 3: Create authenticated httpx client + print("\nStep 2: Creating authenticated HTTP client...") + client = httpx.AsyncClient( + headers={ + "Authorization": f"Bearer {credentials.token}", + "Content-Type": "application/json", + }, + timeout=120.0, + ) + + # Step 4: Build the query request (non-streaming) + # Note: For non-streaming, we need to: + # 1. Create a session + # 2. Use the streaming endpoint with stream_query method + # The :query endpoint only supports session management methods + + user_id = f"test-user-{uuid4().hex[:8]}" + + # First create a session + create_session_request = { + "class_method": "async_create_session", + "input": { + "user_id": user_id, + } + } + + print(f"\nStep 3: Creating session...") + print(f"User ID: {user_id}") + + async with client: + # Create session + print(f"\nSending to: {query_url}") + response = await client.post(query_url, json=create_session_request) + print(f"Create session status: {response.status_code}") + + if response.status_code == 200: + session_data = response.json() + print(f"Session created:\n{json.dumps(session_data, indent=2)}") + + # Extract session_id from response + session_id = session_data.get("output", {}).get("id") or session_data.get("output", {}).get("session_id") + print(f"\nSession ID: {session_id}") + + # Now send the actual query via streamQuery + query_request = { + "class_method": "stream_query", + "input": { + "message": "Hello! What can you do?", + "user_id": user_id, + "session_id": session_id, + } + } + + print(f"\nStep 4: Sending query via streamQuery...") + print(f"Request:\n{json.dumps(query_request, indent=2)}") + + # Use streaming endpoint but collect full response + async with client.stream("POST", stream_url, json=query_request) as stream_response: + print(f"Query status: {stream_response.status_code}") + + if stream_response.status_code == 200: + print("\nResponse:") + full_response = "" + async for line in stream_response.aiter_lines(): + if line: + full_response = line # Keep last line (full response) + + # Parse and display + try: + data = json.loads(full_response) + # Extract the text from the response + content = data.get("content", {}) + parts = content.get("parts", []) + for part in parts: + if "text" in part: + print(f"\nAgent response:\n{part['text']}") + except: + print(full_response) + else: + content = await stream_response.aread() + print(f"Error: {content.decode()}") + else: + print(f"Error creating session: {response.text}") + + +if __name__ == "__main__": + print("=" * 60) + print("Vertex AI Reasoning Engine Test Script") + print("=" * 60) + print(f"\nConfiguration:") + print(f" PROJECT_ID: {PROJECT_ID}") + print(f" PROJECT_NUMBER: {PROJECT_NUMBER}") + print(f" LOCATION: {LOCATION}") + print(f" REASONING_ENGINE_ID: {REASONING_ENGINE_ID}") + print() + + asyncio.run(main()) diff --git a/tests/agent_tests/test_a2a_completion_bridge.py b/tests/agent_tests/test_a2a_completion_bridge.py index 4191821f3de..224809dd7f5 100644 --- a/tests/agent_tests/test_a2a_completion_bridge.py +++ b/tests/agent_tests/test_a2a_completion_bridge.py @@ -201,3 +201,79 @@ async def test_a2a_completion_bridge_bedrock_agentcore(): print(f"Received {len(chunks)} chunks from Bedrock AgentCore") + +# ============================================================ +# Vertex AI Agent Engine Tests +# ============================================================ + +# Configuration - update these for your Vertex AI Reasoning Engine +VERTEX_AGENT_RESOURCE_NAME = "projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888" + + +@pytest.mark.asyncio +async def test_vertex_agent_engine_non_streaming(): + """ + Test non-streaming request to Vertex AI Agent Engine via litellm.acompletion. + + Uses the Reasoning Engine resource ID to call a hosted agent. + """ + + litellm._turn_on_debug() + + # Call via litellm.acompletion with vertex_ai/agent_engine/ prefix + response = await litellm.acompletion( + model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", + messages=[{"role": "user", "content": "Hello! What can you do?"}], + stream=False, + ) + + print(f"\n=== Vertex Agent Engine Non-Streaming Response ===") + print(f"Response: {response}") + + # Basic assertions + assert response is not None + assert hasattr(response, "choices") + assert len(response.choices) > 0 + assert response.choices[0].message is not None + assert response.choices[0].message.content is not None + assert len(response.choices[0].message.content) > 0 + + print(f"Agent response: {response.choices[0].message.content[:200]}...") + + +@pytest.mark.asyncio +async def test_vertex_agent_engine_streaming(): + """ + Test streaming request to Vertex AI Agent Engine via litellm.acompletion. + + Uses the Reasoning Engine resource ID to call a hosted agent with streaming. + """ + #litellm._turn_on_debug() + + # Call via litellm.acompletion with streaming + response = await litellm.acompletion( + model=f"vertex_ai/agent_engine/{VERTEX_AGENT_RESOURCE_NAME}", + messages=[{"role": "user", "content": "Hello! What can you do?"}], + stream=True, + ) + + print(f"\n=== Vertex Agent Engine Streaming Response ===") + + chunks = [] + full_content = "" + async for chunk in response: + print(f"Chunk: {chunk}") + # chunks.append(chunk) + # if hasattr(chunk, "choices") and len(chunk.choices) > 0: + # delta = chunk.choices[0].delta + # if hasattr(delta, "content") and delta.content: + # full_content += delta.content + # print(f"Chunk: {delta.content}", end="", flush=True) + + # # print(f"\n\nReceived {len(chunks)} chunks") + # print(f"Full content: {full_content[:200]}...") + + # # Basic assertions + # assert len(chunks) > 0 + # assert len(full_content) > 0 + diff --git a/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py b/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py new file mode 100644 index 00000000000..cb3a5807d8c --- /dev/null +++ b/tests/litellm/llms/vertex_ai/agent_engine/test_transformation.py @@ -0,0 +1,128 @@ +""" +Tests for Vertex AI Agent Engine transformation. + +Tests the request transformation and streaming chunk parsing without making real API calls. +""" + +import os +import sys + +import pytest + +sys.path.insert(0, os.path.abspath("../../../../..")) + +from litellm.llms.vertex_ai.agent_engine.sse_iterator import ( + VertexAgentEngineResponseIterator, +) +from litellm.llms.vertex_ai.agent_engine.transformation import VertexAgentEngineConfig + + +class TestVertexAgentEngineTransformRequest: + """Tests for transform_request method.""" + + def test_transform_request_basic(self): + """ + Test that transform_request correctly formats messages into Vertex Agent Engine payload. + """ + config = VertexAgentEngineConfig() + + messages = [{"role": "user", "content": "Hello, what can you do?"}] + optional_params = {"user_id": "test-user-123"} + litellm_params = {} + + result = config.transform_request( + model="agent_engine/123456789", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + assert result["class_method"] == "stream_query" + assert result["input"]["message"] == "Hello, what can you do?" + assert result["input"]["user_id"] == "test-user-123" + assert "session_id" not in result["input"] + + def test_transform_request_with_session_id(self): + """ + Test that transform_request includes session_id when provided. + """ + config = VertexAgentEngineConfig() + + messages = [{"role": "user", "content": "Follow up question"}] + optional_params = { + "user_id": "test-user-123", + "session_id": "session-abc-456", + } + litellm_params = {} + + result = config.transform_request( + model="agent_engine/123456789", + messages=messages, + optional_params=optional_params, + litellm_params=litellm_params, + headers={}, + ) + + assert result["class_method"] == "stream_query" + assert result["input"]["message"] == "Follow up question" + assert result["input"]["user_id"] == "test-user-123" + assert result["input"]["session_id"] == "session-abc-456" + + +class TestVertexAgentEngineChunkParser: + """Tests for the streaming chunk parser.""" + + def test_chunk_parser_with_text_content(self): + """ + Test that chunk_parser correctly extracts text from Vertex Agent Engine response format. + """ + iterator = VertexAgentEngineResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "content": { + "parts": [{"text": "Hello! I can help you with financial analysis."}], + "role": "model", + }, + "finish_reason": "STOP", + "usage_metadata": { + "prompt_token_count": 100, + "candidates_token_count": 50, + "total_token_count": 150, + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.content == "Hello! I can help you with financial analysis." + assert result.choices[0].delta.role == "assistant" + assert result.choices[0].finish_reason == "stop" + assert result.usage["prompt_tokens"] == 100 + assert result.usage["completion_tokens"] == 50 + assert result.usage["total_tokens"] == 150 + + def test_chunk_parser_without_finish_reason(self): + """ + Test that chunk_parser handles chunks without finish_reason (intermediate chunks). + """ + iterator = VertexAgentEngineResponseIterator( + streaming_response=iter([]), + sync_stream=True, + ) + + chunk = { + "content": { + "parts": [{"text": "Partial response..."}], + "role": "model", + }, + } + + result = iterator.chunk_parser(chunk) + + assert result.choices[0].delta.content == "Partial response..." + assert result.choices[0].finish_reason is None + assert result.usage is None + From f90dea7315c424fc3468c03b05dcf6e1baf12695 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 16 Dec 2025 08:50:21 +0530 Subject: [PATCH 15/35] fix(docker-compose.yml): move to docker.litellm.ai --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 8898aff62da..988860a7877 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ services: context: . args: target: runtime - image: ghcr.io/berriai/litellm:main-stable + image: docker.litellm.ai/berriai/litellm:main-stable ######################################### ## Uncomment these lines to start proxy with a config.yaml file ## # volumes: From ce113f4e4b61160d2c06e2ea7d53fa3121fbb683 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 15 Dec 2025 19:43:12 -0800 Subject: [PATCH 16/35] [Docs] Add docs on using pydantic ai agents with LiteLLM A2a gateway (#18026) * init A2AProviderConfigManager * move file * move file * add pydnatic ai folder * init providers * test_pydantic_ai_non_streaming * fix import * INIT pydantic * use_a2a_form_fields * test_vertex_agent_engine_streaming * add agent_engine * init transform for agent engine * init agent engine * VertexAgentEngineSSEStreamIterator * sample * ui add new fields * fix vertex_credentials * working SSE iterator * TestVertexAgentEngineTransformRequest * fix code QA check * stash docs * docs fix * fix logo * docs fix * doc pydantic ai * docs pydantic ai * new provider * docs fix --- docs/my-website/docs/a2a.md | 7 +- .../docs/providers/pydantic_ai_agent.md | 121 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + .../out/assets/logos/pydantic.svg | 5 + provider_endpoints_support.json | 17 +++ 5 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/providers/pydantic_ai_agent.md create mode 100644 litellm/proxy/_experimental/out/assets/logos/pydantic.svg diff --git a/docs/my-website/docs/a2a.md b/docs/my-website/docs/a2a.md index 9c94a2fbf29..b41daa37bfb 100644 --- a/docs/my-website/docs/a2a.md +++ b/docs/my-website/docs/a2a.md @@ -16,7 +16,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque | Feature | Supported | |---------|-----------| -| Supported Agent Providers | A2A, LangGraph, Azure AI Foundry, Bedrock AgentCore | +| Supported Agent Providers | A2A, Pydantic AI, LangGraph, Azure AI Foundry, Bedrock AgentCore | | Logging | ✅ | | Load Balancing | ✅ | | Streaming | ✅ | @@ -45,6 +45,7 @@ You can add A2A-compatible agents through the LiteLLM Admin UI. The URL should be the invocation URL for your A2A agent (e.g., `http://localhost:10001`). + ### Add Azure AI Foundry Agents Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway) @@ -57,6 +58,10 @@ Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./prov Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway) +### Add Pydantic AI Agents + +Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway) + ## Invoking your Agents Use the [A2A Python SDK](https://pypi.org/project/a2a/) to invoke agents through LiteLLM. diff --git a/docs/my-website/docs/providers/pydantic_ai_agent.md b/docs/my-website/docs/providers/pydantic_ai_agent.md new file mode 100644 index 00000000000..e96295faaf3 --- /dev/null +++ b/docs/my-website/docs/providers/pydantic_ai_agent.md @@ -0,0 +1,121 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Pydantic AI Agents + +Call Pydantic AI Agents via LiteLLM's A2A Gateway. + +| Property | Details | +|----------|---------| +| Description | Pydantic AI agents with native A2A support via the `to_a2a()` method. LiteLLM provides fake streaming support for agents that don't natively stream. | +| Provider Route on LiteLLM | A2A Gateway | +| Supported Endpoints | `/v1/a2a/message/send` | +| Provider Doc | [Pydantic AI Agents ↗](https://ai.pydantic.dev/agents/) | + +## LiteLLM A2A Gateway + +All Pydantic AI agents need to be exposed as A2A agents using the `to_a2a()` method. Once your agent server is running, you can add it to the LiteLLM Gateway. + +### 1. Setup Pydantic AI Agent Server + +LiteLLM requires Pydantic AI agents to follow the [A2A (Agent-to-Agent) protocol](https://github.com/google/A2A). Pydantic AI has native A2A support via the `to_a2a()` method, which exposes your agent as an A2A-compliant server. + +#### Install Dependencies + +```bash +pip install pydantic-ai fasta2a uvicorn +``` + +#### Create Agent + +```python title="agent.py" +from pydantic_ai import Agent + +agent = Agent('openai:gpt-4o-mini', instructions='Be helpful!') + +@agent.tool_plain +def get_weather(city: str) -> str: + """Get weather for a city.""" + return f"Weather in {city}: Sunny, 72°F" + +@agent.tool_plain +def calculator(expression: str) -> str: + """Evaluate a math expression.""" + return str(eval(expression)) + +# Native A2A server - Pydantic AI handles it automatically +app = agent.to_a2a() +``` + +#### Run Server + +```bash +uvicorn agent:app --host 0.0.0.0 --port 9999 +``` + +Server runs at `http://localhost:9999` + +### 2. Navigate to Agents + +From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent". + +### 3. Select Pydantic AI Agent Type + +Click "A2A Standard" to see available agent types, then select "Pydantic AI". + +![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/1055acb1-064b-4465-8e6a-8278291bc661/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=395,147) + +![Select Pydantic AI](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/0998e38c-8534-40f1-931a-be96c2cae0ad/ascreenshot.jpeg?tl_px=0,52&br_px=2201,1283&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=421,277) + +### 4. Configure the Agent + +Fill in the following fields: + +- **Agent Name** - A unique identifier for your agent (e.g., `test-pydantic-agent`) +- **Agent URL** - The URL where your Pydantic AI agent is running. We use `http://localhost:9999` because that's where we started our Pydantic AI agent server in the previous step. + +![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/8cf3fbde-05f3-48d1-81b6-6f857bd6d360/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=443,225) + +![Configure Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb555808-4761-4c49-a415-200ac1bdb525/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0) + +![Enter Agent URL](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/303eae61-4352-4fb0-a537-806839c234ba/ascreenshot.jpeg?tl_px=0,212&br_px=2201,1443&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=456,277) + +### 5. Create Agent + +Click "Create Agent" to save your configuration. + +![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/914f3367-df7d-4244-bd4d-e99ce0a6193a/ascreenshot.jpeg?tl_px=416,438&br_px=2618,1669&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=690,277) + +### 6. Test in Playground + +Go to "Playground" in the sidebar to test your agent. + +![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/c73c9f3b-22af-4105-aafa-2d34c4986ef3/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=44,97) + +### 7. Select A2A Endpoint + +Click the endpoint dropdown and search for "a2a", then select `/v1/a2a/message/send`. + +![Click Endpoint Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/196d97ac-bcba-47f0-9880-97b80250e00c/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=261,230) + +![Search for A2A](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/26b68f21-29f9-4c4c-b8b5-d2e11cbfd14a/ascreenshot.jpeg?tl_px=0,0&br_px=2617,1463&force_format=jpeg&q=100&width=1120.0) + +![Select A2A Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/41576fb1-d385-4fb2-84e9-142dd7fe5181/ascreenshot.jpeg?tl_px=0,0&br_px=2201,1230&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=307,270) + +### 8. Select Your Agent and Send a Message + +Pick your Pydantic AI agent from the dropdown and send a test message. + +![Click Agent Dropdown](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a96d7967-3d54-4cbf-bd3e-b38f1be9df76/ascreenshot.jpeg?tl_px=0,54&br_px=2201,1285&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=274,277) + +![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/e05a5a6e-d044-4480-b94e-7c03cfb92ac5/ascreenshot.jpeg?tl_px=0,113&br_px=2201,1344&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=290,277) + +![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/29162702-968a-401a-aac1-c844bfc5f4a3/ascreenshot.jpeg?tl_px=91,653&br_px=2292,1883&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,436) + + +## Further Reading + +- [Pydantic AI Documentation](https://ai.pydantic.dev/) +- [Pydantic AI Agents](https://ai.pydantic.dev/agents/) +- [A2A Agent Gateway](../a2a.md) +- [A2A Cost Tracking](../a2a_cost_tracking.md) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 954a27d3182..3c8b23f856d 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -738,6 +738,7 @@ const sidebars = { "providers/petals", "providers/publicai", "providers/predibase", + "providers/pydantic_ai_agent", "providers/ragflow", "providers/recraft", "providers/replicate", diff --git a/litellm/proxy/_experimental/out/assets/logos/pydantic.svg b/litellm/proxy/_experimental/out/assets/logos/pydantic.svg new file mode 100644 index 00000000000..0ff8e5c44c7 --- /dev/null +++ b/litellm/proxy/_experimental/out/assets/logos/pydantic.svg @@ -0,0 +1,5 @@ + + + diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index cec9f9e37c0..c5da3052738 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1946,6 +1946,23 @@ "rerank": false, "a2a": true } + }, + "pydantic_ai_agents": { + "display_name": "Pydantic AI Agents (`pydantic_ai_agents`)", + "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } } } } \ No newline at end of file From cd9b71093ac6c19f68ae5df214e9d1709e24205b Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 16 Dec 2025 01:08:39 -0300 Subject: [PATCH 17/35] Improve issue labeling: add component dropdown and more provider keywords (#17957) - Replace "ML Ops Team" dropdown with "What part of LiteLLM is this about?" with options: SDK, Proxy, UI Dashboard, Docs, Other - Add component dropdown to feature_request.yml template - Rename label-mlops.yml to label-component.yml and update to auto-label issues with sdk, proxy, ui-dashboard, or docs based on selection - Add more provider keywords to issue-keyword-labeler: gemini, cohere, mistral, groq, ollama, deepseek --- .github/ISSUE_TEMPLATE/bug_report.yml | 12 +- .github/ISSUE_TEMPLATE/feature_request.yml | 12 ++ .github/workflows/issue-keyword-labeler.yml | 2 +- .github/workflows/label-component.yml | 144 ++++++++++++++++++++ .github/workflows/label-mlops.yml | 17 --- 5 files changed, 164 insertions(+), 23 deletions(-) create mode 100644 .github/workflows/label-component.yml delete mode 100644 .github/workflows/label-mlops.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 8fbf1b3c5b4..39b46cba999 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -23,13 +23,15 @@ body: description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. render: shell - type: dropdown - id: ml-ops-team + id: component attributes: - label: Are you a ML Ops Team? - description: This helps us prioritize your requests correctly + label: What part of LiteLLM is this about? options: - - "No" - - "Yes" + - "SDK (litellm Python package)" + - "Proxy" + - "UI Dashboard" + - "Docs" + - "Other" validations: required: true - type: input diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 13a2132ec95..96b95cc7f02 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -22,6 +22,18 @@ body: description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. validations: required: true + - type: dropdown + id: component + attributes: + label: What part of LiteLLM is this about? + options: + - "SDK (litellm Python package)" + - "Proxy" + - "UI Dashboard" + - "Docs" + - "Other" + validations: + required: true - type: dropdown id: hiring-interest attributes: diff --git a/.github/workflows/issue-keyword-labeler.yml b/.github/workflows/issue-keyword-labeler.yml index 60c18e3b9af..936f90f747f 100644 --- a/.github/workflows/issue-keyword-labeler.yml +++ b/.github/workflows/issue-keyword-labeler.yml @@ -19,7 +19,7 @@ jobs: id: scan env: PROVIDER_ISSUE_WEBHOOK_URL: ${{ secrets.PROVIDER_ISSUE_WEBHOOK_URL }} - KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic + KEYWORDS: azure,openai,bedrock,vertexai,vertex ai,anthropic,gemini,cohere,mistral,groq,ollama,deepseek run: python3 .github/scripts/scan_keywords.py - name: Ensure label exists diff --git a/.github/workflows/label-component.yml b/.github/workflows/label-component.yml new file mode 100644 index 00000000000..c0f9436288c --- /dev/null +++ b/.github/workflows/label-component.yml @@ -0,0 +1,144 @@ +name: Label Component Issues + +on: + issues: + types: + - opened + +jobs: + add-component-label: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - name: Add SDK label + if: contains(github.event.issue.body, 'SDK (litellm Python package)') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'sdk'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: '0E7C86', + description: 'Issues related to the litellm Python SDK' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add Proxy label + if: contains(github.event.issue.body, 'Proxy') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'proxy'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: '5319E7', + description: 'Issues related to the LiteLLM Proxy' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add UI Dashboard label + if: contains(github.event.issue.body, 'UI Dashboard') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'ui-dashboard'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: 'D876E3', + description: 'Issues related to the LiteLLM UI Dashboard' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); + + - name: Add Docs label + if: contains(github.event.issue.body, 'Docs') + uses: actions/github-script@v7 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const labelName = 'docs'; + try { + await github.rest.issues.getLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName + }); + } catch (error) { + if (error.status === 404) { + await github.rest.issues.createLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + name: labelName, + color: 'FBCA04', + description: 'Issues related to LiteLLM documentation' + }); + } else { + throw error; + } + } + await github.rest.issues.addLabels({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + labels: [labelName] + }); diff --git a/.github/workflows/label-mlops.yml b/.github/workflows/label-mlops.yml deleted file mode 100644 index 37789c1ea76..00000000000 --- a/.github/workflows/label-mlops.yml +++ /dev/null @@ -1,17 +0,0 @@ -name: Label ML Ops Team Issues - -on: - issues: - types: - - opened - -jobs: - add-mlops-label: - runs-on: ubuntu-latest - steps: - - name: Check if ML Ops Team is selected - uses: actions-ecosystem/action-add-labels@v1 - if: contains(github.event.issue.body, '### Are you a ML Ops Team?') && contains(github.event.issue.body, 'Yes') - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - labels: "mlops user request" From 13c0ab898595d8cffe3acba706c91dfbeed3a9a8 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Tue, 16 Dec 2025 01:09:22 -0300 Subject: [PATCH 18/35] Cleanup PR template: remove redundant fields (#17956) - Remove Title section (already in PR title) - Remove screenshot requirement (CI validates tests) Co-authored-by: Krish Dholakia --- .github/pull_request_template.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 8977332ee01..b91b16c955c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,7 +1,3 @@ -## Title - - - ## Relevant issues @@ -11,7 +7,6 @@ **Please complete all items before asking a LiteLLM maintainer to review your PR** - [ ] I have Added testing in the [`tests/litellm/`](https://github.com/BerriAI/litellm/tree/main/tests/litellm) directory, **Adding at least 1 test is a hard requirement** - [see details](https://docs.litellm.ai/docs/extras/contributing_code) -- [ ] I have added a screenshot of my new test passing locally - [ ] My PR passes all unit tests on [`make test-unit`](https://docs.litellm.ai/docs/extras/contributing_code) - [ ] My PR's scope is as isolated as possible, it only solves 1 specific problem From 244d83ff474f129a66a5b645b4d032f6fa61df17 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Mon, 15 Dec 2025 20:11:40 -0800 Subject: [PATCH 19/35] [Docs] Litellm add docs vertex ai engine (#18027) * new provider doc * add to sidebar * stash docs * docs fix * docs vertex agent engine --- docs/my-website/docs/a2a.md | 10 +- .../docs/providers/vertex_ai_agent_engine.md | 216 ++++++++++++++++++ docs/my-website/sidebars.js | 1 + provider_endpoints_support.json | 17 ++ 4 files changed, 241 insertions(+), 3 deletions(-) create mode 100644 docs/my-website/docs/providers/vertex_ai_agent_engine.md diff --git a/docs/my-website/docs/a2a.md b/docs/my-website/docs/a2a.md index b41daa37bfb..d7145e4b83c 100644 --- a/docs/my-website/docs/a2a.md +++ b/docs/my-website/docs/a2a.md @@ -16,7 +16,7 @@ Add A2A Agents on LiteLLM AI Gateway, Invoke agents in A2A Protocol, track reque | Feature | Supported | |---------|-----------| -| Supported Agent Providers | A2A, Pydantic AI, LangGraph, Azure AI Foundry, Bedrock AgentCore | +| Supported Agent Providers | A2A, Vertex AI Agent Engine, LangGraph, Azure AI Foundry, Bedrock AgentCore, Pydantic AI | | Logging | ✅ | | Load Balancing | ✅ | | Streaming | ✅ | @@ -50,14 +50,18 @@ The URL should be the invocation URL for your A2A agent (e.g., `http://localhost Follow [this guide, to add your azure ai foundry agent to LiteLLM Agent Gateway](./providers/azure_ai_agents#litellm-a2a-gateway) -### Add LangGraph Agents +### Add Vertex AI Agent Engine -Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway) +Follow [this guide, to add your Vertex AI Agent Engine to LiteLLM Agent Gateway](./providers/vertex_ai_agent_engine) ### Add Bedrock AgentCore Agents Follow [this guide, to add your bedrock agentcore agent to LiteLLM Agent Gateway](./providers/bedrock_agentcore#litellm-a2a-gateway) +### Add LangGraph Agents + +Follow [this guide, to add your langgraph agent to LiteLLM Agent Gateway](./providers/langgraph#litellm-a2a-gateway) + ### Add Pydantic AI Agents Follow [this guide, to add your pydantic ai agent to LiteLLM Agent Gateway](./providers/pydantic_ai_agent#litellm-a2a-gateway) diff --git a/docs/my-website/docs/providers/vertex_ai_agent_engine.md b/docs/my-website/docs/providers/vertex_ai_agent_engine.md new file mode 100644 index 00000000000..3bd40e98684 --- /dev/null +++ b/docs/my-website/docs/providers/vertex_ai_agent_engine.md @@ -0,0 +1,216 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# Vertex AI Agent Engine + +Call Vertex AI Agent Engine (Reasoning Engines) in the OpenAI Request/Response format. + +| Property | Details | +|----------|---------| +| Description | Vertex AI Agent Engine provides hosted agent runtimes that can execute agentic workflows with foundation models, tools, and custom logic. | +| Provider Route on LiteLLM | `vertex_ai/agent_engine/{RESOURCE_NAME}` | +| Supported Endpoints | `/chat/completions`, `/v1/messages`, `/v1/responses`, `/v1/a2a/message/send` | +| Provider Doc | [Vertex AI Agent Engine ↗](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) | + +## Quick Start + +### Model Format + +```shell showLineNumbers title="Model Format" +vertex_ai/agent_engine/{RESOURCE_NAME} +``` + +**Example:** +- `vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888` + +### LiteLLM Python SDK + +```python showLineNumbers title="Basic Agent Completion" +import litellm + +response = litellm.completion( + model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888", + messages=[ + {"role": "user", "content": "Explain machine learning in simple terms"} + ], +) + +print(response.choices[0].message.content) +``` + +```python showLineNumbers title="Streaming Agent Responses" +import litellm + +response = await litellm.acompletion( + model="vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888", + messages=[ + {"role": "user", "content": "What are the key principles of software architecture?"} + ], + stream=True, +) + +async for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +``` + +### LiteLLM Proxy + +#### 1. Configure your model in config.yaml + + + + +```yaml showLineNumbers title="LiteLLM Proxy Configuration" +model_list: + - model_name: vertex-agent-1 + litellm_params: + model: vertex_ai/agent_engine/projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888 + vertex_project: your-project-id + vertex_location: us-central1 +``` + + + + +#### 2. Start the LiteLLM Proxy + +```bash showLineNumbers title="Start LiteLLM Proxy" +litellm --config config.yaml +``` + +#### 3. Make requests to your Vertex AI Agent Engine + + + + +```bash showLineNumbers title="Basic Agent Request" +curl http://localhost:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "model": "vertex-agent-1", + "messages": [ + {"role": "user", "content": "Summarize the main benefits of cloud computing"} + ] + }' +``` + + + + + +```python showLineNumbers title="Using OpenAI SDK with LiteLLM Proxy" +from openai import OpenAI + +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +response = client.chat.completions.create( + model="vertex-agent-1", + messages=[ + {"role": "user", "content": "What are best practices for API design?"} + ] +) + +print(response.choices[0].message.content) +``` + + + + +## LiteLLM A2A Gateway + +You can also connect to Vertex AI Agent Engine through LiteLLM's A2A (Agent-to-Agent) Gateway UI. This provides a visual way to register and test agents without writing code. + +### 1. Navigate to Agents + +From the sidebar, click "Agents" to open the agent management page, then click "+ Add New Agent". + +![Click Agents](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9a979927-ce6b-4168-9fba-e53e28f1c2c4/ascreenshot.jpeg?tl_px=0,14&br_px=1376,783&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=17,277) + +![Add New Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a311750c-2e85-4589-99cb-2ce7e4021e77/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=195,257) + +### 2. Select Vertex AI Agent Engine Type + +Click "A2A Standard" to see available agent types, then select "Vertex AI Agent Engine". + +![Select A2A Standard](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/5b1acc4c-dc3f-4639-b4a0-e64b35c228fd/ascreenshot.jpeg?tl_px=52,0&br_px=1428,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,271) + +![Select Vertex AI Agent Engine](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/2f3bab61-3e02-4db7-84f0-82200a0f4136/ascreenshot.jpeg?tl_px=0,244&br_px=1376,1013&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=477,277) + +### 3. Configure the Agent + +Fill in the following fields: + +- **Agent Name** - A friendly name for your agent (e.g., `my-vertex-agent`) +- **Reasoning Engine Resource ID** - The full resource path from Google Cloud Console (e.g., `projects/1060139831167/locations/us-central1/reasoningEngines/8263861224643493888`) +- **Vertex Project** - Your Google Cloud project ID +- **Vertex Location** - The region where your agent is deployed (e.g., `us-central1`) + +![Enter Agent Name](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/695b84c7-9511-4337-bf19-f4505ab2b72b/ascreenshot.jpeg?tl_px=0,90&br_px=1376,859&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=480,276) + +![Enter Resource ID](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/ddce64df-b3a3-4519-ab62-f137887bcea2/ascreenshot.jpeg?tl_px=0,294&br_px=1376,1063&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=440,277) + +You can find the Resource ID in Google Cloud Console under Vertex AI > Agent Engine: + +![Copy Resource ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/185d7f17-cbaa-45de-948d-49d2091805ea/ascreenshot.jpeg?tl_px=0,165&br_px=1376,934&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=493,276) + +![Enter Vertex Project](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/a64da441-3e61-4811-a1e3-9f0b12c949ff/ascreenshot.jpeg?tl_px=0,233&br_px=1376,1002&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=501,277) + +You can find the Project ID in Google Cloud Console: + +![Copy Project ID from Google Cloud Console](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9ecad3bb-a534-42d6-9604-33906014fad6/user_cropped_screenshot.webp?tl_px=0,0&br_px=1728,1028&force_format=jpeg&q=100&width=1120.0) + +![Enter Vertex Location](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/316d1f38-4fb7-4377-86b6-c0fe7ac24383/ascreenshot.jpeg?tl_px=0,330&br_px=1376,1099&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=423,277) + +### 4. Create Agent + +Click "Create Agent" to save your configuration. + +![Create Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fb04b95d-793f-4eed-acf4-d1b3b5fa65e9/ascreenshot.jpeg?tl_px=352,347&br_px=1728,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=623,498) + +### 5. Test in Playground + +Go to "Playground" in the sidebar to test your agent. + +![Go to Playground](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/9e01369b-6102-4fe3-96a7-90082cadfd6e/ascreenshot.jpeg?tl_px=0,0&br_px=1376,769&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=55,226) + +### 6. Select A2A Endpoint + +Click the endpoint dropdown and select `/v1/a2a/message/send`. + +![Select Endpoint](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/d5aeac35-531b-4cf0-af2d-88f0a71fd736/ascreenshot.jpeg?tl_px=0,146&br_px=1376,915&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=299,277) + +### 7. Select Your Agent and Send a Message + +Pick your Vertex AI Agent Engine from the dropdown and send a test message. + +![Select Agent](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/353431f3-a0ba-4436-865d-ae11595e9cc4/ascreenshot.jpeg?tl_px=0,263&br_px=1376,1032&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=270,277) + +![Send Message](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/fbfce72e-f50b-43e1-b6e5-0d41192d8e2d/ascreenshot.jpeg?tl_px=95,347&br_px=1471,1117&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=524,474) + +![Agent Response](https://ajeuwbhvhr.cloudimg.io/https://colony-recorder.s3.amazonaws.com/files/2025-12-16/892dd826-fbf9-4530-8d82-95270889274a/ascreenshot.jpeg?tl_px=0,82&br_px=1376,851&force_format=jpeg&q=100&width=1120.0&wat=1&wat_opacity=0.7&wat_gravity=northwest&wat_url=https://colony-recorder.s3.us-west-1.amazonaws.com/images/watermarks/FB923C_standard.png&wat_pad=485,277) + +## Environment Variables + +| Variable | Description | +|----------|-------------| +| `GOOGLE_APPLICATION_CREDENTIALS` | Path to service account JSON key file | +| `VERTEXAI_PROJECT` | Google Cloud project ID | +| `VERTEXAI_LOCATION` | Google Cloud region (default: `us-central1`) | + +```bash +export GOOGLE_APPLICATION_CREDENTIALS="/path/to/service-account.json" +export VERTEXAI_PROJECT="your-project-id" +export VERTEXAI_LOCATION="us-central1" +``` + +## Further Reading + +- [Vertex AI Agent Engine Documentation](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/overview) +- [Create a Reasoning Engine](https://cloud.google.com/vertex-ai/generative-ai/docs/reasoning-engine/create) +- [A2A Agent Gateway](../a2a.md) +- [Vertex AI Provider](./vertex.md) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 3c8b23f856d..c64ac1e30fe 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -632,6 +632,7 @@ const sidebars = { "providers/vertex_speech", "providers/vertex_batch", "providers/vertex_ocr", + "providers/vertex_ai_agent_engine", ] }, { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index c5da3052738..d63b26d55fe 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -1947,6 +1947,23 @@ "a2a": true } }, + "vertex_ai/agent_engine": { + "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", + "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } + }, "pydantic_ai_agents": { "display_name": "Pydantic AI Agents (`pydantic_ai_agents`)", "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", From 0eba879179375961a205f32ee74a1d60dd767ef7 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Mon, 15 Dec 2025 20:12:44 -0800 Subject: [PATCH 20/35] Add ability to add embedding model for milvus --- .../VectorStoreForm.tsx | 94 +++++++++++++++---- .../src/components/vector_store_providers.tsx | 10 +- 2 files changed, 85 insertions(+), 19 deletions(-) diff --git a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx index d1cd3c5e443..8be879b2239 100644 --- a/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx +++ b/ui/litellm-dashboard/src/components/vector_store_management/VectorStoreForm.tsx @@ -1,4 +1,4 @@ -import React, { useState } from "react"; +import React, { useState, useEffect } from "react"; import { TextInput, Button as TremorButton } from "@tremor/react"; import { Modal, Form, Select, Tooltip, Input, Alert } from "antd"; import { InfoCircleOutlined } from "@ant-design/icons"; @@ -10,6 +10,7 @@ import { getProviderSpecificFields, VectorStoreFieldConfig, } from "../vector_store_providers"; +import { fetchAvailableModels, ModelGroup } from "../playground/llm_calls/fetch_models"; import NotificationsManager from "../molecules/notifications_manager"; interface VectorStoreFormProps { @@ -30,6 +31,24 @@ const VectorStoreForm: React.FC = ({ const [form] = Form.useForm(); const [metadataJson, setMetadataJson] = useState("{}"); const [selectedProvider, setSelectedProvider] = useState("bedrock"); + const [modelInfo, setModelInfo] = useState([]); + + useEffect(() => { + if (!accessToken) return; + + const loadModels = async () => { + try { + const uniqueModels = await fetchAvailableModels(accessToken); + if (uniqueModels.length > 0) { + setModelInfo(uniqueModels); + } + } catch (error) { + console.error("Error fetching model info:", error); + } + }; + + loadModels(); + }, [accessToken]); const handleCreate = async (formValues: any) => { if (!accessToken) return; @@ -207,23 +226,62 @@ const VectorStoreForm: React.FC = ({ {/* Provider-specific fields */} - {getProviderSpecificFields(selectedProvider).map((field: VectorStoreFieldConfig) => ( - - {field.label}{" "} - - - - - } - name={field.name} - rules={field.required ? [{ required: true, message: `Please input the ${field.label.toLowerCase()}` }] : []} - > - - - ))} + {getProviderSpecificFields(selectedProvider).map((field: VectorStoreFieldConfig) => { + if (field.type === "select") { + const embeddingModels = modelInfo + .filter((option: ModelGroup) => option.mode === "embedding") + .map((option: ModelGroup) => ({ + value: option.model_group, + label: option.model_group, + })); + + return ( + + {field.label}{" "} + + + + + } + name={field.name} + rules={ + field.required ? [{ required: true, message: `Please select the ${field.label.toLowerCase()}` }] : [] + } + > +