From 831ad45c4df36eeb401d718157ae391151e693ab Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Tue, 2 Dec 2025 18:18:08 +0530 Subject: [PATCH 01/24] Add ragflow support --- docs/my-website/docs/providers/ragflow.md | 244 ++++++++++++ docs/my-website/sidebars.js | 1 + litellm/__init__.py | 1 + litellm/constants.py | 1 + .../get_llm_provider_logic.py | 10 + litellm/llms/ragflow/__init__.py | 8 + litellm/llms/ragflow/chat/__init__.py | 4 + litellm/llms/ragflow/chat/transformation.py | 264 ++++++++++++ litellm/main.py | 30 ++ ...odel_prices_and_context_window_backup.json | 360 +++++++++++++++-- litellm/types/utils.py | 1 + litellm/utils.py | 2 + .../llms/ragflow/chat/__init__.py | 4 + .../chat/test_ragflow_chat_transformation.py | 376 ++++++++++++++++++ 14 files changed, 1283 insertions(+), 23 deletions(-) create mode 100644 docs/my-website/docs/providers/ragflow.md create mode 100644 litellm/llms/ragflow/__init__.py create mode 100644 litellm/llms/ragflow/chat/__init__.py create mode 100644 litellm/llms/ragflow/chat/transformation.py create mode 100644 tests/test_litellm/llms/ragflow/chat/__init__.py create mode 100644 tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py diff --git a/docs/my-website/docs/providers/ragflow.md b/docs/my-website/docs/providers/ragflow.md new file mode 100644 index 00000000000..73223bd07b5 --- /dev/null +++ b/docs/my-website/docs/providers/ragflow.md @@ -0,0 +1,244 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +# RAGFlow + +Litellm supports Ragflow's chat completions APIs + +## Supported Features + +- ✅ Chat completions +- ✅ Streaming responses +- ✅ Both chat and agent endpoints +- ✅ Multiple credential sources (params, env vars, litellm_params) +- ✅ OpenAI-compatible API format + + +## API Key + +```python +# env variable +os.environ['RAGFLOW_API_KEY'] +``` + +## API Base + +```python +# env variable +os.environ['RAGFLOW_API_BASE'] +``` + +## Overview + +RAGFlow provides OpenAI-compatible APIs with unique path structures that include chat and agent IDs: + +- **Chat endpoint**: `/api/v1/chats_openai/{chat_id}/chat/completions` +- **Agent endpoint**: `/api/v1/agents_openai/{agent_id}/chat/completions` + +The model name format embeds the endpoint type and ID: +- Chat: `ragflow/chat/{chat_id}/{model_name}` +- Agent: `ragflow/agent/{agent_id}/{model_name}` + + +## Sample Usage - Chat Endpoint + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" # or your hosted URL + +response = completion( + model="ragflow/chat/my-chat-id/gpt-4o-mini", + messages=[{"role": "user", "content": "How does the deep doc understanding work?"}] +) +print(response) +``` + +## Sample Usage - Agent Endpoint + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" # or your hosted URL + +response = completion( + model="ragflow/agent/my-agent-id/gpt-4o-mini", + messages=[{"role": "user", "content": "What are the key features?"}] +) +print(response) +``` + +## Sample Usage - With Parameters + +You can also pass `api_key` and `api_base` directly as parameters: + +```python +from litellm import completion + +response = completion( + model="ragflow/chat/my-chat-id/gpt-4o-mini", + messages=[{"role": "user", "content": "Hello!"}], + api_key="your-ragflow-api-key", + api_base="http://localhost:9380" +) +print(response) +``` + +## Sample Usage - Streaming + +```python +from litellm import completion +import os + +os.environ['RAGFLOW_API_KEY'] = "your-ragflow-api-key" +os.environ['RAGFLOW_API_BASE'] = "http://localhost:9380" + +response = completion( + model="ragflow/agent/my-agent-id/gpt-4o-mini", + messages=[{"role": "user", "content": "Explain RAGFlow"}], + stream=True +) + +for chunk in response: + print(chunk) +``` + +## Model Name Format + +The model name must follow one of these formats: + +### Chat Endpoint +``` +ragflow/chat/{chat_id}/{model_name} +``` + +Example: `ragflow/chat/my-chat-id/gpt-4o-mini` + +### Agent Endpoint +``` +ragflow/agent/{agent_id}/{model_name} +``` + +Example: `ragflow/agent/my-agent-id/gpt-4o-mini` + +Where: +- `{chat_id}` or `{agent_id}` is the ID of your chat or agent in RAGFlow +- `{model_name}` is the actual model name (e.g., `gpt-4o-mini`, `gpt-4o`, etc.) + +## Configuration Sources + +LiteLLM supports multiple ways to provide credentials, checked in this order: + +1. **Function parameters**: `api_key="..."`, `api_base="..."` +2. **litellm_params**: `litellm_params={"api_key": "...", "api_base": "..."}` +3. **Environment variables**: `RAGFLOW_API_KEY`, `RAGFLOW_API_BASE` +4. **Global litellm settings**: `litellm.api_key`, `litellm.api_base` + +## Usage - LiteLLM Proxy Server + +### 1. Save key in your environment + +```bash +export RAGFLOW_API_KEY="your-ragflow-api-key" +export RAGFLOW_API_BASE="http://localhost:9380" +``` + +### 2. Start the proxy + + + + +```yaml +model_list: + - model_name: ragflow-chat-gpt4 + litellm_params: + model: ragflow/chat/my-chat-id/gpt-4o-mini + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE + - model_name: ragflow-agent-gpt4 + litellm_params: + model: ragflow/agent/my-agent-id/gpt-4o-mini + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE +``` + + + + +```bash +$ litellm --config /path/to/config.yaml + +# Server running on http://0.0.0.0:4000 +``` + + + + +### 3. Test it + + + + +```bash +curl http://0.0.0.0:4000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer sk-1234" \ + -d '{ + "model": "ragflow-chat-gpt4", + "messages": [ + {"role": "user", "content": "How does RAGFlow work?"} + ] + }' +``` + + + + +```python +from openai import OpenAI + +client = OpenAI( + api_key="sk-1234", # Your LiteLLM proxy key + base_url="http://0.0.0.0:4000" +) + +response = client.chat.completions.create( + model="ragflow-chat-gpt4", + messages=[ + {"role": "user", "content": "How does RAGFlow work?"} + ] +) +print(response) +``` + + + + +## API Base URL Handling + +The `api_base` parameter can be provided with or without `/v1` suffix. LiteLLM will automatically handle it: + +- `http://localhost:9380` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` +- `http://localhost:9380/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` +- `http://localhost:9380/api/v1` → `http://localhost:9380/api/v1/chats_openai/{chat_id}/chat/completions` + +All three formats will work correctly. + +## Error Handling + +If you encounter errors: + +1. **Invalid model format**: Ensure your model name follows `ragflow/{chat|agent}/{id}/{model_name}` format +2. **Missing api_base**: Provide `api_base` via parameter, environment variable, or litellm_params +3. **Connection errors**: Verify your RAGFlow server is running and accessible at the provided `api_base` + +:::info + +For more information about passing provider-specific parameters, [go here](../completion/provider_specific_params.md) + +::: + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e467711b59d..55638512ef2 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -625,6 +625,7 @@ const sidebars = { "providers/petals", "providers/publicai", "providers/predibase", + "providers/ragflow", "providers/recraft", "providers/replicate", { diff --git a/litellm/__init__.py b/litellm/__init__.py index 007eff892c8..595a0132099 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1387,6 +1387,7 @@ from .llms.docker_model_runner.chat.transformation import DockerModelRunnerChatC from .llms.v0.chat.transformation import V0ChatConfig from .llms.oci.chat.transformation import OCIChatConfig from .llms.morph.chat.transformation import MorphChatConfig +from .llms.ragflow.chat.transformation import RAGFlowConfig from .llms.lambda_ai.chat.transformation import LambdaAIChatConfig from .llms.hyperbolic.chat.transformation import HyperbolicChatConfig from .llms.vercel_ai_gateway.chat.transformation import VercelAIGatewayConfig diff --git a/litellm/constants.py b/litellm/constants.py index e3de7368c8a..e252c86777c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -586,6 +586,7 @@ openai_compatible_providers: List = [ "cometapi", "clarifai", "docker_model_runner", + "ragflow", ] openai_text_completion_compatible_providers: List = ( [ # providers that support `/v1/completions` diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index b10011befcd..a90d16dba49 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -840,6 +840,16 @@ def _get_openai_compatible_provider_info( # noqa: PLR0915 ) = litellm.ClarifaiConfig()._get_openai_compatible_provider_info( api_base, api_key ) + elif custom_llm_provider == "ragflow": + full_model = f"ragflow/{model}" + ( + api_base, + dynamic_api_key, + _, + ) = litellm.RAGFlowConfig()._get_openai_compatible_provider_info( + full_model, api_base, api_key, "ragflow" + ) + model = full_model if api_base is not None and not isinstance(api_base, str): raise Exception("api base needs to be a string. api_base={}".format(api_base)) diff --git a/litellm/llms/ragflow/__init__.py b/litellm/llms/ragflow/__init__.py new file mode 100644 index 00000000000..17d12bed31c --- /dev/null +++ b/litellm/llms/ragflow/__init__.py @@ -0,0 +1,8 @@ +""" +RAGFlow provider for LiteLLM. + +RAGFlow provides OpenAI-compatible APIs with unique path structures: +- Chat endpoint: /api/v1/chats_openai/{chat_id}/chat/completions +- Agent endpoint: /api/v1/agents_openai/{agent_id}/chat/completions +""" + diff --git a/litellm/llms/ragflow/chat/__init__.py b/litellm/llms/ragflow/chat/__init__.py new file mode 100644 index 00000000000..0e0f47d07b6 --- /dev/null +++ b/litellm/llms/ragflow/chat/__init__.py @@ -0,0 +1,4 @@ +""" +RAGFlow chat completion configuration. +""" + diff --git a/litellm/llms/ragflow/chat/transformation.py b/litellm/llms/ragflow/chat/transformation.py new file mode 100644 index 00000000000..d33a1593be8 --- /dev/null +++ b/litellm/llms/ragflow/chat/transformation.py @@ -0,0 +1,264 @@ +""" +RAGFlow provider configuration for OpenAI-compatible API. + +RAGFlow provides OpenAI-compatible APIs with unique path structures: +- Chat endpoint: /api/v1/chats_openai/{chat_id}/chat/completions +- Agent endpoint: /api/v1/agents_openai/{agent_id}/chat/completions + +Model name format: +- Chat: ragflow/chat/{chat_id}/{model_name} +- Agent: ragflow/agent/{agent_id}/{model_name} +""" + +from typing import Any, List, Optional, Tuple + +import litellm +from litellm.llms.openai.openai import OpenAIConfig +from litellm.secret_managers.main import get_secret, get_secret_str +from litellm.types.llms.openai import AllMessageValues + + +class RAGFlowConfig(OpenAIConfig): + """ + Configuration for RAGFlow OpenAI-compatible API. + + Handles both chat and agent endpoints by parsing the model name format: + - ragflow/chat/{chat_id}/{model_name} for chat endpoints + - ragflow/agent/{agent_id}/{model_name} for agent endpoints + """ + + def _parse_ragflow_model(self, model: str) -> Tuple[str, str, str]: + """ + Parse RAGFlow model name format: ragflow/{endpoint_type}/{id}/{model_name} + + Args: + model: Model name in format ragflow/chat/{chat_id}/{model} or ragflow/agent/{agent_id}/{model} + + Returns: + Tuple of (endpoint_type, id, model_name) + + Raises: + ValueError: If model format is invalid + """ + parts = model.split("/") + if len(parts) < 4: + raise ValueError( + f"Invalid RAGFlow model format: {model}. " + f"Expected format: ragflow/chat/{{chat_id}}/{{model}} or ragflow/agent/{{agent_id}}/{{model}}" + ) + + if parts[0] != "ragflow": + raise ValueError( + f"Invalid RAGFlow model format: {model}. Must start with 'ragflow/'" + ) + + endpoint_type = parts[1] + if endpoint_type not in ["chat", "agent"]: + raise ValueError( + f"Invalid RAGFlow endpoint type: {endpoint_type}. Must be 'chat' or 'agent'" + ) + + entity_id = parts[2] + model_name = "/".join(parts[3:]) # Handle model names that might contain slashes + + return endpoint_type, entity_id, model_name + + def get_complete_url( + self, + api_base: Optional[str], + api_key: Optional[str], + model: str, + optional_params: dict, + litellm_params: dict, + stream: Optional[bool] = None, + ) -> str: + """ + Get the complete URL for the RAGFlow API call. + + Constructs URL based on endpoint type: + - Chat: /api/v1/chats_openai/{chat_id}/chat/completions + - Agent: /api/v1/agents_openai/{agent_id}/chat/completions + + Args: + api_base: Base API URL (e.g., http://ragflow-server:port or http://ragflow-server:port/v1) + api_key: API key (not used in URL construction) + model: Model name in format ragflow/{endpoint_type}/{id}/{model} + optional_params: Optional parameters + litellm_params: LiteLLM parameters (may contain api_base) + stream: Whether streaming is enabled + + Returns: + Complete URL for the API call + """ + # Get api_base from multiple sources: input param, litellm_params, environment, or global litellm setting + if litellm_params and hasattr(litellm_params, 'api_base') and litellm_params.api_base: + api_base = api_base or litellm_params.api_base + + api_base = ( + api_base + or litellm.api_base + or get_secret("RAGFLOW_API_BASE") + or get_secret_str("RAGFLOW_API_BASE") + ) + + if api_base is None: + raise ValueError("api_base is required for RAGFlow provider. Set it via api_base parameter, RAGFLOW_API_BASE environment variable, or litellm.api_base") + + # Parse model name to extract endpoint type and ID + endpoint_type, entity_id, _ = self._parse_ragflow_model(model) + + # Remove trailing slash from api_base if present + api_base = api_base.rstrip("/") + + # Strip /v1 or /api/v1 from api_base if present, since we'll add the full path + # Check /api/v1 first because /api/v1 ends with /v1 + if api_base.endswith("/api/v1"): + api_base = api_base[:-7] # Remove /api/v1 + elif api_base.endswith("/v1"): + api_base = api_base[:-3] # Remove /v1 + + # Construct the RAGFlow-specific path + if endpoint_type == "chat": + path = f"/api/v1/chats_openai/{entity_id}/chat/completions" + else: # agent + path = f"/api/v1/agents_openai/{entity_id}/chat/completions" + + # Ensure path starts with / + if not path.startswith("/"): + path = "/" + path + + return f"{api_base}{path}" + + def _get_openai_compatible_provider_info( + self, + model: str, + api_base: Optional[str], + api_key: Optional[str], + custom_llm_provider: str, + ) -> Tuple[Optional[str], Optional[str], str]: + """ + Get OpenAI-compatible provider information for RAGFlow. + + Args: + model: Model name (will be parsed to extract actual model name) + api_base: Base API URL (from input params) + api_key: API key (from input params) + custom_llm_provider: Custom LLM provider name + + Returns: + Tuple of (api_base, api_key, custom_llm_provider) + """ + # Parse model to extract the actual model name + # The model name will be stored in litellm_params for use in requests + _, _, actual_model = self._parse_ragflow_model(model) + + # Get api_base from multiple sources: input param, environment, or global litellm setting + dynamic_api_base = ( + api_base + or litellm.api_base + or get_secret("RAGFLOW_API_BASE") + or get_secret_str("RAGFLOW_API_BASE") + ) + + # Get api_key from multiple sources: input param, environment, or global litellm setting + dynamic_api_key = ( + api_key + or litellm.api_key + or get_secret_str("RAGFLOW_API_KEY") + ) + + return dynamic_api_base, dynamic_api_key, custom_llm_provider + + def validate_environment( + self, + headers: dict, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + api_key: Optional[str] = None, + api_base: Optional[str] = None, + ) -> dict: + """ + Validate environment and set up headers for RAGFlow API. + + Args: + headers: Request headers + model: Model name + messages: Chat messages + optional_params: Optional parameters + litellm_params: LiteLLM parameters (may contain api_key) + api_key: API key (from input params) + api_base: Base API URL + + Returns: + Updated headers dictionary + """ + # Use api_key from litellm_params if available, otherwise fall back to other sources + if litellm_params and hasattr(litellm_params, 'api_key') and litellm_params.api_key: + api_key = api_key or litellm_params.api_key + + # Get api_key from multiple sources: input param, litellm_params, environment, or global litellm setting + api_key = ( + api_key + or litellm.api_key + or get_secret_str("RAGFLOW_API_KEY") + ) + + if api_key is not None: + headers["Authorization"] = f"Bearer {api_key}" + + # Ensure Content-Type is set to application/json + if "content-type" not in headers and "Content-Type" not in headers: + headers["Content-Type"] = "application/json" + + # Parse model to extract actual model name and store it + # The actual model name should be used in the request body + try: + _, _, actual_model = self._parse_ragflow_model(model) + # Store the actual model name in litellm_params for use in transform_request + litellm_params["_ragflow_actual_model"] = actual_model + except ValueError: + # If parsing fails, use the original model name + pass + + return headers + + def transform_request( + self, + model: str, + messages: List[AllMessageValues], + optional_params: dict, + litellm_params: dict, + headers: dict, + ) -> dict: + """ + Transform request for RAGFlow API. + + Uses the actual model name extracted from the RAGFlow model format. + + Args: + model: Model name in RAGFlow format + messages: Chat messages + optional_params: Optional parameters + litellm_params: LiteLLM parameters (may contain _ragflow_actual_model) + headers: Request headers + + Returns: + Transformed request dictionary + """ + # Get the actual model name from litellm_params if available + actual_model = litellm_params.get("_ragflow_actual_model") + if actual_model is None: + # Fallback: try to parse the model name + try: + _, _, actual_model = self._parse_ragflow_model(model) + except ValueError: + # If parsing fails, use the original model name + actual_model = model + + # Use parent's transform_request with the actual model name + return super().transform_request( + actual_model, messages, optional_params, litellm_params, headers + ) + diff --git a/litellm/main.py b/litellm/main.py index a09a9453017..fba2fc9b9a6 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1989,6 +1989,36 @@ def completion( # type: ignore # noqa: PLR0915 ) raise e + elif custom_llm_provider == "ragflow": + ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths + try: + response = base_llm_http_handler.completion( + model=model, + messages=messages, + headers=headers, + model_response=model_response, + api_key=api_key, + api_base=api_base, + acompletion=acompletion, + logging_obj=logging, + optional_params=optional_params, + litellm_params=litellm_params, + shared_session=shared_session, + timeout=timeout, + client=client, + custom_llm_provider=custom_llm_provider, + encoding=encoding, + stream=stream, + provider_config=provider_config, + ) + except Exception as e: + logging.post_call( + input=messages, + api_key=api_key, + original_response=str(e), + additional_args={"headers": headers}, + ) + raise e elif custom_llm_provider == "xai": ## COMPLETION CALL try: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 9fdc1704f41..f28e9b1290f 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -6717,6 +6717,33 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "claude-opus-4-5": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_creation_input_token_cost_above_1hr": 1e-05, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "anthropic", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "claude-sonnet-4-20250514": { "deprecation_date": "2026-05-14", "cache_creation_input_token_cost": 3.75e-06, @@ -7824,26 +7851,298 @@ "source": "https://www.databricks.com/product/pricing/foundation-model-serving" }, "databricks/databricks-claude-3-7-sonnet": { - "input_cost_per_token": 2.5e-06, - "input_dbu_cost_per_token": 3.571e-05, + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, "litellm_provider": "databricks", "max_input_tokens": 200000, "max_output_tokens": 128000, "max_tokens": 200000, "metadata": { - "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Claude 3.7 conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.7857e-05, - "output_db_cost_per_token": 0.000214286, - "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", "supports_assistant_prefill": true, "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true }, + "databricks/databricks-claude-haiku-4-5": { + "input_cost_per_token": 1.00002e-06, + "input_dbu_cost_per_token": 1.4286e-05, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 5.00003e-06, + "output_dbu_cost_per_token": 7.1429e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-opus-4": { + "input_cost_per_token": 1.5000020000000002e-05, + "input_dbu_cost_per_token": 0.000214286, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 7.500003000000001e-05, + "output_dbu_cost_per_token": 0.001071429, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-opus-4-1": { + "input_cost_per_token": 1.5000020000000002e-05, + "input_dbu_cost_per_token": 0.000214286, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 32000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 7.500003000000001e-05, + "output_dbu_cost_per_token": 0.001071429, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-opus-4-5": { + "input_cost_per_token": 5.00003e-06, + "input_dbu_cost_per_token": 7.1429e-05, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 2.5000010000000002e-05, + "output_dbu_cost_per_token": 0.000357143, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-sonnet-4": { + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-sonnet-4-1": { + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-claude-sonnet-4-5": { + "input_cost_per_token": 2.9999900000000002e-06, + "input_dbu_cost_per_token": 4.2857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-2-5-flash": { + "input_cost_per_token": 3.0001999999999996e-07, + "input_dbu_cost_per_token": 4.285999999999999e-06, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65535, + "max_tokens": 1048576, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 2.49998e-06, + "output_dbu_cost_per_token": 3.5714e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemini-2-5-pro": { + "input_cost_per_token": 1.24999e-06, + "input_dbu_cost_per_token": 1.7857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 1048576, + "max_output_tokens": 65536, + "max_tokens": 1048576, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 9.999990000000002e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving", + "supports_function_calling": true, + "supports_tool_choice": true + }, + "databricks/databricks-gemma-3-12b": { + "input_cost_per_token": 1.5000999999999998e-07, + "input_dbu_cost_per_token": 2.1429999999999996e-06, + "litellm_provider": "databricks", + "max_input_tokens": 128000, + "max_output_tokens": 32000, + "max_tokens": 128000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 5.0001e-07, + "output_dbu_cost_per_token": 7.143e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, + "databricks/databricks-gpt-5": { + "input_cost_per_token": 1.24999e-06, + "input_dbu_cost_per_token": 1.7857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 400000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 9.999990000000002e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-1": { + "input_cost_per_token": 1.24999e-06, + "input_dbu_cost_per_token": 1.7857e-05, + "litellm_provider": "databricks", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 400000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 9.999990000000002e-06, + "output_dbu_cost_per_token": 0.000142857, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-mini": { + "input_cost_per_token": 2.4997000000000006e-07, + "input_dbu_cost_per_token": 3.571e-06, + "litellm_provider": "databricks", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 400000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 1.9999700000000004e-06, + "output_dbu_cost_per_token": 2.8571e-05, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-5-nano": { + "input_cost_per_token": 4.998e-08, + "input_dbu_cost_per_token": 7.14e-07, + "litellm_provider": "databricks", + "max_input_tokens": 400000, + "max_output_tokens": 128000, + "max_tokens": 400000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 3.9998000000000007e-07, + "output_dbu_cost_per_token": 5.714000000000001e-06, + "source": "https://www.databricks.com/product/pricing/proprietary-foundation-model-serving" + }, + "databricks/databricks-gpt-oss-120b": { + "input_cost_per_token": 1.5000999999999998e-07, + "input_dbu_cost_per_token": 2.1429999999999996e-06, + "litellm_provider": "databricks", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 5.9997e-07, + "output_dbu_cost_per_token": 8.571e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, + "databricks/databricks-gpt-oss-20b": { + "input_cost_per_token": 7e-08, + "input_dbu_cost_per_token": 1e-06, + "litellm_provider": "databricks", + "max_input_tokens": 131072, + "max_output_tokens": 131072, + "max_tokens": 131072, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 3.0001999999999996e-07, + "output_dbu_cost_per_token": 4.285999999999999e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "databricks/databricks-gte-large-en": { - "input_cost_per_token": 1.2999e-07, + "input_cost_per_token": 1.2999000000000001e-07, "input_dbu_cost_per_token": 1.857e-06, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -7868,14 +8167,14 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.5e-06, + "output_cost_per_token": 1.5000300000000002e-06, "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, "databricks/databricks-llama-4-maverick": { - "input_cost_per_token": 5e-06, - "input_dbu_cost_per_token": 7.143e-05, + "input_cost_per_token": 5.0001e-07, + "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -7884,13 +8183,13 @@ "notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)." }, "mode": "chat", - "output_cost_per_token": 1.5e-05, - "output_dbu_cost_per_token": 0.00021429, + "output_cost_per_token": 1.5000300000000002e-06, + "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, "databricks/databricks-meta-llama-3-1-405b-instruct": { - "input_cost_per_token": 5e-06, + "input_cost_per_token": 5.00003e-06, "input_dbu_cost_per_token": 7.1429e-05, "litellm_provider": "databricks", "max_input_tokens": 128000, @@ -7900,14 +8199,29 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 1.500002e-05, - "output_db_cost_per_token": 0.000214286, + "output_cost_per_token": 1.5000020000000002e-05, + "output_dbu_cost_per_token": 0.000214286, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, + "databricks/databricks-meta-llama-3-1-8b-instruct": { + "input_cost_per_token": 1.5000999999999998e-07, + "input_dbu_cost_per_token": 2.1429999999999996e-06, + "litellm_provider": "databricks", + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "max_tokens": 200000, + "metadata": { + "notes": "Input/output cost per token is dbu cost * $0.070. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." + }, + "mode": "chat", + "output_cost_per_token": 4.5003000000000007e-07, + "output_dbu_cost_per_token": 6.429000000000001e-06, + "source": "https://www.databricks.com/product/pricing/foundation-model-serving" + }, "databricks/databricks-meta-llama-3-3-70b-instruct": { - "input_cost_per_token": 1.00002e-06, - "input_dbu_cost_per_token": 1.4286e-05, + "input_cost_per_token": 5.0001e-07, + "input_dbu_cost_per_token": 7.143e-06, "litellm_provider": "databricks", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -7916,8 +8230,8 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 2.99999e-06, - "output_dbu_cost_per_token": 4.2857e-05, + "output_cost_per_token": 1.5000300000000002e-06, + "output_dbu_cost_per_token": 2.1429e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, @@ -7932,7 +8246,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 2.99999e-06, + "output_cost_per_token": 2.9999900000000002e-06, "output_dbu_cost_per_token": 4.2857e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true @@ -7948,13 +8262,13 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.9902e-07, + "output_cost_per_token": 1.00002e-06, "output_dbu_cost_per_token": 1.4286e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true }, "databricks/databricks-mpt-30b-instruct": { - "input_cost_per_token": 9.9902e-07, + "input_cost_per_token": 1.00002e-06, "input_dbu_cost_per_token": 1.4286e-05, "litellm_provider": "databricks", "max_input_tokens": 8192, @@ -7964,7 +8278,7 @@ "notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation." }, "mode": "chat", - "output_cost_per_token": 9.9902e-07, + "output_cost_per_token": 1.00002e-06, "output_dbu_cost_per_token": 1.4286e-05, "source": "https://www.databricks.com/product/pricing/foundation-model-serving", "supports_tool_choice": true diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 58267fdfea9..942192e2edf 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2618,6 +2618,7 @@ class LlmProviders(str, Enum): DATABRICKS = "databricks" EMPOWER = "empower" GITHUB = "github" + RAGFLOW = "ragflow" COMPACTIFAI = "compactifai" DOCKER_MODEL_RUNNER = "docker_model_runner" CUSTOM = "custom" diff --git a/litellm/utils.py b/litellm/utils.py index 6c50afc5f49..eb671ce2b99 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7110,6 +7110,8 @@ class ProviderConfigManager: return litellm.CompactifAIChatConfig() elif litellm.LlmProviders.GITHUB_COPILOT == provider: return litellm.GithubCopilotConfig() + elif litellm.LlmProviders.RAGFLOW == provider: + return litellm.RAGFlowConfig() elif ( litellm.LlmProviders.CUSTOM == provider or litellm.LlmProviders.CUSTOM_OPENAI == provider diff --git a/tests/test_litellm/llms/ragflow/chat/__init__.py b/tests/test_litellm/llms/ragflow/chat/__init__.py new file mode 100644 index 00000000000..4e074b84150 --- /dev/null +++ b/tests/test_litellm/llms/ragflow/chat/__init__.py @@ -0,0 +1,4 @@ +""" +RAGFlow chat transformation tests. +""" + diff --git a/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py b/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py new file mode 100644 index 00000000000..90f2504f94c --- /dev/null +++ b/tests/test_litellm/llms/ragflow/chat/test_ragflow_chat_transformation.py @@ -0,0 +1,376 @@ +""" +Test file for RAGFlow chat transformation functionality. + +Tests the model name parsing, URL construction, and request transformation +for RAGFlow's OpenAI-compatible API with custom path structures. +""" + +import os +import sys +from unittest.mock import Mock, patch + +import pytest + +# Add the project root to Python path +sys.path.insert(0, os.path.abspath("../../../../..")) + +import litellm +from litellm.llms.ragflow.chat.transformation import RAGFlowConfig +from litellm.types.llms.openai import AllMessageValues + + +class TestRAGFlowChatTransformation: + """Test suite for RAGFlow chat transformation functionality.""" + + def test_parse_ragflow_model_chat(self): + """Test parsing of chat model format.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + endpoint_type, entity_id, model_name = config._parse_ragflow_model(model) + + assert endpoint_type == "chat" + assert entity_id == "my-chat-id" + assert model_name == "gpt-4o-mini" + + def test_parse_ragflow_model_agent(self): + """Test parsing of agent model format.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + endpoint_type, entity_id, model_name = config._parse_ragflow_model(model) + + assert endpoint_type == "agent" + assert entity_id == "my-agent-id" + assert model_name == "gpt-4o-mini" + + def test_parse_ragflow_model_with_slashes_in_model_name(self): + """Test parsing when model name contains slashes.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/openai/gpt-4o-mini" + endpoint_type, entity_id, model_name = config._parse_ragflow_model(model) + + assert endpoint_type == "chat" + assert entity_id == "my-chat-id" + assert model_name == "openai/gpt-4o-mini" + + def test_parse_ragflow_model_invalid_format(self): + """Test parsing with invalid model format.""" + config = RAGFlowConfig() + + with pytest.raises(ValueError, match="Invalid RAGFlow model format"): + config._parse_ragflow_model("ragflow/chat/model-name") + + with pytest.raises(ValueError, match="Invalid RAGFlow model format"): + config._parse_ragflow_model("invalid/chat/id/model") + + with pytest.raises(ValueError, match="Must start with 'ragflow/'"): + config._parse_ragflow_model("not-ragflow/chat/id/model") + + def test_parse_ragflow_model_invalid_endpoint_type(self): + """Test parsing with invalid endpoint type.""" + config = RAGFlowConfig() + + with pytest.raises(ValueError, match="Invalid RAGFlow endpoint type"): + config._parse_ragflow_model("ragflow/invalid/my-id/model") + + def test_get_complete_url_chat(self): + """Test URL construction for chat endpoint.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + api_base = "http://localhost:9380" + + url = config.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "http://localhost:9380/api/v1/chats_openai/my-chat-id/chat/completions" + + def test_get_complete_url_agent(self): + """Test URL construction for agent endpoint.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + api_base = "http://localhost:9380" + + url = config.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "http://localhost:9380/api/v1/agents_openai/my-agent-id/chat/completions" + + def test_get_complete_url_strips_v1(self): + """Test URL construction when api_base ends with /v1.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + api_base = "http://localhost:9380/v1" + + url = config.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "http://localhost:9380/api/v1/chats_openai/my-chat-id/chat/completions" + + def test_get_complete_url_strips_api_v1(self): + """Test URL construction when api_base ends with /api/v1.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + api_base = "http://localhost:9380/api/v1" + + url = config.get_complete_url( + api_base=api_base, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "http://localhost:9380/api/v1/agents_openai/my-agent-id/chat/completions" + + def test_get_complete_url_from_litellm_params(self): + """Test URL construction with api_base from litellm_params.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + # Create a simple dict-like object for litellm_params + class LiteLLMParams: + def __init__(self): + self.api_base = "http://ragflow-server:9380" + + litellm_params = LiteLLMParams() + + url = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + optional_params={}, + litellm_params=litellm_params, + stream=False, + ) + + assert url == "http://ragflow-server:9380/api/v1/chats_openai/my-chat-id/chat/completions" + + def test_get_complete_url_missing_api_base(self): + """Test URL construction when api_base is missing.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + + with pytest.raises(ValueError, match="api_base is required"): + config.get_complete_url( + api_base=None, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + @patch.dict(os.environ, {"RAGFLOW_API_BASE": "http://env-ragflow:9380"}) + def test_get_complete_url_from_environment(self): + """Test URL construction with api_base from environment variable.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + + url = config.get_complete_url( + api_base=None, + api_key=None, + model=model, + optional_params={}, + litellm_params={}, + stream=False, + ) + + assert url == "http://env-ragflow:9380/api/v1/agents_openai/my-agent-id/chat/completions" + + def test_validate_environment_sets_headers(self): + """Test that validate_environment sets proper headers.""" + config = RAGFlowConfig() + + headers = {} + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + api_key = "test-api-key" + + result_headers = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params={}, + litellm_params={}, + api_key=api_key, + api_base="http://localhost:9380", + ) + + assert result_headers["Authorization"] == "Bearer test-api-key" + assert result_headers["Content-Type"] == "application/json" + + def test_validate_environment_stores_actual_model(self): + """Test that validate_environment stores actual model name.""" + config = RAGFlowConfig() + + headers = {} + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + litellm_params = {} + + config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params={}, + litellm_params=litellm_params, + api_key="test-key", + api_base="http://localhost:9380", + ) + + assert litellm_params["_ragflow_actual_model"] == "gpt-4o-mini" + + @patch.dict(os.environ, {"RAGFLOW_API_KEY": "env-api-key"}) + def test_validate_environment_from_environment(self): + """Test that validate_environment gets api_key from environment.""" + config = RAGFlowConfig() + + headers = {} + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + + result_headers = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params={}, + litellm_params={}, + api_key=None, + api_base="http://localhost:9380", + ) + + assert result_headers["Authorization"] == "Bearer env-api-key" + + def test_validate_environment_from_litellm_params(self): + """Test that validate_environment gets api_key from litellm_params.""" + config = RAGFlowConfig() + + headers = {} + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + # Create a simple object for litellm_params with api_key attribute + class LiteLLMParams: + def __init__(self): + self.api_key = "litellm-params-key" + def __setitem__(self, key, value): + setattr(self, key, value) + + litellm_params = LiteLLMParams() + + result_headers = config.validate_environment( + headers=headers, + model=model, + messages=messages, + optional_params={}, + litellm_params=litellm_params, + api_key=None, + api_base="http://localhost:9380", + ) + + assert result_headers["Authorization"] == "Bearer litellm-params-key" + + def test_transform_request_uses_actual_model(self): + """Test that transform_request uses the actual model name.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + litellm_params = {"_ragflow_actual_model": "gpt-4o-mini"} + + # Test the actual behavior by checking the model in the result + result = config.transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + # The result should contain the actual model name, not the full ragflow path + assert result["model"] == "gpt-4o-mini" + assert result["messages"] == messages + + def test_transform_request_fallback_parsing(self): + """Test that transform_request falls back to parsing if _ragflow_actual_model is missing.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + messages = [{"role": "user", "content": "Hello"}] + litellm_params = {} # Missing _ragflow_actual_model + + result = config.transform_request( + model=model, + messages=messages, + optional_params={}, + litellm_params=litellm_params, + headers={}, + ) + + # Should parse and use the actual model name + assert result["model"] == "gpt-4o-mini" + assert result["messages"] == messages + + def test_get_openai_compatible_provider_info(self): + """Test _get_openai_compatible_provider_info returns correct values.""" + config = RAGFlowConfig() + + model = "ragflow/chat/my-chat-id/gpt-4o-mini" + api_base = "http://localhost:9380" + api_key = "test-key" + + result_api_base, result_api_key, result_provider = config._get_openai_compatible_provider_info( + model=model, + api_base=api_base, + api_key=api_key, + custom_llm_provider="ragflow", + ) + + assert result_api_base == api_base + assert result_api_key == api_key + assert result_provider == "ragflow" + + @patch.dict(os.environ, {"RAGFLOW_API_BASE": "http://env-base:9380", "RAGFLOW_API_KEY": "env-key"}) + def test_get_openai_compatible_provider_info_from_env(self): + """Test _get_openai_compatible_provider_info gets values from environment.""" + config = RAGFlowConfig() + + model = "ragflow/agent/my-agent-id/gpt-4o-mini" + + result_api_base, result_api_key, result_provider = config._get_openai_compatible_provider_info( + model=model, + api_base=None, + api_key=None, + custom_llm_provider="ragflow", + ) + + assert result_api_base == "http://env-base:9380" + assert result_api_key == "env-key" + assert result_provider == "ragflow" + From 427074ac6e80ffe3b30054dd10c2a7427cfbf96f Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Tue, 2 Dec 2025 17:27:50 -0800 Subject: [PATCH 02/24] Fix: Datadog callback regression when ddtrace is installed (#17393) * fix DD agent host logging * docs fix * test_datadog_agent_configuration * test_datadog_ignores_ddtrace_agent_host --- docs/my-website/docs/observability/datadog.md | 18 +++---- litellm/integrations/datadog/datadog.py | 13 ++--- tests/logging_callback_tests/test_datadog.py | 47 +++++++++++++++++-- 3 files changed, 60 insertions(+), 18 deletions(-) diff --git a/docs/my-website/docs/observability/datadog.md b/docs/my-website/docs/observability/datadog.md index 5cb5ab3af2d..b2901650ea6 100644 --- a/docs/my-website/docs/observability/datadog.md +++ b/docs/my-website/docs/observability/datadog.md @@ -71,17 +71,19 @@ DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source. use to different Send logs through a local DataDog agent (useful for containerized environments): ```shell -DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent -DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) -DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) -DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source +LITELLM_DD_AGENT_HOST="localhost" # hostname or IP of DataDog agent +LITELLM_DD_AGENT_PORT="10518" # [OPTIONAL] port of DataDog agent (default: 10518) +DD_API_KEY="5f2d0f310***********" # [OPTIONAL] your datadog API Key (agent handles auth) +DD_SOURCE="litellm_dev" # [OPTIONAL] your datadog source ``` -When `DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: +When `LITELLM_DD_AGENT_HOST` is set, logs are sent to the agent instead of directly to DataDog API. This is useful for: - Centralized log shipping in containerized environments - Reducing direct API calls from multiple services - Leveraging agent-side processing and filtering +**Note:** We use `LITELLM_DD_AGENT_HOST` instead of `DD_AGENT_HOST` to avoid conflicts with `ddtrace` which automatically sets `DD_AGENT_HOST` for APM tracing. + **Step 3**: Start the proxy, make a test request Start proxy @@ -191,8 +193,8 @@ LiteLLM supports customizing the following Datadog environment variables |---------------------|-------------|---------------|----------| | `DD_API_KEY` | Your Datadog API key for authentication (required for direct API, optional for agent) | None | Conditional* | | `DD_SITE` | Your Datadog site (e.g., "us5.datadoghq.com") (required for direct API) | None | Conditional* | -| `DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | -| `DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | +| `LITELLM_DD_AGENT_HOST` | Hostname or IP of DataDog agent (e.g., "localhost"). When set, logs are sent to agent instead of direct API | None | ❌ No | +| `LITELLM_DD_AGENT_PORT` | Port of DataDog agent for log intake | "10518" | ❌ No | | `DD_ENV` | Environment tag for your logs (e.g., "production", "staging") | "unknown" | ❌ No | | `DD_SERVICE` | Service name for your logs | "litellm-server" | ❌ No | | `DD_SOURCE` | Source name for your logs | "litellm" | ❌ No | @@ -201,5 +203,5 @@ LiteLLM supports customizing the following Datadog environment variables | `POD_NAME` | Pod name tag (useful for Kubernetes deployments) | "unknown" | ❌ No | \* **Required when using Direct API** (default): `DD_API_KEY` and `DD_SITE` are required -\* **Optional when using DataDog Agent**: Set `DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required +\* **Optional when using DataDog Agent**: Set `LITELLM_DD_AGENT_HOST` to use agent mode; `DD_API_KEY` and `DD_SITE` are not required diff --git a/litellm/integrations/datadog/datadog.py b/litellm/integrations/datadog/datadog.py index 46e1a2c201f..21e1d562224 100644 --- a/litellm/integrations/datadog/datadog.py +++ b/litellm/integrations/datadog/datadog.py @@ -65,11 +65,11 @@ class DataDogLogger( `DD_SITE` - your datadog site, example = `"us5.datadoghq.com"` Optional environment variables (DataDog Agent): - `DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` - `DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) + `LITELLM_DD_AGENT_HOST` - hostname or IP of DataDog agent, example = `"localhost"` + `LITELLM_DD_AGENT_PORT` - port of DataDog agent (default: 10518 for logs) - Note: If DD_AGENT_HOST is set, logs will be sent to the agent instead of directly to DataDog API. - In this case, DD_API_KEY and DD_SITE are not required (agent handles authentication). + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts + with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ try: verbose_logger.debug("Datadog: in init datadog logger") @@ -85,7 +85,8 @@ class DataDogLogger( ) # Configure DataDog endpoint (Agent or Direct API) - dd_agent_host = os.getenv("DD_AGENT_HOST") + # Use LITELLM_DD_AGENT_HOST to avoid conflicts with ddtrace's DD_AGENT_HOST + dd_agent_host = os.getenv("LITELLM_DD_AGENT_HOST") if dd_agent_host: self._configure_dd_agent(dd_agent_host=dd_agent_host) else: @@ -127,7 +128,7 @@ class DataDogLogger( Args: dd_agent_host: Hostname or IP of DataDog agent """ - dd_agent_port = os.getenv("DD_AGENT_PORT", "10518") # default port for logs + dd_agent_port = os.getenv("LITELLM_DD_AGENT_PORT", "10518") # default port for logs self.intake_url = f"http://{dd_agent_host}:{dd_agent_port}/api/v2/logs" self.DD_API_KEY = os.getenv("DD_API_KEY") # Optional when using agent verbose_logger.debug(f"Datadog: Using DD Agent at {self.intake_url}") diff --git a/tests/logging_callback_tests/test_datadog.py b/tests/logging_callback_tests/test_datadog.py index 13125aa4952..c877f34ac03 100644 --- a/tests/logging_callback_tests/test_datadog.py +++ b/tests/logging_callback_tests/test_datadog.py @@ -633,11 +633,14 @@ async def test_datadog_message_redaction(): def test_datadog_agent_configuration(): """ - Test that DataDog logger correctly configures agent endpoint when DD_AGENT_HOST is set + Test that DataDog logger correctly configures agent endpoint when LITELLM_DD_AGENT_HOST is set. + + Note: We use LITELLM_DD_AGENT_HOST instead of DD_AGENT_HOST to avoid conflicts + with ddtrace which automatically sets DD_AGENT_HOST for APM tracing. """ test_env = { - "DD_AGENT_HOST": "localhost", - "DD_AGENT_PORT": "10518", + "LITELLM_DD_AGENT_HOST": "localhost", + "LITELLM_DD_AGENT_PORT": "10518", } # Remove DD_SITE and DD_API_KEY to verify they're not required for agent mode @@ -654,4 +657,40 @@ def test_datadog_agent_configuration(): assert dd_logger.intake_url == "http://localhost:10518/api/v2/logs", f"Expected agent URL, got {dd_logger.intake_url}" # Verify DD_API_KEY is optional (can be None) - assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) \ No newline at end of file + assert dd_logger.DD_API_KEY is None or isinstance(dd_logger.DD_API_KEY, str) + + +def test_datadog_ignores_ddtrace_agent_host(): + """ + Regression test: Ensure DD_AGENT_HOST set by ddtrace doesn't interfere with LiteLLM logging. + + When users have ddtrace installed for APM tracing, it automatically sets DD_AGENT_HOST. + LiteLLM should ignore DD_AGENT_HOST and only use LITELLM_DD_AGENT_HOST for agent mode. + + This prevents the 404 error when ddtrace's DD_AGENT_HOST points to an APM endpoint + that doesn't support /api/v2/logs. + + Regression test for: https://github.com/BerriAI/litellm/issues/16379 + """ + test_env = { + # User's explicit config for LiteLLM logging (direct API) + "DD_API_KEY": "fake-api-key", + "DD_SITE": "us5.datadoghq.com", + # ddtrace automatically sets these for APM tracing + "DD_AGENT_HOST": "10.176.100.40", + "DD_AGENT_PORT": "8126", + } + + with patch.dict(os.environ, test_env, clear=False): + with patch("asyncio.create_task"): + dd_logger = DataDogLogger() + + # Verify direct API endpoint is used (DD_AGENT_HOST should be ignored) + expected_url = "https://http-intake.logs.us5.datadoghq.com/api/v2/logs" + assert dd_logger.intake_url == expected_url, ( + f"Expected direct API URL '{expected_url}', got '{dd_logger.intake_url}'. " + "DD_AGENT_HOST (set by ddtrace) should be ignored - only LITELLM_DD_AGENT_HOST should trigger agent mode." + ) + + # Verify API key is set correctly + assert dd_logger.DD_API_KEY == "fake-api-key" \ No newline at end of file From 209e9e05aa3a5543639df154fe54974e0b708e9a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 10:13:59 +0530 Subject: [PATCH 03/24] Fix gemini 3 last chunk thinking block --- .../vertex_and_google_ai_studio_gemini.py | 5 +++++ .../test_gemini_reasoning_content.py | 20 +++++++++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 tests/local_testing/test_gemini_reasoning_content.py diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 5fef8c1ec49..cfc13f5601d 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1091,6 +1091,11 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): if "thoughtSignature" in part: part_copy = part.copy() part_copy.pop("thoughtSignature") + + text_content = part_copy.get("text") + if isinstance(text_content, str) and text_content.strip() == "": + continue + thinking_blocks.append( ChatCompletionThinkingBlock( type="thinking", diff --git a/tests/local_testing/test_gemini_reasoning_content.py b/tests/local_testing/test_gemini_reasoning_content.py new file mode 100644 index 00000000000..7e516ae8439 --- /dev/null +++ b/tests/local_testing/test_gemini_reasoning_content.py @@ -0,0 +1,20 @@ +import json +from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexGeminiConfig + + +def test_empty_part_does_not_create_thinking_block(): + parts = [{"text": "", "thoughtSignature": "sig-1"}] + config = VertexGeminiConfig() + thinking_blocks = config._extract_thinking_blocks_from_parts(parts) + assert thinking_blocks == [] + + +def test_non_empty_part_creates_thinking_block(): + parts = [{"text": "Some thinking", "thoughtSignature": "sig-2"}] + config = VertexGeminiConfig() + thinking_blocks = config._extract_thinking_blocks_from_parts(parts) + assert len(thinking_blocks) == 1 + block = thinking_blocks[0] + # thinking should be valid JSON containing the text + parsed = json.loads(block["thinking"]) if isinstance(block["thinking"], str) else None + assert parsed is not None and parsed.get("text") == "Some thinking" From 40c203e32b5991cefa14d9d7befaac6385ee0538 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 11:04:53 +0530 Subject: [PATCH 04/24] Make thought sign in tool call id as a beta feat --- .../vertex_and_google_ai_studio_gemini.py | 12 +- .../test_thought_signature_in_tool_call_id.py | 277 ++++++++++++------ 2 files changed, 188 insertions(+), 101 deletions(-) diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index 5fef8c1ec49..1c2efe1eaa7 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -1205,14 +1205,16 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): } # Embed thought signature in ID for OpenAI client compatibility if thought_signature: - _tool_response_chunk[ - "id" - ] = _encode_tool_call_id_with_signature( - _tool_response_chunk["id"] or "", thought_signature - ) _tool_response_chunk["provider_specific_fields"] = { # type: ignore "thought_signature": thought_signature } + # Only embed in ID if preview features are enabled + if litellm.enable_preview_features: + _tool_response_chunk[ + "id" + ] = _encode_tool_call_id_with_signature( + _tool_response_chunk["id"] or "", thought_signature + ) _tools.append(_tool_response_chunk) cumulative_tool_call_idx += 1 if len(_tools) == 0: diff --git a/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py b/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py index 68c0f3bdbfc..46bb8930a7a 100644 --- a/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py +++ b/tests/test_litellm/llms/vertex_ai/gemini/test_thought_signature_in_tool_call_id.py @@ -4,9 +4,13 @@ Tests for embedding thought signatures in tool call IDs for OpenAI client compat When using OpenAI clients (instead of LiteLLM SDK), provider_specific_fields are not preserved. This test suite validates that thought signatures can be embedded in tool call IDs and extracted when converting back to Gemini format. + +Note: Embedding signatures in tool call IDs is a beta feature that requires +enable_preview_features=True to be enabled. """ import pytest +import litellm from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import ( VertexGeminiConfig, ) @@ -62,36 +66,57 @@ def test_encode_tool_call_id_without_signature(): assert decoded_signature is None -def test_tool_call_id_includes_signature_in_response(): - """Test that tool call IDs in responses include embedded thought signatures""" +@pytest.mark.parametrize("enable_preview_features", [True, False]) +def test_tool_call_id_includes_signature_in_response(enable_preview_features): + """Test that tool call IDs in responses include embedded thought signatures only when preview features are enabled""" test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - parts_with_signature = [ - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "Paris"}, - }, - thoughtSignature=test_signature, + # Save original state + original_flag = litellm.enable_preview_features + litellm.enable_preview_features = enable_preview_features + + try: + parts_with_signature = [ + HttpxPartType( + functionCall={ + "name": "get_current_temperature", + "args": {"location": "Paris"}, + }, + thoughtSignature=test_signature, + ) + ] + + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=parts_with_signature, + cumulative_tool_call_idx=0, + is_function_call=False, ) - ] - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=parts_with_signature, - cumulative_tool_call_idx=0, - is_function_call=False, - ) + # Verify tool call exists + assert tools is not None + assert len(tools) == 1 + tool_call_id = tools[0]["id"] + + # Verify signature is always in provider_specific_fields + assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == test_signature - # Verify tool call ID includes thought signature - assert tools is not None - assert len(tools) == 1 - tool_call_id = tools[0]["id"] - assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id - - # Verify we can decode it using the factory function - tool_obj = {"id": tool_call_id, "type": "function"} - decoded_sig = _get_thought_signature_from_tool(tool_obj) - assert decoded_sig == test_signature + if enable_preview_features: + # When preview features enabled, signature should be embedded in ID + assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id + # Verify we can decode it using the factory function + tool_obj = {"id": tool_call_id, "type": "function"} + decoded_sig = _get_thought_signature_from_tool(tool_obj) + assert decoded_sig == test_signature + else: + # When preview features disabled, signature should NOT be embedded in ID + assert THOUGHT_SIGNATURE_SEPARATOR not in tool_call_id + # But we can still extract from provider_specific_fields + tool_obj = {"id": tool_call_id, "type": "function", "provider_specific_fields": {"thought_signature": test_signature}} + decoded_sig = _get_thought_signature_from_tool(tool_obj) + assert decoded_sig == test_signature + finally: + # Restore original state + litellm.enable_preview_features = original_flag def test_get_thought_signature_backward_compatibility(): @@ -168,97 +193,157 @@ def test_convert_to_gemini_with_embedded_signature(): assert gemini_parts[0]["thoughtSignature"] == test_signature -def test_openai_client_e2e_flow(): +@pytest.mark.parametrize("enable_preview_features", [True, False]) +def test_openai_client_e2e_flow(enable_preview_features): """ End-to-end test simulating OpenAI client usage: 1. LiteLLM receives response from Gemini with thought signature - 2. LiteLLM embeds signature in tool call ID + 2. LiteLLM embeds signature in tool call ID (if preview features enabled) 3. OpenAI client sends message back with same tool call ID - 4. LiteLLM extracts signature from ID and sends to Gemini + 4. LiteLLM extracts signature from ID/provider_specific_fields and sends to Gemini """ test_signature = "Co4CAdHtim/rWgXbz2Ghp4tShzLeMASrPw6JJyYIC3cbVyZnKzU3uv8/wVzyS2sKRPL2m8QQHHXbNQhEEz500G7n/4ZMmksdTtfQcJMoT76S1DGwhnAiLwTgWCNXs3lEb4M19EVYoWFxhrH5Lr9YMIquoU9U4paydGwvZyIyigamIg4B6WnxrRsf0KZV12gJed0DZuKczvOFtHz3zUnmZRlOiTzd5gBVyQM+5jv1VI8m4WUKd6cN/5a5ZvaA0ggiO6kdVhlpIVs7GczSEVJD8KH4u02X7VSnb7CvykqDntZzV0y8rZFBEFGKrChmeHlWXP4D1IB3F9KQyhuLgWImMzg4BajKVxxMU737JGnNISy5" - # Step 1: Gemini returns function call with thought signature - gemini_parts = [ - HttpxPartType( - functionCall={ - "name": "get_current_temperature", - "args": {"location": "Paris"}, - }, - thoughtSignature=test_signature, - ) - ] + # Save original state + original_flag = litellm.enable_preview_features + litellm.enable_preview_features = enable_preview_features - # Step 2: LiteLLM transforms to OpenAI format with embedded signature - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=gemini_parts, - cumulative_tool_call_idx=0, - is_function_call=False, - ) - - assert tools is not None - assert len(tools) == 1 - tool_call_id = tools[0]["id"] - assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id - - # Step 3: OpenAI client sends back assistant message (preserves tool_call_id) - openai_assistant_message = { - "role": "assistant", - "content": "", - "tool_calls": [ - { - "id": tool_call_id, # Preserved from response - "type": "function", - "function": { + try: + # Step 1: Gemini returns function call with thought signature + gemini_parts = [ + HttpxPartType( + functionCall={ "name": "get_current_temperature", - "arguments": '{"location": "Paris"}', + "args": {"location": "Paris"}, }, + thoughtSignature=test_signature, + ) + ] + + # Step 2: LiteLLM transforms to OpenAI format + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=gemini_parts, + cumulative_tool_call_idx=0, + is_function_call=False, + ) + + assert tools is not None + assert len(tools) == 1 + tool_call_id = tools[0]["id"] + + if enable_preview_features: + # When preview features enabled, signature should be embedded in ID + assert THOUGHT_SIGNATURE_SEPARATOR in tool_call_id + else: + # When preview features disabled, signature should NOT be embedded in ID + assert THOUGHT_SIGNATURE_SEPARATOR not in tool_call_id + + # Step 3: OpenAI client sends back assistant message + # For the disabled case, we simulate that the client might have provider_specific_fields + # or we use the embedded ID if preview features were enabled + if enable_preview_features: + openai_assistant_message = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": tool_call_id, # Preserved from response (with embedded signature) + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + }, + } + ], + } + else: + # When preview features disabled, simulate that provider_specific_fields might be preserved + # (though in real OpenAI client usage, this might not happen) + # For this test, we'll use provider_specific_fields to show extraction still works + openai_assistant_message = { + "role": "assistant", + "content": "", + "tool_calls": [ + { + "id": tool_call_id, # ID without embedded signature + "type": "function", + "function": { + "name": "get_current_temperature", + "arguments": '{"location": "Paris"}', + }, + "provider_specific_fields": {"thought_signature": test_signature}, + } + ], } - ], - } - # Step 4: LiteLLM converts back to Gemini format, extracting signature - gemini_parts_converted = convert_to_gemini_tool_call_invoke( - openai_assistant_message - ) + # Step 4: LiteLLM converts back to Gemini format, extracting signature + gemini_parts_converted = convert_to_gemini_tool_call_invoke( + openai_assistant_message + ) - # Verify signature is preserved through the round trip - assert len(gemini_parts_converted) == 1 - assert "thoughtSignature" in gemini_parts_converted[0] - assert gemini_parts_converted[0]["thoughtSignature"] == test_signature + # Verify signature is preserved through the round trip + assert len(gemini_parts_converted) == 1 + assert "thoughtSignature" in gemini_parts_converted[0] + assert gemini_parts_converted[0]["thoughtSignature"] == test_signature + finally: + # Restore original state + litellm.enable_preview_features = original_flag -def test_parallel_tool_calls_with_signatures(): +@pytest.mark.parametrize("enable_preview_features", [True, False]) +def test_parallel_tool_calls_with_signatures(enable_preview_features): """Test that parallel tool calls preserve signatures correctly""" signature1 = "signature_for_first_call" # Only first call has signature (Gemini behavior for parallel calls) - gemini_parts = [ - HttpxPartType( - functionCall={"name": "get_temperature", "args": {"location": "Paris"}}, - thoughtSignature=signature1, - ), - HttpxPartType( - functionCall={"name": "get_temperature", "args": {"location": "London"}}, - # No signature for second parallel call - ), - ] + # Save original state + original_flag = litellm.enable_preview_features + litellm.enable_preview_features = enable_preview_features - function, tools, _ = VertexGeminiConfig._transform_parts( - parts=gemini_parts, - cumulative_tool_call_idx=0, - is_function_call=False, - ) + try: + gemini_parts = [ + HttpxPartType( + functionCall={"name": "get_temperature", "args": {"location": "Paris"}}, + thoughtSignature=signature1, + ), + HttpxPartType( + functionCall={"name": "get_temperature", "args": {"location": "London"}}, + # No signature for second parallel call + ), + ] - assert tools is not None - assert len(tools) == 2 + function, tools, _ = VertexGeminiConfig._transform_parts( + parts=gemini_parts, + cumulative_tool_call_idx=0, + is_function_call=False, + ) - # First tool call has signature in ID - assert THOUGHT_SIGNATURE_SEPARATOR in tools[0]["id"] - sig1 = _get_thought_signature_from_tool({"id": tools[0]["id"], "type": "function"}) - assert sig1 == signature1 + assert tools is not None + assert len(tools) == 2 - # Second tool call has no signature in ID - assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] - sig2 = _get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) - assert sig2 is None + # First tool call should have signature in provider_specific_fields + assert tools[0].get("provider_specific_fields", {}).get("thought_signature") == signature1 + + if enable_preview_features: + # When preview features enabled, first tool call has signature in ID + assert THOUGHT_SIGNATURE_SEPARATOR in tools[0]["id"] + sig1 = _get_thought_signature_from_tool({"id": tools[0]["id"], "type": "function"}) + assert sig1 == signature1 + else: + # When preview features disabled, signature should NOT be in ID + assert THOUGHT_SIGNATURE_SEPARATOR not in tools[0]["id"] + # But we can extract from provider_specific_fields + sig1 = _get_thought_signature_from_tool({ + "id": tools[0]["id"], + "type": "function", + "provider_specific_fields": {"thought_signature": signature1} + }) + assert sig1 == signature1 + + # Second tool call has no signature in ID (regardless of flag) + assert THOUGHT_SIGNATURE_SEPARATOR not in tools[1]["id"] + sig2 = _get_thought_signature_from_tool({"id": tools[1]["id"], "type": "function"}) + assert sig2 is None + finally: + # Restore original state + litellm.enable_preview_features = original_flag From 099ccf56a746c9bca8afa4c0c62fdb52fa5abfa9 Mon Sep 17 00:00:00 2001 From: Richard Song <9144514+richardmcsong@users.noreply.github.com> Date: Wed, 3 Dec 2025 00:57:07 -0500 Subject: [PATCH 05/24] Refactor add_schema_to_components to move definitions to components/schemas and add corresponding unit test (#17389) --- .../proxy/common_utils/custom_openapi_spec.py | 2 +- .../common_utils/test_custom_openapi_spec.py | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c448742f6dc..69472c2cda4 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -72,7 +72,7 @@ class CustomOpenAPISpec: openapi_schema["components"]["schemas"] = {} # Add the schema - openapi_schema["components"]["schemas"][schema_name] = schema_def + CustomOpenAPISpec._move_defs_to_components(openapi_schema, {schema_name: schema_def}) @staticmethod def add_request_body_to_paths(openapi_schema: Dict[str, Any], paths: List[str], schema_ref: str) -> None: diff --git a/tests/test_litellm/proxy/common_utils/test_custom_openapi_spec.py b/tests/test_litellm/proxy/common_utils/test_custom_openapi_spec.py index 7549b4259af..5fef35eb821 100644 --- a/tests/test_litellm/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/test_litellm/proxy/common_utils/test_custom_openapi_spec.py @@ -90,6 +90,35 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema +def test_defs_rewritten_in_add_schema_to_components(): + """ + Test that defs are rewritten to components/schemas in add_schema_to_components. + """ + + openapi_schema = {} + schema_name = "SchemaName" + schema_def = { + "type": "object", + "properties": { + "messages": { + "type": "array", + "items": { + "anyOf": [ + {"$ref": "#/$defs/UserMessage"}, + {"$ref": "#/$defs/AssistantMessage"} + ] + } + } + }, + "$defs": { + "UserMessage": {"type": "object"}, + "AssistantMessage": {"type": "object"} + } + } + CustomOpenAPISpec.add_schema_to_components(openapi_schema=openapi_schema, schema_name=schema_name, schema_def=schema_def) + assert "$defs" not in openapi_schema + assert openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_move_defs_to_components(): """ From f22bc0aab20e9b5336e73d67ba1631176cbacfd6 Mon Sep 17 00:00:00 2001 From: Matt Greathouse Date: Wed, 3 Dec 2025 01:00:19 -0500 Subject: [PATCH 06/24] Support Deepseek 3.2 with Reasoning (#17384) * Add openrouter/deepseek/deepseek-v3.2 * Added deepseek-provided v3.2 * Allow reasoning effort param for openrouter models that support it * Added tests --- .../llms/openrouter/chat/transformation.py | 15 ++++++++++ model_prices_and_context_window.json | 30 +++++++++++++++++++ .../test_openrouter_chat_transformation.py | 27 +++++++++++++++++ 3 files changed, 72 insertions(+) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index f1eafe4e294..b5610852fd2 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -10,6 +10,7 @@ from enum import Enum from typing import Any, AsyncIterator, Iterator, List, Optional, Tuple, Union, cast import httpx +import litellm from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException @@ -28,6 +29,20 @@ class CacheControlSupportedModels(str, Enum): class OpenrouterConfig(OpenAIGPTConfig): + def get_supported_openai_params(self, model: str) -> list: + """ + Allow reasoning parameters for models flagged as reasoning-capable. + """ + supported_params = super().get_supported_openai_params(model=model) + try: + if litellm.supports_reasoning( + model=model, custom_llm_provider="openrouter" + ) or litellm.supports_reasoning(model=model): + supported_params.append("reasoning_effort") + except Exception: + pass + return list(dict.fromkeys(supported_params)) + def map_openai_params( self, non_default_params: dict, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 19ed734c5f8..f82abce525f 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -9629,6 +9629,21 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "deepseek/deepseek-v3.2": { + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "deepseek.v3-v1:0": { "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", @@ -20565,6 +20580,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/deepseek/deepseek-v3.2": { + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/deepseek/deepseek-v3.2-exp": { "input_cost_per_token": 2e-07, "input_cost_per_token_cache_hit": 2e-08, diff --git a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index 64ac299fd79..d5a73b3fd12 100644 --- a/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/test_litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -489,3 +489,30 @@ def test_openrouter_cost_tracking_streaming(): # Verify cost field is preserved in the Usage object - this is the key data for cost tracking # The chunk_parser converts the dict to a Usage Pydantic model which includes the cost field assert result2.usage.cost == 0.0001 + + +def test_openrouter_reasoning_models_allow_reasoning_effort_param(): + """ + OpenRouter reasoning-capable models should accept the reasoning_effort param. + """ + config = OpenrouterConfig() + + supported_params = config.get_supported_openai_params( + model="openrouter/deepseek/deepseek-v3.2" + ) + + assert "reasoning_effort" in supported_params + assert supported_params.count("reasoning_effort") == 1 + + +def test_openrouter_non_reasoning_models_do_not_add_reasoning_effort(): + """ + Models without reasoning support should not gain reasoning-specific params. + """ + config = OpenrouterConfig() + + supported_params = config.get_supported_openai_params( + model="openrouter/anthropic/claude-3-5-haiku" + ) + + assert "reasoning_effort" not in supported_params From ae633184f72d8a12ae442a03ce86f51fc4f649fc Mon Sep 17 00:00:00 2001 From: mossbanay <2216177+mossbanay@users.noreply.github.com> Date: Wed, 3 Dec 2025 17:02:57 +1100 Subject: [PATCH 07/24] Add model price & details for Bedrock model global.anthropic.claude-opus-4-5-20251101-v1:0 (#17380) --- model_prices_and_context_window.json | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f82abce525f..999c88dde05 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -23850,6 +23850,32 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, From 6b5ad5d5a6c9cce7008d0eb57258a826e6d82808 Mon Sep 17 00:00:00 2001 From: Ali Saleh Date: Wed, 3 Dec 2025 11:03:54 +0500 Subject: [PATCH 08/24] docs: Update Instructions For Phoenix Integration (#17373) --- .../docs/observability/phoenix_integration.md | 79 +++++++++++++++---- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/docs/my-website/docs/observability/phoenix_integration.md b/docs/my-website/docs/observability/phoenix_integration.md index ad337439934..898d780668d 100644 --- a/docs/my-website/docs/observability/phoenix_integration.md +++ b/docs/my-website/docs/observability/phoenix_integration.md @@ -6,7 +6,7 @@ Open source tracing and evaluation platform :::tip -This is community maintained, Please make an issue if you run into a bug +This is community maintained. Please make an issue if you run into a bug: https://github.com/BerriAI/litellm ::: @@ -31,19 +31,16 @@ litellm.callbacks = ["arize_phoenix"] import litellm import os -os.environ["PHOENIX_API_KEY"] = "" # Necessary only using Phoenix Cloud -os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "" # The URL of your Phoenix OSS instance e.g. http://localhost:6006/v1/traces -os.environ["PHOENIX_PROJECT_NAME"]="litellm" # OPTIONAL: you can configure project names, otherwise traces would go to "default" project +# Set env variables +os.environ["PHOENIX_API_KEY"] = "d0*****" # Set the Phoenix API key here. It is necessary only when using Phoenix Cloud. +os.environ["PHOENIX_COLLECTOR_HTTP_ENDPOINT"] = "https://app.phoenix.arize.com/s//v1/traces" # Set the URL of your Phoenix OSS instance, otherwise tracer would use https://app.phoenix.arize.com/v1/traces for Phoenix Cloud. +os.environ["PHOENIX_PROJECT_NAME"] = "litellm" # Configure the project name, otherwise traces would go to "default" project. +os.environ['OPENAI_API_KEY'] = "fake-key" # Set the OpenAI API key here. -# This defaults to https://app.phoenix.arize.com/v1/traces for Phoenix Cloud - -# LLM API Keys -os.environ['OPENAI_API_KEY']="" - -# set arize as a callback, litellm will send the data to arize +# Set arize_phoenix as a callback & LiteLLM will send the data to Phoenix. litellm.callbacks = ["arize_phoenix"] - -# openai call + +# OpenAI call response = litellm.completion( model="gpt-3.5-turbo", messages=[ @@ -52,8 +49,9 @@ response = litellm.completion( ) ``` -### Using with LiteLLM Proxy +## Using with LiteLLM Proxy +1. Setup config.yaml ```yaml model_list: @@ -66,12 +64,63 @@ model_list: litellm_settings: callbacks: ["arize_phoenix"] +general_settings: + master_key: "sk-1234" + environment_variables: PHOENIX_API_KEY: "d0*****" - PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the GRPC endpoint - PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/v1/traces" # OPTIONAL, for setting the HTTP endpoint + PHOENIX_COLLECTOR_ENDPOINT: "https://app.phoenix.arize.com/s//v1/traces" # OPTIONAL - For setting the gRPC endpoint + PHOENIX_COLLECTOR_HTTP_ENDPOINT: "https://app.phoenix.arize.com/s//v1/traces" # OPTIONAL - For setting the HTTP endpoint ``` +2. Start the proxy + +```bash +litellm --config config.yaml +``` + +3. Test it! + +```bash +curl -X POST 'http://0.0.0.0:4000/chat/completions' \ +-H 'Content-Type: application/json' \ +-H 'Authorization: Bearer sk-1234' \ +-d '{ "model": "gpt-4o", "messages": [{"role": "user", "content": "Hi 👋 - i'm openai"}]}' +``` + +## Supported Phoenix Endpoints +Phoenix now supports multiple deployment types. The correct endpoint depends on which version of Phoenix Cloud you are using. + +**Phoenix Cloud (With Spaces - New Version)** +Use this if your Phoenix URL contains `/s/` path. + +```bash +https://app.phoenix.arize.com/s//v1/traces +``` + +**Phoenix Cloud (Legacy - Deprecated)** +Use this only if your deployment still shows the `/legacy` pattern. + +```bash +https://app.phoenix.arize.com/legacy/v1/traces +``` + +**Phoenix Cloud (Without Spaces - Old Version)** +Use this if your Phoenix Cloud URL does not contain `/s/` or `/legacy` path. + +```bash +https://app.phoenix.arize.com/v1/traces +``` + +**Self-Hosted Phoenix (Local Instance)** +Use this when running Phoenix on your machine or a private server. + +```bash +http://localhost:6006/v1/traces +``` + +Depending on which Phoenix Cloud version or deployment you are using, you should set the corresponding endpoint in `PHOENIX_COLLECTOR_HTTP_ENDPOINT` or `PHOENIX_COLLECTOR_ENDPOINT`. + ## Support & Talk to Founders - [Schedule Demo 👋](https://calendly.com/d/4mp-gd3-k5k/berriai-1-1-onboarding-litellm-hosted-version) From 566adebdec41d3f2790de33e43d93892f529f597 Mon Sep 17 00:00:00 2001 From: Mariano Hielpos <108539968+mhielpos-asapp@users.noreply.github.com> Date: Wed, 3 Dec 2025 03:06:51 -0300 Subject: [PATCH 09/24] update model_prices_and_context_window.json (#17376) * update model_prices_and_context_window.json * update * update --- ...odel_prices_and_context_window_backup.json | 88 +++++++++++++++++-- model_prices_and_context_window.json | 88 +++++++++++++++++-- 2 files changed, 160 insertions(+), 16 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 932508824af..464f9c185f9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -10772,25 +10772,25 @@ "supports_tool_choice": true }, "ft:babbage-002": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 1.6e-06, "input_cost_per_token_batches": 2e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 16384, "mode": "completion", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 2e-07 }, "ft:davinci-002": { - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1.2e-05, "input_cost_per_token_batches": 1e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 16384, "mode": "completion", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 1e-06 }, "ft:gpt-3.5-turbo": { @@ -10853,6 +10853,7 @@ "supports_tool_choice": true }, "ft:gpt-4o-2024-08-06": { + "cache_read_input_token_cost": 1.875e-06, "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 1.875e-06, "litellm_provider": "openai", @@ -10865,6 +10866,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -10885,8 +10887,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_tool_choice": true }, "ft:gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 1.5e-07, @@ -10905,8 +10906,79 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_tool_choice": true + }, + "ft:gpt-4.1-2025-04-14": { + "cache_read_input_token_cost": 7.5e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:gpt-4.1-mini-2025-04-14": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 8e-07, + "input_cost_per_token_batches": 4e-07, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "output_cost_per_token_batches": 1.6e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:gpt-4.1-nano-2025-04-14": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "output_cost_per_token_batches": 4e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:o4-mini-2025-04-16": { + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 1.6e-05, + "output_cost_per_token_batches": 8e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "gemini-1.0-pro": { "input_cost_per_character": 1.25e-07, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 999c88dde05..dbaa60e0f1d 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -10852,25 +10852,25 @@ "supports_tool_choice": true }, "ft:babbage-002": { - "input_cost_per_token": 4e-07, + "input_cost_per_token": 1.6e-06, "input_cost_per_token_batches": 2e-07, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 16384, "mode": "completion", - "output_cost_per_token": 4e-07, + "output_cost_per_token": 1.6e-06, "output_cost_per_token_batches": 2e-07 }, "ft:davinci-002": { - "input_cost_per_token": 2e-06, + "input_cost_per_token": 1.2e-05, "input_cost_per_token_batches": 1e-06, "litellm_provider": "text-completion-openai", "max_input_tokens": 16384, "max_output_tokens": 4096, "max_tokens": 16384, "mode": "completion", - "output_cost_per_token": 2e-06, + "output_cost_per_token": 1.2e-05, "output_cost_per_token_batches": 1e-06 }, "ft:gpt-3.5-turbo": { @@ -10933,6 +10933,7 @@ "supports_tool_choice": true }, "ft:gpt-4o-2024-08-06": { + "cache_read_input_token_cost": 1.875e-06, "input_cost_per_token": 3.75e-06, "input_cost_per_token_batches": 1.875e-06, "litellm_provider": "openai", @@ -10945,6 +10946,7 @@ "supports_function_calling": true, "supports_parallel_function_calling": true, "supports_pdf_input": true, + "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -10965,8 +10967,7 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_tool_choice": true }, "ft:gpt-4o-mini-2024-07-18": { "cache_read_input_token_cost": 1.5e-07, @@ -10985,8 +10986,79 @@ "supports_prompt_caching": true, "supports_response_schema": true, "supports_system_messages": true, - "supports_tool_choice": true, - "supports_vision": true + "supports_tool_choice": true + }, + "ft:gpt-4.1-2025-04-14": { + "cache_read_input_token_cost": 7.5e-07, + "input_cost_per_token": 3e-06, + "input_cost_per_token_batches": 1.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 1.2e-05, + "output_cost_per_token_batches": 6e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:gpt-4.1-mini-2025-04-14": { + "cache_read_input_token_cost": 2e-07, + "input_cost_per_token": 8e-07, + "input_cost_per_token_batches": 4e-07, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 3.2e-06, + "output_cost_per_token_batches": 1.6e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:gpt-4.1-nano-2025-04-14": { + "cache_read_input_token_cost": 5e-08, + "input_cost_per_token": 2e-07, + "input_cost_per_token_batches": 1e-07, + "litellm_provider": "openai", + "max_input_tokens": 1047576, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "chat", + "output_cost_per_token": 8e-07, + "output_cost_per_token_batches": 4e-07, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, + "ft:o4-mini-2025-04-16": { + "cache_read_input_token_cost": 1e-06, + "input_cost_per_token": 4e-06, + "input_cost_per_token_batches": 2e-06, + "litellm_provider": "openai", + "max_input_tokens": 200000, + "max_output_tokens": 100000, + "max_tokens": 100000, + "mode": "chat", + "output_cost_per_token": 1.6e-05, + "output_cost_per_token_batches": 8e-06, + "supports_function_calling": true, + "supports_parallel_function_calling": false, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true }, "gemini-1.0-pro": { "input_cost_per_character": 1.25e-07, From 17faea96bb75be88f6761398e3ad2130a0d29cfc Mon Sep 17 00:00:00 2001 From: Jonathan Yang Date: Wed, 3 Dec 2025 07:09:57 +0100 Subject: [PATCH 10/24] fix: conditionally pass enable_cleanup_closed to aiohttp TCPConnector (#17367) * fix: conditionally pass enable_cleanup_closed to aiohttp TCPConnector Fixes deprecation warning on Python 3.12.7+ and 3.13.1+ where enable_cleanup_closed is no longer needed since the underlying CPython SSL connection leak bug was fixed. See: https://github.com/python/cpython/pull/118960 * chore: add aiohttp source reference to AIOHTTP_NEEDS_CLEANUP_CLOSED --- litellm/constants.py | 7 +++++++ litellm/llms/custom_httpx/http_handler.py | 3 ++- litellm/proxy/proxy_server.py | 3 ++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index 6a67a9a0e18..db617a2e475 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,4 +1,5 @@ import os +import sys from typing import List, Literal DEFAULT_HEALTH_CHECK_PROMPT = str( @@ -103,6 +104,12 @@ _DEFAULT_TTL_FOR_HTTPX_CLIENTS = 3600 # 1 hour, re-use the same httpx client fo AIOHTTP_CONNECTOR_LIMIT = int(os.getenv("AIOHTTP_CONNECTOR_LIMIT", 0)) AIOHTTP_KEEPALIVE_TIMEOUT = int(os.getenv("AIOHTTP_KEEPALIVE_TIMEOUT", 120)) AIOHTTP_TTL_DNS_CACHE = int(os.getenv("AIOHTTP_TTL_DNS_CACHE", 300)) +# enable_cleanup_closed is only needed for Python versions with the SSL leak bug +# Fixed in Python 3.12.7+ and 3.13.1+ (see https://github.com/python/cpython/pull/118960) +# Reference: https://github.com/aio-libs/aiohttp/blob/master/aiohttp/connector.py#L74-L78 +AIOHTTP_NEEDS_CLEANUP_CLOSED = ( + (3, 13, 0) <= sys.version_info < (3, 13, 1) or sys.version_info < (3, 12, 7) +) # WebSocket constants # Default to None (unlimited) to match OpenAI's official agents SDK behavior diff --git a/litellm/llms/custom_httpx/http_handler.py b/litellm/llms/custom_httpx/http_handler.py index c35e910ab08..b06e8463abd 100644 --- a/litellm/llms/custom_httpx/http_handler.py +++ b/litellm/llms/custom_httpx/http_handler.py @@ -17,6 +17,7 @@ from litellm.constants import ( _DEFAULT_TTL_FOR_HTTPX_CLIENTS, AIOHTTP_CONNECTOR_LIMIT, AIOHTTP_KEEPALIVE_TIMEOUT, + AIOHTTP_NEEDS_CLEANUP_CLOSED, AIOHTTP_TTL_DNS_CACHE, DEFAULT_SSL_CIPHERS, ) @@ -798,7 +799,7 @@ class AsyncHTTPHandler: limit=AIOHTTP_CONNECTOR_LIMIT, keepalive_timeout=AIOHTTP_KEEPALIVE_TIMEOUT, ttl_dns_cache=AIOHTTP_TTL_DNS_CACHE, - enable_cleanup_closed=True, + enable_cleanup_closed=AIOHTTP_NEEDS_CLEANUP_CLOSED, **connector_kwargs, ), trust_env=trust_env, diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 99789162318..a1e01caddc1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -32,6 +32,7 @@ from litellm._uuid import uuid from litellm.constants import ( AIOHTTP_CONNECTOR_LIMIT, AIOHTTP_KEEPALIVE_TIMEOUT, + AIOHTTP_NEEDS_CLEANUP_CLOSED, AIOHTTP_TTL_DNS_CACHE, AUDIO_SPEECH_CHUNK_SIZE, BASE_MCP_ROUTE, @@ -635,7 +636,7 @@ async def _initialize_shared_aiohttp_session(): limit=AIOHTTP_CONNECTOR_LIMIT, keepalive_timeout=AIOHTTP_KEEPALIVE_TIMEOUT, ttl_dns_cache=AIOHTTP_TTL_DNS_CACHE, - enable_cleanup_closed=True, + enable_cleanup_closed=AIOHTTP_NEEDS_CLEANUP_CLOSED, ) session = ClientSession(connector=connector) From 43dd9e4a90281e42fd1015cd72cbca5280fc4e4c Mon Sep 17 00:00:00 2001 From: Jonathan Yang Date: Wed, 3 Dec 2025 07:12:55 +0100 Subject: [PATCH 11/24] fix: replace deprecated .dict() with .model_dump() in streaming_handler (#17359) Replace Pydantic v1 `.dict()` method with v2 `.model_dump()` to fix PydanticDeprecatedSince20 warnings. The `.dict()` method is deprecated in Pydantic v2 and will be removed in v3. Fixes #5987 --- .../litellm_core_utils/streaming_handler.py | 53 +++++++++++-------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index 4d8e109d882..a7f460fab59 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -96,9 +96,9 @@ class CustomStreamWrapper: self.system_fingerprint: Optional[str] = None self.received_finish_reason: Optional[str] = None - self.intermittent_finish_reason: Optional[str] = ( - None # finish reasons that show up mid-stream - ) + self.intermittent_finish_reason: Optional[ + str + ] = None # finish reasons that show up mid-stream self.special_tokens = [ "<|assistant|>", "<|system|>", @@ -735,7 +735,7 @@ class CustomStreamWrapper: and completion_obj["function_call"] is not None ) or ( - "tool_calls" in model_response.choices[0].delta + "tool_calls" in model_response.choices[0].delta and model_response.choices[0].delta["tool_calls"] is not None ) or ( @@ -889,7 +889,6 @@ class CustomStreamWrapper: ## check if openai/azure chunk original_chunk = response_obj.get("original_chunk", None) if original_chunk: - if len(original_chunk.choices) > 0: choices = [] for choice in original_chunk.choices: @@ -906,7 +905,6 @@ class CustomStreamWrapper: print_verbose(f"choices in streaming: {choices}") setattr(model_response, "choices", choices) else: - return model_response.system_fingerprint = ( original_chunk.system_fingerprint @@ -1435,9 +1433,9 @@ class CustomStreamWrapper: _json_delta = delta.model_dump() print_verbose(f"_json_delta: {_json_delta}") if "role" not in _json_delta or _json_delta["role"] is None: - _json_delta["role"] = ( - "assistant" # mistral's api returns role as None - ) + _json_delta[ + "role" + ] = "assistant" # mistral's api returns role as None if "tool_calls" in _json_delta and isinstance( _json_delta["tool_calls"], list ): @@ -1533,7 +1531,7 @@ class CustomStreamWrapper: async def _call_post_streaming_deployment_hook(self, chunk): """ Call the post-call streaming deployment hook for callbacks. - + This allows callbacks to modify streaming chunks before they're returned. """ try: @@ -1544,15 +1542,17 @@ class CustomStreamWrapper: # Get request kwargs from logging object request_data = self.logging_obj.model_call_details call_type_str = self.logging_obj.call_type - + try: typed_call_type = CallTypes(call_type_str) except ValueError: typed_call_type = None - + # Call hooks for all callbacks for callback in litellm.callbacks: - if isinstance(callback, CustomLogger) and hasattr(callback, "async_post_call_streaming_deployment_hook"): + if isinstance(callback, CustomLogger) and hasattr( + callback, "async_post_call_streaming_deployment_hook" + ): result = await callback.async_post_call_streaming_deployment_hook( request_data=request_data, response_chunk=chunk, @@ -1560,11 +1560,14 @@ class CustomStreamWrapper: ) if result is not None: chunk = result - + return chunk except Exception as e: from litellm._logging import verbose_logger - verbose_logger.exception(f"Error in post-call streaming deployment hook: {str(e)}") + + verbose_logger.exception( + f"Error in post-call streaming deployment hook: {str(e)}" + ) return chunk def cache_streaming_response(self, processed_chunk, cache_hit: bool): @@ -1687,7 +1690,7 @@ class CustomStreamWrapper: response, "usage" ): # remove usage from chunk, only send on final chunk # Convert the object to a dictionary - obj_dict = response.dict() + obj_dict = response.model_dump() # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: @@ -1852,7 +1855,7 @@ class CustomStreamWrapper: processed_chunk, "usage" ): # remove usage from chunk, only send on final chunk # Convert the object to a dictionary - obj_dict = processed_chunk.dict() + obj_dict = processed_chunk.model_dump() # Remove an attribute (e.g., 'attr2') if "usage" in obj_dict: @@ -1872,11 +1875,15 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) processed_chunk._hidden_params["usage"] = usage - + # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: - processed_chunk = await self._call_post_streaming_deployment_hook(processed_chunk) - + processed_chunk = ( + await self._call_post_streaming_deployment_hook( + processed_chunk + ) + ) + return processed_chunk raise StopAsyncIteration else: # temporary patch for non-aiohttp async calls @@ -1890,9 +1897,9 @@ class CustomStreamWrapper: chunk = next(self.completion_stream) if chunk is not None and chunk != b"": print_verbose(f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk}") - processed_chunk: Optional[ModelResponseStream] = ( - self.chunk_creator(chunk=chunk) - ) + processed_chunk: Optional[ + ModelResponseStream + ] = self.chunk_creator(chunk=chunk) print_verbose( f"PROCESSED CHUNK POST CHUNK CREATOR: {processed_chunk}" ) From e289f5e454140042eda28ca27b8797078e2fe9f6 Mon Sep 17 00:00:00 2001 From: Deepak Tammali <45919384+deepaktammali@users.noreply.github.com> Date: Wed, 3 Dec 2025 11:44:48 +0530 Subject: [PATCH 12/24] feat: make streaming chunk size configurable in bedrock converse and invoke handlers (#17357) --- litellm/llms/bedrock/chat/converse_handler.py | 10 ++++++++-- litellm/llms/bedrock/chat/invoke_handler.py | 20 ++++++++++++------- 2 files changed, 21 insertions(+), 9 deletions(-) diff --git a/litellm/llms/bedrock/chat/converse_handler.py b/litellm/llms/bedrock/chat/converse_handler.py index fd1f6f0c893..d5bd054118d 100644 --- a/litellm/llms/bedrock/chat/converse_handler.py +++ b/litellm/llms/bedrock/chat/converse_handler.py @@ -29,6 +29,7 @@ def make_sync_call( logging_obj: LiteLLMLoggingObject, json_mode: Optional[bool] = False, fake_stream: bool = False, + stream_chunk_size: int = 1024, ): if client is None: client = _get_httpx_client() # Create a new client if none provided @@ -66,7 +67,7 @@ def make_sync_call( ) else: decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING logging_obj.post_call( @@ -102,6 +103,7 @@ class BedrockConverseLLM(BaseAWSLLM): fake_stream: bool = False, json_mode: Optional[bool] = False, api_key: Optional[str] = None, + stream_chunk_size: int = 1024, ) -> CustomStreamWrapper: request_data = await litellm.AmazonConverseConfig()._async_transform_request( model=model, @@ -143,6 +145,7 @@ class BedrockConverseLLM(BaseAWSLLM): logging_obj=logging_obj, fake_stream=fake_stream, json_mode=json_mode, + stream_chunk_size=stream_chunk_size, ) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, @@ -260,6 +263,7 @@ class BedrockConverseLLM(BaseAWSLLM): ): ## SETUP ## stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) unencoded_model_id = optional_params.pop("model_id", None) fake_stream = optional_params.pop("fake_stream", False) json_mode = optional_params.get("json_mode", False) @@ -356,7 +360,8 @@ class BedrockConverseLLM(BaseAWSLLM): json_mode=json_mode, fake_stream=fake_stream, credentials=credentials, - api_key=api_key + api_key=api_key, + stream_chunk_size=stream_chunk_size, ) # type: ignore ### ASYNC COMPLETION return self.async_completion( @@ -433,6 +438,7 @@ class BedrockConverseLLM(BaseAWSLLM): logging_obj=logging_obj, json_mode=json_mode, fake_stream=fake_stream, + stream_chunk_size=stream_chunk_size, ) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, diff --git a/litellm/llms/bedrock/chat/invoke_handler.py b/litellm/llms/bedrock/chat/invoke_handler.py index 7a960fd45d2..5e33a266449 100644 --- a/litellm/llms/bedrock/chat/invoke_handler.py +++ b/litellm/llms/bedrock/chat/invoke_handler.py @@ -192,6 +192,7 @@ async def make_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, ): try: if client is None: @@ -235,7 +236,7 @@ async def make_call( json_mode=json_mode, ) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) elif bedrock_invoke_provider == "deepseek_r1": decoder = AmazonDeepSeekR1StreamDecoder( @@ -243,12 +244,12 @@ async def make_call( sync_stream=False, ) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) else: decoder = AWSEventStreamDecoder(model=model) completion_stream = decoder.aiter_bytes( - response.aiter_bytes(chunk_size=1024) + response.aiter_bytes(chunk_size=stream_chunk_size) ) # LOGGING @@ -281,6 +282,7 @@ def make_sync_call( fake_stream: bool = False, json_mode: Optional[bool] = False, bedrock_invoke_provider: Optional[litellm.BEDROCK_INVOKE_PROVIDERS_LITERAL] = None, + stream_chunk_size: int = 1024, ): try: if client is None: @@ -321,16 +323,16 @@ def make_sync_call( sync_stream=True, json_mode=json_mode, ) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) elif bedrock_invoke_provider == "deepseek_r1": decoder = AmazonDeepSeekR1StreamDecoder( model=model, sync_stream=True, ) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) else: decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) # LOGGING logging_obj.post_call( @@ -729,6 +731,7 @@ class BedrockLLM(BaseAWSLLM): ## SETUP ## stream = optional_params.pop("stream", None) + stream_chunk_size = optional_params.pop("stream_chunk_size", 1024) provider = self.get_bedrock_invoke_provider(model) modelId = self.get_bedrock_model_id( @@ -1003,6 +1006,7 @@ class BedrockLLM(BaseAWSLLM): headers=prepped.headers, timeout=timeout, client=client, + stream_chunk_size=stream_chunk_size, ) # type: ignore ### ASYNC COMPLETION return self.async_completion( @@ -1048,7 +1052,7 @@ class BedrockLLM(BaseAWSLLM): decoder = AWSEventStreamDecoder(model=model) - completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=1024)) + completion_stream = decoder.iter_bytes(response.iter_bytes(chunk_size=stream_chunk_size)) streaming_response = CustomStreamWrapper( completion_stream=completion_stream, model=model, @@ -1168,6 +1172,7 @@ class BedrockLLM(BaseAWSLLM): logger_fn=None, headers={}, client: Optional[AsyncHTTPHandler] = None, + stream_chunk_size: int = 1024, ) -> CustomStreamWrapper: # The call is not made here; instead, we prepare the necessary objects for the stream. @@ -1183,6 +1188,7 @@ class BedrockLLM(BaseAWSLLM): messages=messages, logging_obj=logging_obj, fake_stream=True if "ai21" in api_base else False, + stream_chunk_size=stream_chunk_size, ), model=model, custom_llm_provider="bedrock", From 4c6604b0da6bcdd9c230632f203921047163d636 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Wed, 3 Dec 2025 03:25:26 -0300 Subject: [PATCH 13/24] Cleanup: Remove orphan docs pages and Docusaurus template files (#17356) * docs: update getting started page - Add Core Functions table with link to full list - Add Responses API section - Add Async section with acompletion() example - Add "Switch Providers with One Line" example - Clarify Basic Usage supports multiple endpoints - Update models to current versions (openai/gpt-4o, anthropic/claude-sonnet-4) - Use provider/model format throughout - Fix deprecated import: from openai.error -> from openai - Keep original structure: community key, More details links, observability env vars * Cleanup: Remove orphan docs pages and Docusaurus template files - Remove orphan getting_started.md (not linked in sidebar) - Remove Docusaurus template intro.md - Remove tutorial-basics/ directory (Docusaurus template) - Remove tutorial-extras/ directory (Docusaurus template) --- docs/my-website/docs/getting_started.md | 108 ------------- docs/my-website/src/pages/intro.md | 47 ------ .../src/pages/tutorial-basics/_category_.json | 8 - .../pages/tutorial-basics/congratulations.md | 23 --- .../tutorial-basics/create-a-blog-post.md | 34 ---- .../tutorial-basics/create-a-document.md | 57 ------- .../pages/tutorial-basics/create-a-page.md | 43 ----- .../pages/tutorial-basics/deploy-your-site.md | 31 ---- .../tutorial-basics/markdown-features.mdx | 150 ------------------ .../src/pages/tutorial-extras/_category_.json | 7 - .../img/docsVersionDropdown.png | Bin 25427 -> 0 bytes .../tutorial-extras/img/localeDropdown.png | Bin 27841 -> 0 bytes .../tutorial-extras/manage-docs-versions.md | 55 ------- .../tutorial-extras/translate-your-site.md | 88 ---------- 14 files changed, 651 deletions(-) delete mode 100644 docs/my-website/docs/getting_started.md delete mode 100644 docs/my-website/src/pages/intro.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/_category_.json delete mode 100644 docs/my-website/src/pages/tutorial-basics/congratulations.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/create-a-document.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/create-a-page.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/deploy-your-site.md delete mode 100644 docs/my-website/src/pages/tutorial-basics/markdown-features.mdx delete mode 100644 docs/my-website/src/pages/tutorial-extras/_category_.json delete mode 100644 docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png delete mode 100644 docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png delete mode 100644 docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md delete mode 100644 docs/my-website/src/pages/tutorial-extras/translate-your-site.md diff --git a/docs/my-website/docs/getting_started.md b/docs/my-website/docs/getting_started.md deleted file mode 100644 index 6b2c1fd531e..00000000000 --- a/docs/my-website/docs/getting_started.md +++ /dev/null @@ -1,108 +0,0 @@ -# Getting Started - -import QuickStart from '../src/components/QuickStart.js' - -LiteLLM simplifies LLM API calls by mapping them all to the [OpenAI ChatCompletion format](https://platform.openai.com/docs/api-reference/chat). - -## basic usage - -By default we provide a free $10 community-key to try all providers supported on LiteLLM. - -```python -from litellm import completion - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "your-api-key" -os.environ["COHERE_API_KEY"] = "your-api-key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages) - -# cohere call -response = completion("command-nightly", messages) -``` - -**Need a dedicated key?** -Email us @ krrish@berri.ai - -Next Steps 👉 [Call all supported models - e.g. Claude-2, Llama2-70b, etc.](./proxy_api.md#supported-models) - -More details 👉 - -- [Completion() function details](./completion/) -- [Overview of supported models / providers on LiteLLM](./providers/) -- [Search all models / providers](https://models.litellm.ai/) -- [Build your own OpenAI proxy](https://github.com/BerriAI/liteLLM-proxy/tree/main) - -## streaming - -Same example from before. Just pass in `stream=True` in the completion args. - -```python -from litellm import completion - -## set ENV variables -os.environ["OPENAI_API_KEY"] = "openai key" -os.environ["COHERE_API_KEY"] = "cohere key" - -messages = [{ "content": "Hello, how are you?","role": "user"}] - -# openai call -response = completion(model="gpt-3.5-turbo", messages=messages, stream=True) - -# cohere call -response = completion("command-nightly", messages, stream=True) - -print(response) -``` - -More details 👉 - -- [streaming + async](./completion/stream.md) -- [tutorial for streaming Llama2 on TogetherAI](./tutorials/TogetherAI_liteLLM.md) - -## exception handling - -LiteLLM maps exceptions across all supported providers to the OpenAI exceptions. All our exceptions inherit from OpenAI's exception types, so any error-handling you have for that, should work out of the box with LiteLLM. - -```python -from openai.error import OpenAIError -from litellm import completion - -os.environ["ANTHROPIC_API_KEY"] = "bad-key" -try: - # some code - completion(model="claude-instant-1", messages=[{"role": "user", "content": "Hey, how's it going?"}]) -except OpenAIError as e: - print(e) -``` - -## Logging Observability - Log LLM Input/Output ([Docs](https://docs.litellm.ai/docs/observability/callbacks)) - -LiteLLM exposes pre defined callbacks to send data to MLflow, Lunary, Langfuse, Helicone, Promptlayer, Traceloop, Slack - -```python -from litellm import completion - -## set env variables for logging tools (API key set up is not required when using MLflow) -os.environ["LUNARY_PUBLIC_KEY"] = "your-lunary-public-key" # get your public key at https://app.lunary.ai/settings -os.environ["HELICONE_API_KEY"] = "your-helicone-key" -os.environ["LANGFUSE_PUBLIC_KEY"] = "" -os.environ["LANGFUSE_SECRET_KEY"] = "" - -os.environ["OPENAI_API_KEY"] - -# set callbacks -litellm.success_callback = ["lunary", "mlflow", "langfuse", "helicone"] # log input/output to MLflow, langfuse, lunary, helicone - -#openai call -response = completion(model="gpt-3.5-turbo", messages=[{"role": "user", "content": "Hi 👋 - i'm openai"}]) -``` - -More details 👉 - -- [exception mapping](./exception_mapping.md) -- [retries + model fallbacks for completion()](./completion/reliable_completions.md) -- [tutorial for model fallbacks with completion()](./tutorials/fallbacks.md) diff --git a/docs/my-website/src/pages/intro.md b/docs/my-website/src/pages/intro.md deleted file mode 100644 index 8a2e69d95f9..00000000000 --- a/docs/my-website/src/pages/intro.md +++ /dev/null @@ -1,47 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Tutorial Intro - -Let's discover **Docusaurus in less than 5 minutes**. - -## Getting Started - -Get started by **creating a new site**. - -Or **try Docusaurus immediately** with **[docusaurus.new](https://docusaurus.new)**. - -### What you'll need - -- [Node.js](https://nodejs.org/en/download/) version 16.14 or above: - - When installing Node.js, you are recommended to check all checkboxes related to dependencies. - -## Generate a new site - -Generate a new Docusaurus site using the **classic template**. - -The classic template will automatically be added to your project after you run the command: - -```bash -npm init docusaurus@latest my-website classic -``` - -You can type this command into Command Prompt, Powershell, Terminal, or any other integrated terminal of your code editor. - -The command also installs all necessary dependencies you need to run Docusaurus. - -## Start your site - -Run the development server: - -```bash -cd my-website -npm run start -``` - -The `cd` command changes the directory you're working with. In order to work with your newly created Docusaurus site, you'll need to navigate the terminal there. - -The `npm run start` command builds your website locally and serves it through a development server, ready for you to view at http://localhost:3000/. - -Open `docs/intro.md` (this page) and edit some lines: the site **reloads automatically** and displays your changes. diff --git a/docs/my-website/src/pages/tutorial-basics/_category_.json b/docs/my-website/src/pages/tutorial-basics/_category_.json deleted file mode 100644 index 2e6db55b1eb..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/_category_.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "label": "Tutorial - Basics", - "position": 2, - "link": { - "type": "generated-index", - "description": "5 minutes to learn the most important Docusaurus concepts." - } -} diff --git a/docs/my-website/src/pages/tutorial-basics/congratulations.md b/docs/my-website/src/pages/tutorial-basics/congratulations.md deleted file mode 100644 index 04771a00b72..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/congratulations.md +++ /dev/null @@ -1,23 +0,0 @@ ---- -sidebar_position: 6 ---- - -# Congratulations! - -You have just learned the **basics of Docusaurus** and made some changes to the **initial template**. - -Docusaurus has **much more to offer**! - -Have **5 more minutes**? Take a look at **[versioning](../tutorial-extras/manage-docs-versions.md)** and **[i18n](../tutorial-extras/translate-your-site.md)**. - -Anything **unclear** or **buggy** in this tutorial? [Please report it!](https://github.com/facebook/docusaurus/discussions/4610) - -## What's next? - -- Read the [official documentation](https://docusaurus.io/) -- Modify your site configuration with [`docusaurus.config.js`](https://docusaurus.io/docs/api/docusaurus-config) -- Add navbar and footer items with [`themeConfig`](https://docusaurus.io/docs/api/themes/configuration) -- Add a custom [Design and Layout](https://docusaurus.io/docs/styling-layout) -- Add a [search bar](https://docusaurus.io/docs/search) -- Find inspirations in the [Docusaurus showcase](https://docusaurus.io/showcase) -- Get involved in the [Docusaurus Community](https://docusaurus.io/community/support) diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md b/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md deleted file mode 100644 index ea472bbaf87..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-blog-post.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -sidebar_position: 3 ---- - -# Create a Blog Post - -Docusaurus creates a **page for each blog post**, but also a **blog index page**, a **tag system**, an **RSS** feed... - -## Create your first Post - -Create a file at `blog/2021-02-28-greetings.md`: - -```md title="blog/2021-02-28-greetings.md" ---- -slug: greetings -title: Greetings! -authors: - - name: Joel Marcey - title: Co-creator of Docusaurus 1 - url: https://github.com/JoelMarcey - image_url: https://github.com/JoelMarcey.png - - name: Sébastien Lorber - title: Docusaurus maintainer - url: https://sebastienlorber.com - image_url: https://github.com/slorber.png -tags: [greetings] ---- - -Congratulations, you have made your first post! - -Feel free to play around and edit this post as much you like. -``` - -A new blog post is now available at [http://localhost:3000/blog/greetings](http://localhost:3000/blog/greetings). diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-document.md b/docs/my-website/src/pages/tutorial-basics/create-a-document.md deleted file mode 100644 index ffddfa8eb8a..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-document.md +++ /dev/null @@ -1,57 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Create a Document - -Documents are **groups of pages** connected through: - -- a **sidebar** -- **previous/next navigation** -- **versioning** - -## Create your first Doc - -Create a Markdown file at `docs/hello.md`: - -```md title="docs/hello.md" -# Hello - -This is my **first Docusaurus document**! -``` - -A new document is now available at [http://localhost:3000/docs/hello](http://localhost:3000/docs/hello). - -## Configure the Sidebar - -Docusaurus automatically **creates a sidebar** from the `docs` folder. - -Add metadata to customize the sidebar label and position: - -```md title="docs/hello.md" {1-4} ---- -sidebar_label: 'Hi!' -sidebar_position: 3 ---- - -# Hello - -This is my **first Docusaurus document**! -``` - -It is also possible to create your sidebar explicitly in `sidebars.js`: - -```js title="sidebars.js" -module.exports = { - tutorialSidebar: [ - 'intro', - // highlight-next-line - 'hello', - { - type: 'category', - label: 'Tutorial', - items: ['tutorial-basics/create-a-document'], - }, - ], -}; -``` diff --git a/docs/my-website/src/pages/tutorial-basics/create-a-page.md b/docs/my-website/src/pages/tutorial-basics/create-a-page.md deleted file mode 100644 index 20e2ac30055..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/create-a-page.md +++ /dev/null @@ -1,43 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Create a Page - -Add **Markdown or React** files to `src/pages` to create a **standalone page**: - -- `src/pages/index.js` → `localhost:3000/` -- `src/pages/foo.md` → `localhost:3000/foo` -- `src/pages/foo/bar.js` → `localhost:3000/foo/bar` - -## Create your first React Page - -Create a file at `src/pages/my-react-page.js`: - -```jsx title="src/pages/my-react-page.js" -import React from 'react'; -import Layout from '@theme/Layout'; - -export default function MyReactPage() { - return ( - -

My React page

-

This is a React page

- - ); -} -``` - -A new page is now available at [http://localhost:3000/my-react-page](http://localhost:3000/my-react-page). - -## Create your first Markdown Page - -Create a file at `src/pages/my-markdown-page.md`: - -```mdx title="src/pages/my-markdown-page.md" -# My Markdown page - -This is a Markdown page -``` - -A new page is now available at [http://localhost:3000/my-markdown-page](http://localhost:3000/my-markdown-page). diff --git a/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md b/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md deleted file mode 100644 index 1c50ee063ef..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/deploy-your-site.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -sidebar_position: 5 ---- - -# Deploy your site - -Docusaurus is a **static-site-generator** (also called **[Jamstack](https://jamstack.org/)**). - -It builds your site as simple **static HTML, JavaScript and CSS files**. - -## Build your site - -Build your site **for production**: - -```bash -npm run build -``` - -The static files are generated in the `build` folder. - -## Deploy your site - -Test your production build locally: - -```bash -npm run serve -``` - -The `build` folder is now served at [http://localhost:3000/](http://localhost:3000/). - -You can now deploy the `build` folder **almost anywhere** easily, **for free** or very small cost (read the **[Deployment Guide](https://docusaurus.io/docs/deployment)**). diff --git a/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx b/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx deleted file mode 100644 index 0337f34d6a5..00000000000 --- a/docs/my-website/src/pages/tutorial-basics/markdown-features.mdx +++ /dev/null @@ -1,150 +0,0 @@ ---- -sidebar_position: 4 ---- - -# Markdown Features - -Docusaurus supports **[Markdown](https://daringfireball.net/projects/markdown/syntax)** and a few **additional features**. - -## Front Matter - -Markdown documents have metadata at the top called [Front Matter](https://jekyllrb.com/docs/front-matter/): - -```text title="my-doc.md" -// highlight-start ---- -id: my-doc-id -title: My document title -description: My document description -slug: /my-custom-url ---- -// highlight-end - -## Markdown heading - -Markdown text with [links](./hello.md) -``` - -## Links - -Regular Markdown links are supported, using url paths or relative file paths. - -```md -Let's see how to [Create a page](/create-a-page). -``` - -```md -Let's see how to [Create a page](./create-a-page.md). -``` - -**Result:** Let's see how to [Create a page](./create-a-page.md). - -## Images - -Regular Markdown images are supported. - -You can use absolute paths to reference images in the static directory (`static/img/docusaurus.png`): - -```md -![Docusaurus logo](/img/docusaurus.png) -``` - -![Docusaurus logo](/img/docusaurus.png) - -You can reference images relative to the current file as well. This is particularly useful to colocate images close to the Markdown files using them: - -```md -![Docusaurus logo](./img/docusaurus.png) -``` - -## Code Blocks - -Markdown code blocks are supported with Syntax highlighting. - - ```jsx title="src/components/HelloDocusaurus.js" - function HelloDocusaurus() { - return ( -

Hello, Docusaurus!

- ) - } - ``` - -```jsx title="src/components/HelloDocusaurus.js" -function HelloDocusaurus() { - return

Hello, Docusaurus!

; -} -``` - -## Admonitions - -Docusaurus has a special syntax to create admonitions and callouts: - - :::tip My tip - - Use this awesome feature option - - ::: - - :::danger Take care - - This action is dangerous - - ::: - -:::tip My tip - -Use this awesome feature option - -::: - -:::danger Take care - -This action is dangerous - -::: - -## MDX and React Components - -[MDX](https://mdxjs.com/) can make your documentation more **interactive** and allows using any **React components inside Markdown**: - -```jsx -export const Highlight = ({children, color}) => ( - { - alert(`You clicked the color ${color} with label ${children}`) - }}> - {children} - -); - -This is Docusaurus green ! - -This is Facebook blue ! -``` - -export const Highlight = ({children, color}) => ( - { - alert(`You clicked the color ${color} with label ${children}`); - }}> - {children} - -); - -This is Docusaurus green ! - -This is Facebook blue ! diff --git a/docs/my-website/src/pages/tutorial-extras/_category_.json b/docs/my-website/src/pages/tutorial-extras/_category_.json deleted file mode 100644 index a8ffcc19300..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/_category_.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "label": "Tutorial - Extras", - "position": 3, - "link": { - "type": "generated-index" - } -} diff --git a/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png b/docs/my-website/src/pages/tutorial-extras/img/docsVersionDropdown.png deleted file mode 100644 index 97e4164618b5f8beda34cfa699720aba0ad2e342..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25427 zcmXte1yoes_ckHYAgy#tNK1DKBBcTn3PU5^T}n!qfaD-4ozfv4LwDEEJq$50_3{4x z>pN@insx5o``P<>PR`sD{a#y*n1Gf50|SFt{jJJJ3=B;7$BQ2i`|(aulU?)U*ArVs zEkz8BxRInHAp)8nI>5=Qj|{SgKRHpY8Ry*F2n1^VBGL?Y2BGzx`!tfBuaC=?of zbp?T3T_F&N$J!O-3J!-uAdp9^hx>=e$CsB7C=`18SZ;0}9^jW37uVO<=jZ2lcXu$@ zJsO3CUO~?u%jxN3Xeb0~W^VNu>-zc%jYJ_3NaW)Og*rVsy}P|ZAyHRQ=>7dY5`lPt zBOb#d9uO!r^6>ERF~*}E?CuV73AuO-adQoSc(}f~eKdXqKq64r*Ec7}r}qyJ7w4C& zYnwMWH~06jqoX6}6$F7oAQAA>v$K`84HOb_2fMqxfLvZ)Jm!ypKhlC99vsjyFhih^ zw5~26sa{^4o}S)ZUq8CfFD$QZY~RD-k7(-~+Y5^;Xe9d4YHDVFW_Dp}dhY!E;t~Sc z-`_twJHLiPPmYftdEeaJot~XuLN5Ok;SP3xcYk(%{;1g9?cL4o&HBdH!NCE4sP5eS z5)5{?w7d>Sz@gXBqvPX;d)V3e*~!Vt`NbpN`QF~%>G8?k?d{p=+05MH^2++^>gL7y z`OWR^!qO_h+;V4U=ltx9H&l0NdF}M{WO-%d{NfymLh?uGFRreeSy+L=;K`|3Bnl0M zUM>D-bGEXv<>loyv#@k=dAYW}1%W`P<`!PiGcK&G-`-w7>aw=6xwN*)z{qlNbg;3t z^O)Pi!#xywEfk@@yuK+QDEwCaUH{;SoPy%*&Fy2_>@T??kjrXND+-B>Ysz{4{Q2bO zytdB!)SqeR7Z*b#V`wz;Q9sbwBsm#*a%;Z0xa6Pm3dtYF3Ne7}oV>>#H$FLyfFpTc z@fjI^X>4kV`VsTHpy&bqaD992>*x36$&m_u8MOgAKnr zix1C^4Kv*>^8IV-8_jZkZSn%yscddBFqkpaRTTAnS5A$!9KdgBseck^JSIQS`wRWHIZ&85f`i++% z68t8XiOy$@M67#u+Xi6bxpuq+`HWa<2?N@OcnUhX?Fa0ucuMgFJFc-@1+=(NlQ>>F zRDxG-|GOh}P`zp=#(X0xY7b!pCjittaWhLjHXBB#-Po`?sO81ZebXXp;sg3B6U;yT z7ltQRr)1+s9JQ^V!592xtqynFYr$yy)8J4=_Fovpb*N%#EBk3~TNxng@wp@YN7Lqp zrjUU+o-9X*B{;#FfWF+8xsS-jI`K=*Kw`Xfb@RSO_U)QsNHa<|mWk9yQ?OwtR*_xq zmD=jg&|q#_bdPo=j-*xO@t@Lx#ApL+J`iqWlGkq6;4fv@4RCK_O9tc(xtrrh=-c5R z69GA#i8S&gK?|;>DM8&0G0qF?C*`-kOcVP3)1oi%f47pC4CS=HBdpf`E)$Hno3D*LM*Mxsl@|fX(Xf%aXWP!}X9^S#Vk`h=79=r%L^l^YWXw_fRl+4teQ3x9_*k%}TKmP12k&)U zMNC;?1$T%`tp^#EZUUbydm4SOs@A)}3PP>tiL3j_W06pb3vSHu)DJU-0m)ledRGV0 zJ|rcZ1U@_hCyPE6_-wiimvjR3t);y*Qdi`BKX*PP29RBAsD8W-^u0fLrRq zwCLWC=t#&Nb(JimFikS-+jq}=-klKJuPf|#4pY8f?a%e6U2$1>GPfs~QJLAlns4;O zgz6*qdCCdKNu92Gtjo^ob%T4S7Qi-4NMGg1!+m0yH08I3TITyT6-g}m=2u_lckZ^e zq;^$v+pjrNbh#BOPdii=sJ1bq8F?sZTJcTI5o-P0V#bJPYY`?awnv-41^CJh$BpLP z@aNtrc;&0^lO>O1M4Is=8YA9!yo9_AI^mA7`Aw!579-QByLL>P$1D=@r}QPn38D;% zpBWvkXSRS?b^4Pq$yjf%7Lcq#0#b>rLc!^-G|4-BD83fHp~~6CQ_U~u{@(n0go&P^ zDHT6>h=0KJ)xPF^Wh5@tUEbM@gb&7vU*9YcX;|;ESv3bj^6HmWbTMt;Zj&y(k;?)$ z!J2pIQeCULGqRb5%F}d?EV$v(x+Zqs7+Bj<=5FIW5H^? z1(+h@*b0z+BK^~jWy5DgMK&%&%93L?Zf|KQ%UaTMX@IwfuOw_Jnn?~71naulqtvrM zCrF)bGcGsZVHx6K%gUR%o`btyOIb@);w*? z0002^Q&|A-)1GGX(5lYp#|Rrzxbtv$Z=Yht;8I!nB~-^7QUe4_dcuTfjZzN&*WCjy z{r9Sr^dv=I%5Td#cFz>iZ_RSAK?IMTz<%#W)!YSnmft3Nlq~(I`{`Uk-Wm83Cik$W zA>ZEh#UqV*jtmtV`p(`VsJb>H>??z9lR#V(`9^UEGvTix4$!-_w1?L1)oZ^W!E0k* zCB7_q(G~1Q3x6mPdH1`hse+Jq;+?Cw?F&D*LQhHFoFJdd@$J@~sOg%)cymn7a4znI zCjvkBKBOSb2*i~|Qom$yT*r{rc!0nX+M`4zPT|h~`eXtS!4FPTH0(?%$=fr9Tr*nb z(TR6>{L$7k2WHlqIT4J->W-mYgM)ac(R(z56AY2Kiex&W>I$p+&x#bMNS&|p@eWOy zGD7es5=6U#uG^J26B@SERc=i`I+l4_*`E_OxW=&=4|rH=p;$GB!%As!i|~ypyq`M{ zX5L!TI*|QR-pt7Y$irT5b=w9KcWKG5oX;$>v|GNckJ5XfdZ#KHirMyigcqZ9UvabrO{ z8rDp1z0Fr%{{|@&ZFm^_46S#?HL)}=bp45eUvA1gf(mODfe+cGcF$6-ZaI;NvMu;v zcbHrkC+lE z7RwO#m?)*hw^|}s-z?wPDEMJ2%Ne3)j0Dnt?e(@i?bf<+s^BM?g^S5YKU~rg%aeTl zJf0#GyUY|~Y;9SV_?#uV9<{xsFjl^YeW{@1$61GkUgc9Xv6cL@uB^M?d@o7H zHKV^XV(Q|Q%Geas3dw$Jn&atPqxYB>>Ii<#Zv+@N8GYs#vrxfbS_%zJ#18<+55b3yBCV#A}|5J8EAtdUd zn{=~8r&YaM_GB^l@6D_xfSvmbrbJP^&RZ{np(I^~Osf9d>=xz;@EnY?(Egg`%_&Vt zJA2@>$gsV@XFKh@>0z#d4B>B{^W%bCgT;)f6R|f%yK=!bN2w`BOC_5VHz(Q+!7ID^ zl#oQ>nDe2!w&7tLJ8#8wzN%$7@_>{Hh2xdID<0$kb*>G$17$S3grFXLJQ>4!n!>-B zn>~N~Ri%vU@ccS?y8BTR)1#fe2q zlqzp;&z9I1lrZ*4NJn00*0|iPY)Z0d$3NTJ9HNQ+?JI;37?VSbqMkdoqyCsG=yp1B z-3WO8>t^=Fj^?PT?(-0dZ8y_FL2Z9`D!m-7Dgr7r>V~Rm8RQ@w>_PrbFo$N_#jGzx zKC&6u^^M`8cdv1&AJ-O}jSqCR94J?FnYw!JN3(k7cejfuS`7-j*t4GNaKH@|kkrB_uY?<%tF27r;kVj(nzxph1JsFr z#*%R0;+(NAevpx|F8|sz9}SI%^z@E#+KR{}h1fyNXo6z$e*+nNx|qKR4DoCl0?&Q@ zs8_MHOw&gA$VQz4yIo@Zg{!M@m9v_4{_V!x@I>5ZaG$rcOvUm9O0DW9tR>#oyg@l8O!7%+a(wcN zU}SdcI3?TjNeNXmMJ!GUx@tFbszrKU5?ewMLA zJ)^SSUMDXb)yO8<*A&?2bBN&NEk{+9q~*w%k^+OUs)b@Fs#!)#9E-|}*u zWAn}H61Uy!41$}d1d44D;guxTx^kD367XWM%5Dea)6$5&n;))D;D^r~G=m$CqS7L! zmLX|kejC<`PU-rS#;n2Y0*4;&?(ROps&9eVSDoY%G@-4kyG5AX|Fu&1M5Gm0(-Z6v%1@fS9$`LGCB zlH8i;1e!(dUd#1c@G(-^QedB)$yJ~Yke{h3 z$#|*Md8c7)??v!utM3QJT7mN@DE%_r@BYhvf))3qME|n>shVP(03fO0{Iye<3)wv9 zoYDZ$wDak&n*QW`-s6KKDk5X1OQ_ramOCv4gjh1}jy%9GX!s!hq`NW)&%o9y+YrmT z+u!YGVhHBA*{|c;^}Xg)elpF+dMcpHNALqheHQIX<8J#~;Ah^+Dw~L#CynKWfTWCu zCEbY3ybkQ225nUxd$i6(3SN^?}z{r>!_8$YiwX~LE`rzuT=q!8;h{UbMWDGL@VpWm; zZtr3$23sHj`&Co0No!R|5#Vt7{9}j|TwplkHdT=aUeQ*;9XQ2uW1WUTbA%kHwMR|UUq0xTEetKps9KmNYAS5aY+L31z8w-k=r7r5hSK=6A!^nU z8C>n~S?X}?D5`5c5&2wA0cxo;KgFAi4N2T%LF4fWoMQ=CTo>=1mjvBvW;|iPUB>xW z?K5>~6VIpJYo28I)EFl&7dAhqrB6A-(e-)leVf;X*$GA~eVokc6j+rvRq{{fZth{*dW0`N_!2w6Ll9fV z{aJuKFd-zavy0~QH9hD;H%Q(_Zn7nY>AkaeKuL7Q@G02wArkDPH53Qg5JGaH{_ehi z35yHf_=pB1wY&Ak3EZ-^Ml}MxJh6d_Z}jDN7RTDy68ton&H$4=>#b4w904+;t6CcZ zMtV{hLGR06a?g$sZA#7RlKPF4Bqk=}`#oc=#~O;oUX7hbb^NY3f2Nin?(&;E?zVkm zN}OTyV%mP6T5(MT-syZn(K?c9sk)z$K0AQvvk9#%4%)evu)aOXbB;x-*G5ljx|A;$ zZmCV}y(IS$SYPVS%g#3~I9lE#erA)7BgOkZC}~2)7B_BBStEVtr1+0nv{(A%zhmjT zsE;^zwY5(ZCyf%wwr*SJyK_?Gv_p!Oc-8$W?a03T_8q zb=XB6)**gF9AoG(=dN9-4yO7)FI}g2!0UFua`5ASTp*W2K#(fpZHPv2}6 zuI3YRPb*T9uhpKUc zPNT}NbGpABC}F~2UYA?vuN z*c2)mWKvZn<+PL%-Oq3lAhrw_j}+<$Tfvgoo)dRh((_MP7Iz=PwI|1>aObW5-b8qW zI@O0@c{EbVHN5a6k}i4y2?Jh~=Jd-MZnv)h^T1;2CAllrl%EHm`1{XUiW<7g+6{XS z&hVyh5*+TiVaO)+4PE3HcnsJajGx>gwo1EcWg^*Rn0l!#MVM%(Ywui_UjM8Dgspk@ z4`gne14lZ*`698%UOOx^(v_~kQiYj`WkY>(f5KDC5I{-Wi!KoINK)H^9m|SUliD=d zE;N>?`0x*{61(==UBrN}mpsdhOZ2N~I>oQ1avz|nvyfQQW_R6VAnn;IzqlxDB)0_Zw_Csf#5sdmb4LBwIyBk zv$NL*@acUJc4`FtA^-PzoHR zKXm{;9xP9kWW6MEPYuCeDqX@UiY(8GShF|L{-)R4_acdmp+&W~4nBxde z;pI70##wwE$hfIrpx@VQ`Yc>|xSP$S8~WoVKTg5Z*KMWE)Yp>$m>ZoNQ(u!z-#`mL z1jJZHKZ}Tc5Ap^(*KIg6ol~wx)s~So91kdWaF2c{?F58%EDiT9uV&xYWvS{aFS{hE zg--eu{(>bL!0h)=md^{aR(APus_Mr}+}|%Rb(>B&dHn3fw9>d3rkDH6x0-@)^Dkwj zjb75;-8>7gmW&$y_4x~rPX!&!>l3d<-kfo+g{PIl%s;UQ)Y+u z4&z}r;Sd{hco!{2a3}F*4CAcydj7`#V0_iRg%G&NxtQpm=(5VbGfiRW^NoBJ1rPE# zzYktZRk7>`{fdU((V`a+T{&n=cnr4LaS!S|hDOtXWb>_e-LwH+@FmdGw>6+B9J6~} zcBaNb(<-c6&|ghc-%o3xG(Op-q&pXd1CfV zgPNdKX~vGy-LS;4Q=161sLAoMaXGG7weBcT%KmWHZ${+6bC6yehCjqK36LdH>fR!{ z>Xe}eUaWsRp8U1&?E`K@0*oHDY-p{^+u0T&$b)J}|G6C(lSRuN&WgUd(rH=0h9hUz zj|U@1UmNWdbn)SLk^KR_nRxbB`hNKP>?@ocdEL;;1l||Q0{~Zx5N5FT_ z8{|xM9~@McIdv|?#WPK>1b&f`?=bvMO>?(;W^}|VZ|%*&C_rsnS5&E~%`>$1I#;~* zn=Wx?omuI3X^Q4D$;n_~HEv`6`Rwl7C)iTwB5O~BB+$PgQTGE~V(6h;78q+*a8tK* zi)1P_7BY;9ea2|o@l#u>z4b#X%;a|nTq^l*V({7P;k z=t-%I--DL{uv#dVtaWg|q`lNci7#N7sC(@vBesWbHEY@Gb4`DozcU20N<=vl;-%s5 z!WzFm74mydG1Hjwdk!c_6!|q+Noz5>DrCZ!jSQ+Yjti$3pBqeRl}Wv|eimpd!GOY~ zDw@@tGZHFbmVLNc^ilgjPQ1os7*AOkb2*LRb{O-+C97i_n z2I@>^O)#WwMhxr4s;^U&se%2V#g)$UMXcXHU)C<7ih`meC7t?9h6U9|gRL%vjBW=4 zyJ(KaCRlNg`fO6a(x7h==WMvQG|_Skr4D&0<8t`N`#*Y0lJn{f4xjR5Q%h*qiJ!9l z{{3xuZ%nm38N+XqLO_y}X{{=Z1sg+iy?Wk0(xmzIV8KVwj}M}&csjjc2tOdzyInRf zj&mB~+`^C>=hnyxW|Ah^U8Pcl0}jx|K^QWjuTpX%S?_Y({asp@tk2!qmNiJscA|3v`}jyo*ALZ(Rr*ar91T`}p~N<62j4RJ|PDBQI3t8Cdh) z?R$X25f31}sp@&0jG5+in zs$WmohuauhuK4uZ1iNJsy2T@EuDDT=`&$LT=jKS^o}44OK5cA$zAzZq&gS)a(=xC7 zC(q}(#ncl6@1^p;YG?lVnJ)t^7Ky53%ZtMKP6FKlx|zSaeDQD~}Xbf@cZU>-AI+P+4hN52dWFDA$qg=0!5}U9qLoblC z?2V$GDKb=Lv@me&d%DST)ouSOrEAoGtLxcGg1~Kmzbq?}YUf=NjR9D?F9<}N_ZiNa zZhdC>2_z-iy!(9g9{n11i3|~!hxmAYX6z9olmC=&YcsiKI;&XK#&iSd&6&{u1@Hd^ z&}sU>_G+y}Gi-8`-k*Exr{a$>MNGj_u%u$;s_fOjknwYR-qt1G|mi}nQ%CB|0Vp`=0tc2y(3 zJ}XmzSQQ~(SfJW-|mT1TaDmxNCml#nWVyhIvX z5(>8xARd*joOU-U;Dfj+E+nUJC25bpe>!0L^f@BXZEW73UVfjT$=FTfw8u@h@$hDQ zVua*ub@?Dlc%%H2Kt+bYLb>$(@roZ+vrM&so0RO(eTY12?=Hk4*qI39-0yU@%aQU) zh(=Pxi6yISqhKQ$i^SEeyiioo-1GNY25sM+qoj*Y3&qp^8_)87sMwbecGG~;>|9TP zREo(Axioj6Z+vp*b2~Yp&YghcPwB1H+J6C`1#2tPkLCkZ%eJSah9>34C6}Wx52PW# z^-a1fn~bY&PC$SE9!mvprG5JAMZ8#PQ1utYB%g4fm*YwmC=|j!Ynky<|7ZL;!BWr3 zFawY3dr};&T$Ip3YmV+)De<*8`l~v0VwiNIPNf3|&X$o&6@|n6LRM@CjYQR1 zWBH=K@#i3!;27}0=N!39tP9ZWSn8M>14nC%WHmBMuFJAk%Lb z3uC1S9h$5}_+BVizP47z7mQl9&0QY+JB+^dI{s zw`OaYK6by8i7`3&)Phx%c((j7B1YUWiF2MMqu4sv*rJ!i;BLj(fq}XbxPz*4fPY?O z@*Ky#cmpT^|NpZ9uUqz`68dgR9jtzXj=}e&QRIn}pQRT9PLxt|PUrc*i*0b!XrG!5 zn0}>27K&TEtQcrzD<@JD6Z~^YE+@bp^w7O54P0!hf0Y2>E)Q-^2GDnxCg+6##J=z7 z@ngMS&`rDgl6d+JcSuka%Z?(3I;F~=S0|1#j5>jeKEQlh=sBqfv!hBN|;yTWLomu=my`^LYikzJ(>0epsIY)kU18UXtB-3pcSlnHT_D|^@nAOvSZ&U8G z2j{}BU*x=`J<)n1d{C?*L9G7(UY zOa>7`PWnsf0_A36hyo=b^S{8-brz>TuX+X?u5rOaa-i+Qwt#GO{msTqNOcGW+e>Es zB9jlrN(d>)QU5{6)p@F-7=X4^mJ_o0PmD`XJxKX3yEPtUxGs`3c=nmm=R})T1N{pn z-4`5~hgSH{OLb&X7JJ{Kc!m~cw^Px|bf;E_^&_m2-RyF$>hpwb^&OK2x<&5mZY$DQ zM*Ba9X2yg~f2CrRi%7#Gmj8ToW&RX3woB;vaQS~RStNrN_ip=L(D5O`5ARa1*tbl$ zz*z9~cch#eZ(SfXecVU8>@a)YoW^a+0f3~j0Y?^-$NJeZx)){fSvT?~Oz zr|rs5)}M)5nL!oe|LIs_Tje3%Izv_8s~up;gZHa$tJ2apK4+*%@ezaqN}(Z)Knf?w z50}vMb<0<55q_7mTNOQDi&W|)caK!E^KS2+JE#Q+@^xmQv>inXC5o`mvE&$TOke$B zV8GSwhlTR2rzJ#_;)bk${WP%Ih)i=EYN8{o&z8%2I_q?VymrtR;v$zLkjrg{wpYbS zvAcy#5)@jAvZp4FuHHU2=>%7yAaF;Pr;R4Fs{JD~J3=fZ1&XUJg-%A~!KmHC3n)>YIEi}NEb z%--g1St?_*DOh+gnZHtmEkxs@isI}eRrc0wU8l;2b@mCiAM#Nn997Q+LV*)|qbtKQkb_f0o-p5pdd)@GMF*DshM3Aa+3F#`qRIwJ0hm)o|YEL#OaBEakx*CoYj z!aPt=uH3>5{Lo)X0vnhRQ)s3fJD8{|J(JOpEw+)Rk z`bt&Qmfn=@fB#v0H(jRr&%qMgqOh#^u@wR@511#rdFm|rRDW^uR0I;SFNFONvL|T< zNgTUA$F0a)aQgw8fuB6MGPB@qT?~BCYk5+Jsf=?}Mb;HKNTkLenT0K8t8|H}D?|hE zSgX!{rJBv{`q@9kgrWLKN$Lc=(eX|?lLDj zTIgDs2{@)$i(H$~)t&t0ljddg!CF6;h;#+vfsiOq1m6z-@3HjZf9Cwjssl8*? z-Zk;h*SQd?Jne_EnSeuFHFb<4o#^De>LcvXXN-SWl?t8{*wYg3myaD#!ASmyRX(M* zGTP9W!pDwsi#ZmX__)rLPoItw3NlJ2we~Weclgdr7?3%+JE=SOCt;iGP}}vJ5Q|LG zVyV6tvP?5JtW=tF&6vZPw&HPWnzz1x|7JWQiR85>W`0|GOLyooBAJSsXr;fTClQ*2 zaK)sev-vb*PP9gBV5`_Qo%^@(nz4=7wneRMzW!+lzgV`U{S>?Un=WkYC)GrP*^Co~ z39gtoderj4l0kRRPB`Ahk_XC*5YRAEO&?q0Mzru!IeuE^lBSp;^j8_6-!y50K|n_p zGMdRWFh-Fi>Ry&?gYb(4RdA{FOqob;0q^4FiX*<}mB;zWot5?G&X7RqtC)_A4|jTu z$#`}>b~R$z#yqsMjRktG(!I2WS~hnaPgt1B%D#`8tL9}l{0BaIb*@{Pzt#{=K}Oe* zDAsQ#vX=-a{P_Eyl10+;FIVppTs>K45GY321_I8QO(l>aZ1$65njm1IL>Tmd^bv>K zqvaOE2UgLp-Yu%rF$JfIMhMuRr(^h3Hp`{LBoH54u5@YGjy6Wg?Q*O?XEIX6kMCO~ z<_kZcb1u98AU{a8r7g=xIgs_PH3)hJ5I+6utGV-%RP@*Qi)z02$Wuo9%2dn$3FhdS z;i52o@P_mdzh~c5s^ah~8Ps7Wp+76`e#%y5agtQuPd3{4@zh;+PJ;Ul(o51qE_WV^ zg+~a_eJ|*Xi=4jabrA&e^&&@I6=VSbgQoPeA2W5wnF#LY-O>}Ljj#`MCRMaV%vO{76cz-Og(S_6~uR>qnR(*x+nLISCR#;o3%W_6?D!w;_CpEp6{@(I+A~0_7 zs}lPdr=NoC&$L2h;r!KHMBq)8eU7#yV&?{?? z=4x^BMDRXs3k2G`S|TGIzZ0Hg;o-%T^9GFBO*20Lb>W?krt$`*_Y)pIqLTXjE~di< ziI$JBW{M?JgMOp7XK0RqD!` zyjnzWp^?d+&R3;V!S}YBsE3^$ov%4ipg*$x>0&cLpey(^IE*D!A^->G&P+M7+J2(; zwd>Ep{Zo-~HYh#S%R%s38W8{Ca=WoD??Y3{$m(9%xV*`*LEmoP1$uIW>TgrB$+onv z_ndvbMOIqVFhw~TrM%u2A6A4v!m5V5;SK21dr|_++u|ReV)&#sK6$=&(H*ZZXM7U< z=e@Z}9GCKoq)cAQ9euu8+|}amPkIa3BNZHT6d18a1P&$d5_02Ht2I0xoGDxi-;5;j0tI=XFRNl62_x%#|RTOCW zg*`>@ux)y<;|r##9cIl^Q&4#~Z3CkHHz`X=;xCJy_@caXbk+{w{=u4_bgn+6>EKRa z8dA{~?4*L&vu;0?5LGS{cbn;+@q!-7usGB$?e_1K0#gE|Ot9ixD#X(4>uu)f#}~A3 z3@nGY`HD_hpAqWw8U%*?yVSuzvJm;5G+nq@Cd+=}W!n*06lvdQCuXal{9Xs<5I5oC zcw%nh=Wg?~Ugk@T1@^y}Np7w%vxB-A9tdKDt{<)FX^ubm$7SZacAr-%L-a1JwG)#C1c0gU_I^Cd_qciW@*(2ezbRpD6!<$ zQ+C*RGs|w;)ZO`^revsDl);H7f(3E%K@i2Y%eE!3cq&}mnmjtQ*Z=hEWe2W_A^XH?Nys^bJZp5h>K5an>5p6yjNY zREWvikLx;$(K_`V*R=<8<|J@62`31~=7iCV$p6c%Lg1YAc$h-uj ziA#pcUoF0HIj*$$+!IpLE!H*6%e?c8aHZ~W{8>f@QlFmqcJUBtER_3}jheE>hx}mv zf%%k^5;hsmrzrQC;sDn(d(nBjd1K!gR*&*-DQ4;zv;)vaatjg36nGZ?Rq_l;c6lQA zQhH0eWpKygvHd1%l_?G78|(|eJ53Tsg#N4Hvjo0QDebJQL;DKH#&_8b>p%_AdE^@3 zLP(ASqIYgP6n3POQ=*_HPw&ScHtu&nQK-?0+ z8>8|df?xb$oR$yQ8MoZfbQyr0elR$(MT?`-AAlb&Ga4F{{$^zoyi|S#Y2?CZrv_8g zaK5GIo1kiS5{V~y@0UpiT9TI|Vx*t!eaK9kRthIgdFvr#q?-1&t(a;pT=yrB*xZmb zYw8R5P*fjZoZoV$hSYocS7&0+G_-lb)kFC+Q>p$|lmq`}9KRe3H$HuG_y|Xz*Ykic zBp$CVTqZL0olc9!_rqG86IPu{8Iq!Y?GKoMknsM|jFN<nmkWW$R)0;=-v0xAm_otSVoWlb^RlPVJ7p1U|d^4=E>-zP*-Rmrv6} ze|&GPS7f_&uWb1R`Q&)TSwU~0v1a<`-)o6LgtM9rGA0LiJ@Ue`$XcxSFf)nQC^6NuI4*n18HDDl~3>VPbX+k7zOT>bP zjw?xBP7GAvQDt>BQx!=@sw8)=gBtaH=3ce`T>Xns6feL{J+BW8)Q#=W-7NmHaV*F~ z>UmFhh7MkTGy+xsl^XpR;qG_do8Awha7b-nS4*taqw15O=A{`zjy!fUT4*O~Px9G* z&%KU#?o;#N;>89$=?gplzj3XFNdj^3RMIHRL=~;oyK7Quk=^>0g#CAZ(QGGeUGLU* zWPaROHN4T{eRhQdB8Y!9jcDKvnUVfi)uLU;QxRVsz{0S7@3sEf+Q?Ls|HWY4W83@} zlSXj&#g|UeKk!d^F8}ntYOtDT?R^m4cwFr4JG~o|z8Zm1yM5aW({Yy@f~BU11L!v#Td7eeD4W$>lcjaG!42YE?~f3MI=4r% zoOf_vBji`oQ?lj_PxRf%pt#H=+;A1r#K4^1?Htf{euOeDW4^2m#LA%gz+PfcvYKB@ z{l5(10Q&Plb>;K9_`Jn-xRvcD^qdB-b$9yeMaHX`lv9~f(0}6fFn#1NHFDl)U4XX~ zltY}5+&}s?L_h~eET8)X6I%nfweCW?o!6vD{DiG}w?pr%+YfFCFf-a6yId6Ra|pe; zDl_g&Cv!gUMl0Z_t9nh5KE)coN>{ zg&1(j`%gkFBL`Uj=dI12!|rM*w?!U{waw}fJ_H(zB}-9=p|eJ;sfV<_S)YhAe7eDS z{-N^pB#iLATr#NLu{RO!>S;pwW=9=;trCin9igtoOlB&izD{7ASKh z(CzzkugUVut^bL;3>2f~%R9WEhM%m4uk8P(3g_CM>~SJy%}G!J2{hm1T1XXM;$Nx< zvJ>kKg7*&8803!xLR5KkS8}@!TpVFYhM@Q4tv7{NMwN?-8Ku8G-eOxwZUgt(3=6ku z31x;jRmhmiv^Xlb2w?7W5OlqdT#XaE5q-_MGSi%fF7Ds>Ic$5Otyo1~V#Yyo$>HZh zPZe}g8O%F1w+%SQX;*l^WxmvUQ&N5%JYQ;hfA9Y5s8Xx?TASV~=_EpR32`iLB7uC4Lj=X$lBnh3I zAtk%flc?{lm>QjJhL6FP*IzJugn z5FL63L);PtTf0G#iPK0T&aY7OESEL@kG;N>SRc>->6$NM z2j0(*rwMhfDRh0gf$lx8dvfpYx#D2>k7XT8!~5PqGifS5zl^X|?z;dW>t6;)d<#^U zqpau3c!`tBk%yTSPM>VZLXi$PMqeV1LgvwnFtkPxPgjRfvVg7ax0Xr^R;&%IPtWN` zA5SCheRx72%iHFEbeJaExY1ElK+?^&?iS>TAUdMBcMr@A%n{(^2RH+ud)j7?B;I^^ z7rkfli|k(%_b%e@w{>p57WU-$O{YdI+TV+mby<|-#*lt?XmB#+(b(wfKEBm`AY(B} zAZnYZD|DDnpBb>>Q7ZEq95BDq z&uh}x=%dYlNY1S?M_&pI&)5JYVBPFYqUc-8!Vem&)86BebiW?QAtFDVy}0NH26r_( zC_^CO?cMW|=e_!Nd;`}}wIe#2rjbs;ifve-VvB7)GI_S+Nsq$S5JY$8#w^grTZsOb zUyoAYclwpn;7>Ci@(v@DI(;8$4<&tHXlW*;hWslB|D-5>6-zKX+2bVjkSQ8?!9MgK zl=N~I!}?@~Kx<^NrI^q0srRS28Q~9lflYBLXVmE~H-TOQPE~(*4@#$PheP8^EAU}f zm+WSP;g*ei&p2L;l@4F7HzwvVyZLh&&an%n~F2LIKZGsoGGdXNS^^gkCKD8wC{ zOn978*5SMH1Cf!Pil1ixa+!!Ro4xRSy)@zYLPs7Fyinlr`RnQAu(hV9V3Uz}C;^ z-~Y9jxm+%8+u;v_3xQt^9}E{~dg`y&k_IL-boMLUMr9GA>}o>^!B)g*B8rgz=En8c zEK9pm`|y*X?2q_#wSx_BP5}w*8X6!2tqcCUtG(2FdmF>*`x6R~l!xbak@?Q#VXxG=k(YY-43Z+D2$B08B6(u7e=DG~ z*%5MY)s?k;<$!wd{Mz})9SNS2BBclkhNAYGR=Yc9eI@Gtv!DgL3xps?>l1#V*6K|I z@g6biLi{Ynk8TBO%+c=d^WA~VrcEsG)?TmrPdXwVR*O*orI~)IESKLQEv<$euHRV0 zUPn>T+x>w-@sS`pGlN?9>_rh7SfhqmoWUbl!t=cqsYqT!VHZ?eccRCm5S-9?!v&=- z+Jeh%?!&){ecKh#*;pOrlRLHF|528F&6}$#V0U~vK(#a_$BEQ`{zWkUKYenVJE9>7;rk|eSgj=7Uhnz3xm0Qy^^Hui9 zY7}x$DkL_sWncCgDbupk5VZMn-;o*FQ1Mt z2U`xQCp(2}Bg4`+`iC%H9Tf4sY*L~$W{*be^*Y%4MZV8(`SR)b@`qbsSWL5$uZ%GF zjM=n+$!a%_F=CE3MuW3+McnFQ1MtXU-E6p(YrX)pV>Dqtp-+cnY_W zd6t8G6`!Bvka-in3^?bveED>Ixf3Gl)fQG*Y`aenBlz0qAXALrc|ep17;{X9@R-8v zbs8||w|x0@eEHTEGPjTjRUj%~kJ_aIh4Cph9?uqYMFN32jbQ<|1u4J2l3al~zvauP z$SrpD^VHWJ3&Q$?NSEJQ}*?%ctYZ@oc|`spkf7Fia_oS2yFCcrly1 z1B*s!8Iz$^^q*A|3`=7QzC4t=pD)K`zthg^Ep3E}5G|MBU&RLp#o|IPI}ghR$q+u@ zJc5{|sde-oO!?>VTH%FCKcI-(x=FE!a+1wn)^OP3S z(e#KhTllu^uAeWD&p01Gr5^Y5;c%fFa$K72}j&d--OdYuktp4cwI{afY9wWwjpF#aIES^M$8mK{XJxHGf9|=N=EJAbe+>37@0iVs&W_;h*kQQ?1r-@eW+XFHl4c>?#k=+r=%NW>Ns-Y9A@!k)T?e6*WHg!^ zZ*0Y^BoAG^SUXT#3*y5Xg0uru4D^-_w7Ja<7f}O-7K+riTwU5)p$~=j{lfnLnTbiJ ztqb?QEjgM@GJobA=9_=M^Pe-{{NpBw-~L>F?&eA9|5hLVo9&$cPoK+Qju$*3*X&2z2QXa0Jn?Fjrh&=BsW6$h6(K|%>!6&+!pvWwM{YSE z-2liDar?!20&>3lzSo(znGVlddBXUF`MD5V%%BUKj&q%DB? z?(HOR|MMsL%d7R%4K@2w_Mb<|Q^^Uhgn&XATZ;2|AYPH?##y0*@^LUOfpalPq!6JvF303@uKISoQlV}P z;dN)hq%Sw?ryFYaqwE5Y!yq-CZt6$H z#2>jt`9vS*VVD%krkk(_CHEw{n=AF@X8p8Te_pef?agkSTuDb&SHOk(^L9eyq9lor z*!d1Y5E7ImLI=ua!rZa?6dV^A1}7KA)>ih>xDY`v_jyH+B!yE9gV&ovv`fV)MfWhzOU)&HxmiDL)}Pnx zy8SCjpR-l1*1x;@QGd?Z+JU#FR!L$ZLW}^hTu4yAh@yn@#CC>hw6)NkH2692`O@_X zew2#*_2<$AS*3p3tUs^W8yf!5EHv``gq`TK@^r`*qK;7+j`0vpxpx(Yp5vD$g-eM9 zH6}_iz+3_=Lp3!9T4*(@5+yFCWwqN^Fip$M%(wVx5R#GzQ$J5ljbNE2WqEdanY@g$ zu#n9z9G3g#<^B8jjTQHY4oh$-iHqcKEKeMcz4u4{La%=)7%a6{daG(5?Aa&#PYOXf zh(*(6@=2C8MOG9gPWF`SH10itp@(GrL@D{qK-xH#q@m^9#<5jU(+%Vb85aHSqaLE@AhvVfD_AhL| zf45ltDTva)W|!2{Sm z86>a_1xtQO>^f??ee3bw!=voDab>}uYT0#Y%du9`e(>NYhh83JWevavq&4tvcmd#d z;_(p^-~jm#SBQ@2sfOHC z02lPvx8w_uh2!BT_A)%xW$S;~Ki&T6n&S|1S*MR69`L{Ipy8nczO7)95$-tB%3$2U zd*s~dA7J10>>uCu04Os918r@$0P*WMeK>5jMAh@O1%{n}WWo%C-6V9DbE_=dA^3$v z;=&0(5DPo+ljeOMpEF#a$)zYN0HaVf+J~XyG=CjMy90W5)~h{-pd0i8zCK%x`Yd`n zK(4#{!m{D+`j_%&8Bbr$ID<6}(a6Gy{ft2J7Iu7JKjROc7Z9o;&2Z2{K}W6dJXyxG zWPkS|TMhC-R;OdAAK!qUvB@Mux{Nz{)tT7JFeV`qmK^`4#L|A!aY(Z zaXnwzl^OErpkBLubZKJRdfmO5Co{G%2x?@Qb{mG|qB!qc9iQ|^#ydJrbay9CA>?1f zae%Nz^5qyO>Zb!3wO9aiYuC~eZ@1sF542&fQ0zr}DnZvt-Ej2^*wM>@Xpn4X&Ax6x zj^3q_y~U4m$C*7o)K3-1wcLetu|!?CmVkU);Bh*Pg)FRWKEN|l}@@xnE+VKi1y@|grKE@d29@hVW94nddvm$4qF@#)iA38?`kMa(2 zYwTE)C8**5;vjk5s9+S_|0@ts!2e0iPma&S#*51^=serm*Vs>^+9ku}GMrO_zSE2N zLeCi)PjsKS-2Lz4)Ht~L7z+a;>_RyPM?`hUC>Rl?t)a7BdVJ2?r|sk+=H#KEGo(#& zZW*p_5X@n?UdWo5=92Q)dx8-r=HGd__BDaOFbg${6W zaB?IT;lI3HZAe>L8kYUhKZR}xNvu)P^hf_V7!U?*tOKbv=?^6{11&C*FmiFa+Qv+@ z7TuBr{1{sGj^3^$5iF%wRu?7}XP1$wRwqA7M_Ee?L)mJ}^v?7{7=|v>|Al>?_axO0 z`)^@RYQE07_w+vJxzGE)=bpS5m=6p#whwX|*Bx~(JGp+^cBp%CA>X@EzGo?k?$@gM@@XA3JdtC;1BMaq#z94|#pA zSblq+=4^r@uwC3NLk-o3i=cwX==$aF$juKEYOkB@LO z7Ru4DiFqxeK}|GB3gE`WD&pP4-20>QyG~EoQ+-|lFE5`t>DzEHBLy#Z9w@1G%48NW z4Fp{9R${JLU#Kz(+d1sDLs(*P8P~=FjiqaTe}ntR0cRE0Paiud(=7|WF6K9%o~&*` zcr_OfXP{w#T_ye($O-!CJ-WlTZ*J}r_{;R(FYiO2PYLk^_T*9^r?R}9cp$nmk)TxE zLLpP%2;{HliSvXw)n`_ot#Y&k@&p^-=P1m7357@`u3-dd{0QX(?jMi&NMt_owo5|3 z*FRbQ1L`B1uw2QBL9`9cGBndP3JQ)x?&0xgGBwP|*TSTH%uha9w%}Mi_NO)kopsCt z;=F-KhpRpVuFnPrE0P2CaLM~C`vWxqiCa z)@^h2N`CV)-;8g%d}i8HJw2X*q-RD2bs6@z0&|KP{-tbg?pOHJ^6z~N!Rd3wLBO$S z^XlB?I}nt%ipoO$T_Fqr@6Ha(vz?t+i7f@Wz?Im3dH=a+dqg1Lo>xfI-hD;v=LtDD zJ1>w&G!Wb}*b)8+tQFA+`M&-sX8b=H*wGowqLyfuX_U}X1aW3DnI#R-NCv%*Pj!=2C7QHA3)eS_FkwD{$YQAhj%#G^mTu*B-j@lfSkj3 z^poc>p?)_aRqt;;}`z4RAb{PNh?NI+sq*GA2=eIP*7E%lh$h$p-J6 zTv%Li*t$ErJGuTGKHrT7KVTg6w+F^JnMHgnlc8X!Y1rF>9YegHyH#;ht;kU+hIMes8y?Bjt{=Q~0N`J=28lA*{@BFxf?_V00KyGLc zZ!t8Y6OU8Fump1KRzYqU7>Rplr7P*iDnO2RteG&496k42uW71pli)@!mDYiGPEYHz zvss;xd*U^jxlu4~T5g*v6i4L3x!SVMHrp{-e}03%PyuZbbs`2@8wA5c6|oD!%H)ON zCa>2XeDX&?-hZL5qGBvYp@(xG@WX>|a8^aDBtJL&%tK{7aX5v}+zO&DBQ4|A>6bG(`TZ# z#t%;m-+#Mn7y>yUeB1c`r%>W+0;pyQN~bEcll z0dO;&0@kxSo^;(a2ZABC$8ooW$?$@v^dd}$sMr?UB)@sI%E<_*!OaUnH>boQzc3I= zChIHVk~evWKeit(Nmd4vNlu>M0^GN@#H<4M9;G?N{~!BNH))$pu}_A84zGYu^bDV0mm14lT~SlmoA^kU z@1T)|%^uvM@w{{OEZPX<+`iEGr-zhaLeBjQTEF##Q7qsqij4$vZMHe8|-k-8PCs6~sXt@<3^0X#ifJ zYmAfRN$PmA!`syV!4tdP4wiQ$JNkIFA5EYwXd7@ti=auhPDut>XRFK8MPGDqE!Rot zOZ7#ldYDe*h{U9xj6|jkl15M9Z)=MwqKDoV1-v>57)+cRO6SNW92t%_ZKebcv*00+ zh{Ar$c=+b=t|9Dvw_bboV3YM`PQFz24}X2U{pq{gt9n?#t!=0TWWvl*ogvb1``_9| z|2e!*?|%R6`=4`JAP%T!iMFo)0<>GRt-rK#D&;&Syo-d}DBJLr`-F##e(Lg)-+Y}rKBaBHumqDMK=C9B_F zbjmb!IpS1`Fy!t_OJe}Be}msy8?CC9{M~t5XJ==f4P zs|jyy6^trzzoPUe!!NF=Q8+RB7aW)HNzUF>+RWv|JxHUZ;3TB!nc-c^)Ct%BSx?@I zC>MIn3WN9hf46=q+e~h^egS%Cv(3$|&0n#Hg&*X`TF?3?Dpd&cCR-X><=ZmswITz)b-g- zsQHweYoeX&QRlMC-_2D;2Rj!&bSyaXBI%OZ;`2$l?=xI=YWu~J>N!LSaX=2^PR_?Y zO6O0|tG!Yf2EzVVIY`oqq>_V`lNlTz;ewUr2KTbx-AMfU)^1L@B(UeDw;(`zj{5M*?krKO|L&2$Sxi)o#+n zncgm~q*C7@`JV5o_kG^C-n>B|3azO3xLkTX&ia-=$o}21SrCi^<^Wntv@SlM$an>| zsxUEcwian+o^b&tE-nx)J^2$<6;@yh;lnd1EW~VYpZq9n|C6^5U-7CH(@X#7XPTLJ zKi@#X$DiK)B%UQazkWRZDxH+?1vv4(uNrsXACLb#o=jh-0d(WE0gBtrrgil9ojoDK z_m)K9vlLl^4G+uu@ggYx$C95n-TZyT_}C6>yz@4jDbEVmnMmZJ5MywiiSwA^Fu%eQ zWFXG-nKDs_J%8z5*AExwS^6KJ9_KAl*}wZSP#@v z4OsJ))wG(nW!uS4AR6$|o6zL@H#G{q^A5Y_P^u?qMx{r5_@EDnVfSSytzg{ky{~EmH3< zISG2j=?e(ZWr7#Mfn|ZYNne@+1LX0zKLi~0!wK_OHn}Rk>r9v7^$>oWr#54tv1AZ-) zPmP)NvCQ*~NGm>gNhhl73+p!(|lwi6D8DHy?kYV`#y z9(4PM4}qQU18+e6RX9}m*R8G9?XB%apuhNr(K7be4KX`82S9; zP1um;k%fPd+aT(Nf@RqS<9$^802Vc2r7hmE1p3(l5n zFN3N47|aLpO=z)8Zz6H2Y@90&ubB^pOwc@K=IgVpe}2B}e%f=3s3;yM=%W7I)%V}@ z?_OC^bCIH2q)~@h_f;g(&wRW;jn7uC0`eCkB(843&A$kU1W=Vh6fSUp0m0IeD1VGb z*`Hzm16P5V@9nGx&H}@YH?LRaVKp$tDK?L6!6%?$+nhQKC(+=6FASA ztfDNRJ5IEOxf#;nQS*Skp3ey70>pQPL|>Qn=U{ucG)W~i?BC7$>2OXh!k_rsEoXbh zNzvXC>8}s_csvuNkM7B9Alf>ME=h|h8wBoDC*IqJMT<$o*}S9y#1W72hhyx&%XmR< zhTJVfKr9)}2V*$i=@bgs|Hb~}&hY5t@CcRiaQ>xf%0ky1#k8m&pZ7qekgLQm2sKi# zn`0q3%8hX8;S#7^irtCd}uAhI4M}>Md9A9L0MApc=UB@7ro?1Tm%E- z`q;l4pz}jSL=vX$qicb^YdI_X`>p8Sqn)#l2%o|1?C^=Y_K|S89RHys=WdWywjn2P z$juTI`#+3#q`FshJiC;Z426ZTa zH4`AX7TeU6Wo1UVPp@_v+stDzHbY}r8ev;%wY8W0YRjQpkAvwRkNDXqe;i9&0_d*W z{@sxkFg+Y@5AdPDbt&61nZH~))@PP=!`{!ShA-6$Lx_V0#p%#reg`w<}`0l9$Q+4@@8d9r^X0tj&>w3wavvd2eQAFk%q+^7nQ zN7UQ?<>SNov)Ygel`Dx4G>7}J)(i3u5QF>-*sFz1VaKs~&l8Gr{tY;;+;e#0OL1;f z6G3SzMeR~AXP5#DvL4{6yT|%y&wP(p(d3-&clBM}exJ3|cl&$i?lXru;607vKlY17 z6};!}Z22laDw~K1TPqPtEoY_DTH;I2`^y-=`}x(!x1axR|8m##L0{ay>GB>i;Q-jI z&u5mFHU%O6S}>TZv-U7WII&B7V>85i`F!Iq_Z$jN#OP4-=2vC{#)VF_z7~}AMNEjX zXb~6AmCh16e;f{DQj)zpJvn~xX@BoraiD(p9X~(fvysSvGzqH%JV(@AF}%WYIQ=hv z{L}vBu09kS1WK2`c-wC_U&3OKcm3m&U045; z{@&kyEBbpwzCRv~jKCP;5@i}6v*dh6N5aLH$}9Iv8~^40)- diff --git a/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png b/docs/my-website/src/pages/tutorial-extras/img/localeDropdown.png deleted file mode 100644 index e257edc1f932985396bf59584c7ccfaddf955779..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 27841 zcmXt9WmFtZ(*=S%B)EHUciG??+-=biEVw%f7J?HT77G@f5ZpbB1Pku&vgoqxemw6v z-;X&{JzZV*cFmohnLgcd+M3FE*p%2vNJx09Dhj$tNXVWq2M^|}mn)^e9a~;bs1CC4 zWs#5?l5k+wXfI`CFI{Chq}oa9BP66(NZK0uiU1Kwn&3K0m`=xIMoxdVZ#+ zp?hKSLSSimjhdEzWp#6Tbpr;2A08YY9vwczVR!d;r)Q^kw|6h$pbtRyO;c2US2)Ho=#3q?{4m1GWOCI`k&9;zl9YDhH|l{oVck{{HdF$xGeh(%RX@ITa1V-QE4arPZ_3^N0KUo15FS^Rt74gNyU?f6HsD z>zmu#+n1LY=NIRf7Z*oIN2_aF7nc`%dwaXPyVf>#Q`56+>svGPi|1!&J3Bj8*0u|a zE61nDOKTge8(T{&>(jIU{?5$PF)%N#t}iaHQc%;Ky=4F7L{Hzy*Vp$Mj`%zGZ+7k< zCpRC^+V1HYCi6}{?rS`Ew80CL%d5-LF)(<1lJAQ_QE}I< z?$m+XE%JR|)Y|g5*Z=3YjLfXkvht|tSaC_|$oh1*A78S&%grr-Q|oi0ai*n%^?I3Z zz4Ifn)p1zW0ShuJU zjT*W!;4n~Y)3m5E=4m0n9;cN(k*j`y5!~j2)ij4x1#tx zB&it>z`(yY6BF>DU9?)rvOb2G!4AbPa`$!ju_}{}N=X3%ljy@XN?Dz5W~L8#vn;(% zS0y`!_FK8bT{5iuza9iPzyFntcC0hEUgCyxwZgrs_lXv54ZHujy!d4_U`~v!&Xq6w z_%CfMkDLt!D3SDYg>XEZ!YJH*s~-dg$LmS&Mt_;Y7X9a!>IDr+ded%2&q%}2^ODhk zoJMHe1;<*D7+WnelW=pb#;#*9m22_D0Uy+B;{x z(r=4T(e9>b$HL=1ZhtTnMZ8m?T*4WlE1nANJoY~M+S`a~oAzPxq?IY|K;|faC(Qf6 z6st=g2Oa&+>GJF*AU5<{Q1pIIjk9IOz}i1XThs0R)dBg}u}I!L^(JejuqE{$Bx0WH zK_L%2hekVKCo%({=C&4>8XPbm?HVjtj7;pR;Nl%bO7u_%gfl5w5S;(8b>qCb9KY=2 zcH1B8#T*pZQMR+_zF|mDvyu5p%arE^>?K|9F#FDuJCyu6$KPjjPBMq7j0f$|h@y!QXH+UdeH3iv*9ArYX^V-S2rxolaBRROkUH4!AxVghY-$mqUuOg%w5X}J1K z3LIKED&GtI+|Bu|l2OgJXS@ z##5m-UU-??q5BVBs3e%jt&;*!MXilSO_r%{gmW&qj$2WWx8M1Us?Tzp=Of?r=^y=m zDDr>5Z2+yUUf9O3Kqm?KxT9VJX#G6EP&E+e7EkxJF5QqcBPy@TsIFiD!!LWKz2ftR za<|^DinsXw>aBe|0DWOEi#5cV&B>!$i8?+vTr3ZDMK}XFeg)Ime5=*V++LLjj6sSf>5d+I|6V|cU`LfQPC z;p|(TN|j&~8CO`*qIi-79281;uL=cj-kt$ zx5MwWh>2LRlqjdUEGgk)P@$`Rs3-3sSlqxdxpG@!K`;a)V2m#wvau8$FIZuT9T00v znI8L>LHCkAZsu+5PUedUKs5fY2Ehv7Lqr}Ue$h;p6jBeeweEDUn2p#fwkvxk%Z<-6 zlgcD$>a-9H1#>^}Ku>>wLa`FkP^$V?ys$YQ&1L$o#0R}|{e?+I{K?~0CPz_*Bh#mo zh#!|PeV|ebfXa=JD#~>$?!*)i)b@eZZ`$qTk#-n$b{Cnhx2wH9N;PkqOwfS5FPe4A z!^5G+7=f|QUkN8gZmRRF-gxA&%`!7|FLGzf?uPu9E>P4d zrO@YSB$ z8Q{^@GSty5G&7xHSPy#pErSb3Yym^l5+QhvVlc)ItslUVgKOTQyYw8QX+2%`A%uhb zCJ{CE9{zUB(&-v8uRN|49S2Np{L4XRjFWz9R?)%ikl#d@WJtzM$=odVE^A1_CR5$l zs~b7y&?qM}RqSq1_-7&^wqiGh$yZuM2alHG{5LL=^QiF^u2prn!rcZ9%AF_!mJaxS9)8?8ha{9;`m^(Fx7`o(9*^- zI+OEv7<`;JEbKrNAh#EhBOA3x9E1Hr;lS)5pbY@p_LBMGn<&!Nxl41i9>dX%V}P+N zR;}+{G5WqCjnW#@f9ZNd^d5R<+ViQpx-L3$P}Nkiph3->K~K9)Sw$@INj*8YJLj@f z*+Rh+naB!_+NtSnzwWfLhq1;bmSozM80Xik(oGSLM*c)>iC_Wvd=JP|df1=roC3iU zoG&xR@$6d-6s0^VR}3V5OFQndgqfbboOay9Tf7RQmygGWgZ+DD(=|p9Aw+)O_j8?HRA#~+mIn^!H zQ6fcNW1FIjQ#SN_nK%EQV_F{VV77VfT5B(ea{vC|K#&-RTdcH#OR%(Mr#R1?jLzzq zSC-hN{(b^Ik^Q{uB|gq70;JUnM+#nmHCHA@PxC-sYqdnHZfEu1VHP*(8?jf)TsXH7 z`d(w{qU>V+81-UywGHL+AD7SV`|6-5PENL9RC02nnu15q_;*RRA_g8|!M(z88r&2? zCYs;1K=%c4QceJr-h+O=+K2tbY%HGQfyO1=9--HP5(yo2@2ad|TVK+$67(dBRpKI9 zcTvYDh?n^D9&qCvQhZoHb7DSvql}UJ8B+>~m5-ISatyypAR9WnfzbiDmXq*ctR3Xu z(~YwCAKYipx{EI8!HwsIlC6i`0rhcb>6<%+Cp)h@mK*_1d8_q6dg4>n}&ihP)NGiUvb81U?bXk&I< zbcqui@YB^CK-jFfu@*XpEERc^Mh(aJ)LBA@| ze4m|#Gs|Rc+0u4VvgE2s^$ ztYjCc@_u6&>iu~fe+ed*pr>hTdj(LcVf&SE`t2uXleZ(mhZd7kd|U$5HrJHPQ@IZ7 zz1w#&@Hi?VMVg$?DV~d{6LYoL8SFlWmuiYZxE8-M?^q32JSt7GoOVzZ8#I13;Ax`h zy=DXkH>H2B>%O@Ual0AO#Lh>Z`q=%r{iaZi3fZKcmBtmff&=e!GF%sO1~^L| z<3g?B>etUeZ?Suv6A<@bH;i=|KtG0mk@t4!qPRX4+^*osf+?77qg=U_OjVUxbTvh% z8DC!P=LlXRVFEd#m0i*Ka(b7e+3E&CC^Yv2#TgpoU(C>Wsp4))0%aRYtPxSr1x zO6uJUAMROWMj1L@;~jX6gRh(+e1ZqC_CTY4s&GfB-E;b?6+vEb;^bSE6j9xTFW;oq z9(1ndc$4}qdAB6ta4BN@p|T{**jB2P48}=Ya*Jc5#3mv|J&XRD;~yH>^DLwT>bp@)BbsVm+*3t=;598_Aj{ zF(?v`d_@ky*e%9dvu#A7+LtE~P$5VDCRJz{ZCt3Qh5aQ==>mF~k7bTCZxZg$!jnP8he7?WmJYT*1>c{*tJR|Ie+ScEevd4@gG>!gnL_ZL0 zKC)4$4wIXHIG~yE4+vZ~gh~Du9&92xJVUy91zt6P+$SZ9%)_wNU7KW~uGu2PF`KM6 z)UjHJQr%bRkMmIKABTD;BRcKhrdAbU;gFURvdg`TDW)T{)k8(vFbmtSAMueO{E8RHEQz-$F2C0;smk?8Q*e=qM%6O z6aGCJV;h1Tf3qvPEYi~fsz?&nlrg71v(eKqA!&F7d&p(^Xy#{`bl-!6%zc6pwsB;^ z+s#(uj7tu(L!ti&l1T51?Zuxg`16)sS-XNZm6tV-9#MfVeX#M39*XRuyFiJrxU@lO zA94#H%u0U~Ea9b26Qf{o;FeeG*!6uF*bYv#%%B^zN~9gqX{FS&&Ba|4AuSA${f^sf z7tg9}O%6m})g#&j5f%_eXA&}AZI!vQtzb=^sQxVZi~_}R^pgdM?5WD3%5Gx)%~qaP zgb4y1pEi3Ut}qG#QQ8SxhEkYe1Iy%QMz~|VS zKNsn5WGa%en;uc#7;LpDxYo4^@zL&dT*?Movr0f}Fry~2?+=LVy&$9SKV5+@SE-{M z4E!tmqebqFV%O~LO=L7??~zNUu90ECkq2Dut+Q$C#QJ*uQ33)=L?sH^oM|)e*HvE5J+C=qp79zhoRrLcNRA%1 zo?(m~(so82vOoC7`kQMWO5~^(`_b!C)8yq_VgnO5blD*sV`=DhQ}{$VtHxJJ@hixJ@hcZ z!Y6lPxZ6KphBnMJ)Ki2qFXY=iKs$GnX#1@Z7~hW~TuZju?)u=y?>z5W?Gv0-coA#k zCeo>mYl2HbT(xw!L&23l5KXaDk)yq}eBc&oPdWOPI`+f_o2cgW5QeU+)?Z2SHRplP z^{WM#a*z=ndtAjrTjbW0xE@*Ir~X+Bi-n#;6t1um9|^H4v%4b8X{_t71*TeupTOxB zM!=Yir}l!cM!GzQSnjS?@tOr){-JXhj8oH5p=g?cX47@jYyLLVq#|_Nsv3>>?X=ey zqHoKr;KTdI-GBAo?{+YUsVsacvsXS>8d?dLdU_)>MB*glDaE}%bBrd^98i+k4NQ8s zc0?8Fbqr&)Wq3Wd=YVyyUH$oZkbSRGYQQj1NofbRth{_t5aE##Z zRgYXbJ@On89x{nXLRlW`84WcfoXw=cPcZZH9T^b zcb#iuU7-qyv~G@U`}AkosbCYozUSeB3Hxyoirpqhcbvd|soGDf8>z48$4OE>XaW4E zM`Bd>uV&vA8~mC0n0*yWn z!;O|1HnCN1ghEB898BR#@4Bo&&oP9!4dcdtLZ@`un@&0 zzvF-GJhEY|FLF{hrM=dB7|h@3bEZZVJc3@GCJk0{ONwS8^g2F0`roJtV2uvN1O)|| zIfYh)=}lZzT`5BbTHcM6zo=WwB7-gyvx+Cm)a}&MT+1M^^h@h5kMVlZF*~3?Y5n)L zG9~s#<;5)1%>+_Ny*GZHAebop+bfp3&+eUH&4)I7Bc%5<40;DxP0G8{l|7Ufj)b!u zw?zWRNHyLJzYlCQj^pLwN#g~68@bp>+KA=l8QJkW-|B;3+XPeez-@9TIs${Q*6_9g zgZY+gF6*%)arn3AJUkn5bhfZ9zut{n6VIK=XKt|=rtOVmc&6zImd8%#b}Bw)vQ<=y zZ*)E`F>yPlf=T61Cm%u&Swgy**c63kVp0V|yM7_vkz7jkw+1H3?_NcbXa2QR`&1S! z+&YBgY5aZe3Oz3Y&y0-J_SoE$OJ?^Y5E^umyENba+t#hf=fjWb@y_QD-S_*?k6rg& zYCqi76Dk6v!l>?hqKLvuFrKkCcX`eYORriHtB{LekCARf*i6xO%HyN*j5mwg%*8!T z_-nF5R#R3`E%JC%un?Z*bLKZbmC(`y?h5hS4~y5*hgyC*ji|t|>+*|`-dcqG*G|Tt zEST8(?OF|TW>rp<0OymrGE9zAlwD*|y}VO>>~H8Z91s2Imik`Rq+^-6$BW;-O~_dA z!0~$@ir)8VZEok*1Z^bx^25FUR#w|5ZBYL3o!iz3!TIR!4dM0kJ3M$Uu6oT8;CKYy50-UD6m_X=r8s9+5$+sA0zy6pqH_&Z@W^+??+HTsDpji* zpJYPs-t|l<_3g9}ngwho*oRGjLvmgR^?mB%vOAB;nrI30-@eap3v)1iCsy6LJHpO1J< zyJZ4Wh4TL8e$;A)3J{xrvG(WSc=))?Jb7Ude7PQzrs^QKFUs80=y)usVamepIs@|w z`Iz`#mm;4!p8c?~+N=@YBv*C$SE3I503HJZ0R|PT!IyVtgvYdpEy__RjV?qXKeZS8 zQn;w-0EHEP$J1*7n@+9+ndkivReVrStsXO#HIyz74ueJ3uc5Y(sVEe}?RntR{lQiH z`Z!qQ;Og%AD&~>mulH;=Kz}3H2_E@LZb@~4srs2{vY?%@)Kl!Nap4D79D{9}Z!`{& z?#?MOm>og((zofbkjOl>6O9@pvqoooVcjc^C-#xV?L|D3rXAR!rX4PzRkgx;H70*D zI_Pqi!x-h~CVp;&e0Ji8#XXONI@+S1=SSfqMQ>WVhhw!ZpqKaFLfG@O*E!;9JweoR z?{TX1XS6B@-~)hQV+wZL_soD`{+?KKnJh{Y4z>ugj&n-b6_}jBe(jSLX6P z&9H{W>AHrLNjvzbPKRmV@tT%0mYUCuBT1kvP^GO=`ICpra+8UwYXrd(pWPuzm_4{& zWk{u~y0Zv8Qlt(vtPO(#zX5n?`VDW3Ct(plTSM;$<*Wqlw`Z7-AN6CITh2!btkaDu zrf!`e&u14f%tSP&(Dnr<9bp(XcXW%tYO*s963nBWA=#0746gunNA6vAeP1s zh3fwN_Xo-D)nJ}kr8L9iLhlp8zQQ{nY4Q$@E9VtETvY3caFqEe?wB~cpWg4cy=Whdd?Z? zXPs;EKDvGsP6*bHo;Asedj+UOAyPE`Cwl8av`E7KMRPx4{M5Nm)na^3~o1fyYQucv~N{FBO$#$%a?f> z_2b|tKXBB$5)5npHFNe?Zy-grTI8sM+$}L__i>e2nemkwx%9r!i}lDhBEL!$_8+d6 z#LJ6vr&OO=-?Wf@W*)yvCLByyX|NQV|ecCy7=VAOB)9BI*Nhl6$m2&;G5gX z7X%M-WD-iH8(`K^IByV*KC4pkE;Q%d_{*#4?^g1OlJz4do+x=4js7@ z4A1i5J{^EH#kWeooG$|j7@#2|@kwpNNOp2q5tS?TUv|0sCwg@^U#G?D|NVyEHk3@4 zh9QWPx@!?z6UooVSfd6QY0LCJiII2vLNZ0~Jqnz~Z^l-ou^A;QU;}AhM{s6oqmA>R zx?|OM=&u!W1Uio$0m&-Ry7O|=MSkJHZ2nMCm3cd2v986rcYhXj>{)~`rp~In^`jTf zFrXGkn7tKYRu$h+~JfC4LO`D=-Is- z`O52#2dQHUn`kg1yFQXPBn)1doD3>%Z#Qc1db!Om^YRfrJIQst z-;fRaT=uTy2I$-qS|{FdP~V|NDf7ik?ZkYCef!_RSVV*5*a4(SshTJnq8S~a`-xao zsx;}%hcFK5ULvK;gHS_-z^^qx#frvEWpEI~{rtfbuS8wSnx+wfU>o`2dC=x3`D zBhoCot?)M$PTo$u&5L;JYCKUEb(v4VM%h4az4C?X?!Y6cb3KdhwS}?e9dC7;HdnO7P%wI_DM;;s)@@Z%bXbtAz>;d_JUlP#%eF{9 z&G?mfv!)Kp4BGm-`S$V!e>YW%_7wOu6Y@dH03UOV54u#?t3zN87%+2DV4y8UA)tjRAF;L2r0P4{}i zS>CSrwAQsVg`0^P+-P9(t8Inr_eUS#5t?4*HluhdNj63cJr5&s250OW1_Y*Veacuo z)0zW>;IdzS14@>TV9}D^5NujBuLsVE+*^zGaRsMzd40GW&lUtN9c}wb{~oH-rn5i@ z8}x~^(V56NJ>0RjWulsd{#z*g#MP3;$Kift?|Xb^>Pq7n-uera3;fa&%Kqq+sTISU z>9I?T5p%nzkJI+%EB3-pvu^_`-K4BPitQJr=<|A1pF^2$^d||Im4!Lx+DZc#;0d%Z zU}NxmZU|4p(!59eAHdzA{rqw6Ka=ssc2YVTy@Kr%TweSx7~PHI0$Ux(MH2xP>83k; zbDo^brmW`!))Eo*!~#*~(W4nwS!=Y1;yzh_{9+ERu~TOO)jk9Zv~B;)rYQX6mHFEK z$FpwAYy(lY1r9y+I7I{>9?geW)UF1iXT09htM#|*5w)gCZMKyi*_Ji;8TO`jkr6_D z6d^;@Cn2~1@1t9zQh@LC&YnCIm}xot2eOM8;p8qUQN8+;{_dBN&^VM~s_~5G#LV6m z_E3xKqtq!foUe8JYAMWpG6L66c?}#MBe-snYIx34#${6zQ+joY8Si;6OdZ&ke9RI9 zhJVE8S27lRcxM1to&zo06ulR~=)s2%EoSb-}Kq8vZm%56`3bWG&{95m-EEyf%f3 zH>Hp1P(-{>oBt2RmrZ0^^02K|$)u`-lkn!CnYo`C98s@Jf)-Nt3YGS7qu+WJ#ig-Q zFrQrF(9BS8SkgJ;+Ad7Nb-pL%EFha^nT1{-?E>u#tIcaiqZ19=37#rTd8pgB7g#`{ z3R`W-FmER}xBCpl>6-zNKPtsGV+;sy5|;j2PzH**0v8xbiA$I)z;nGF=f0kD;9o80 zk9RY17@+hFh@PzHbGN#U;3$|?cr@7<-4>(%aAapZ`iHIwt+VtBy0LH(1}{C)3kg3a z$axD|Iyt-X`@2lAY5noiw7Ges2e_Qy#ZG7g7!r}~R1hs0kXTsZV6s<#V!mFs#>11$)A=<$Kuz z!efePeRv291X1dfQaDLD&pz&rySTeJ)gM_}RHN4$p39$|V&}Hy&}+?dW^|({y!MySY<7Jzg!O zf^s9Ppls*TLgM-SI9c;jdIIB_?_E}SC2dbL5<#e@~e!>h*T}3V7Qjuwb}kpd$k{i8yIhNxcWp5 zmhr}|T%BZqGQI3rUBDr76MVryhwI4_s>U>$O&%JFqpibpT73JynWfVyP9vAd8#TkF z@b21lX~Xp&JvEw!njH%gzR#bLZ(HQc-x>V%ncNiNZVJK&R)GfUJ{=r%@BYj|e?tAE z^QvUXJVicpo4=Ku(9&oBMNT}AFs6q4)YmcNKs}&Yl3qAPrANKvAX)cQ0-_JnGLH^% zib2!LEZ+!2?9Xjt;Vsr#lw0vn26t$134ju@;-k>6A|D<1f9{NA&6lpAq^(bHU;73`4+N|^gyuiqNV6V>4tiHuh2}gS>rpliJMYF> z8oV`hL{!l3Cr!jFuS`U(PLYOcg;mf+q*tapy-Rrq73i4^Zr_D8w5!nj+I0u!FF(jA zaa|Fie9MYyVD zY+|f$aJ?0^#q(7Bv(_Rf>!-!26{dkm`vv5_{yhqlfE=-JnrnR3CE&==9oG^BPJ~kT zwR#L%pm6XWo_o>~-xFwsnFCS-K3SEG*9n3OmOIw$y|;&`Jh_54%d_jy$;Tc2Y_spR zsaIH2IH@qw%s;q1T8%_~*JZ&ytt);Fy%vh>g z0w_CsOn#JW{R5GsH?OEs1xr47FZzM7B-{&lNe2bAnJ#CYkWk}CK065tB0jzXv_Ue+ z&!kU}(r(0*6z9AtXe^RO8lX0D<%I!#-wUlmC}2X3R^;0)cuXyXl#01U9aAYGBNq07 zQ0C`^>CvlIsr|X$a@#JlI=!B?psUQx$bJ$^?{z*pe0X~bm^`c#V&s{0MlZ2T-y>}F z;qPquk(Pkc+@>~ButddAyRL%Hp<*0=QjboBwPSW-PHOEB-@Y}(p8aa|yNnqY5iwd} zMW09Non<@D_S6*Yt^2H1H_*KaVR?1$sYP$fe%28z_TYR*uvmX_{;5wg$t{cwp()qhVL2-qx3)1wM*a1-Qko7WOS|m_n5#TglB_)$&TDF_|oOK~F z5`+$vb~~{DgX@<_1p#;oVwb#0EZ3TI6$r55L4sS>BE@dTA#G0aD>84pQZg}wEWXX` zi!o|(wQ#4Y+7TC_zH2&(JiwOOYq`B)ZMOS$()lGjP?Re|ONa!QYMvwZxST#y zqxy;V%ft%25Xi@T@m(kD!pOvW$-@7ISP-Y%N|Ru>0)+_1!Xqh6yx_LcFNm{O`PE!f z1~@)qX~N_wIEb^f5u-?lm)di~;Jr!!^i2p381+NQa^Cc41Q-KE0Pi#aTB>o!<@$c% z*Q&0@cBXHDTZ2s@7*To0m*BYhWJwxEsgU+sx@6~uz6~lY%RS;a{p~AC-LG>IUop{T zr=uIPav^B@XZ77ba;qQ)w|Dxt$Q-fY!I+bh=a*g~Nhdb4cY<~1N)F-&Ui>SR1l(Zm@ zU~{AX%FoF4u=?X-SNV(5k>HE$9dJyNJ1i`5o7!u7exC)~47YqFkDvB6Qvg#`GnW$m zy^C0qY~lL3`HdJoR6L$C-K(+><84eipiDHzaN)Qv$Lvk($43+H>IVoTphDA%<1OV7 zN*wIOIb>eQ)`8RyzvwEjennj>vn!@tYo7b3bB?40+SdR)E#yrS^OTn6TmN05HqK%l zP)ZuCwf1Dqt9nt}M75{7)xl28WCdmP&nv%F5L&v^Csh6lR4+6qW$%QBQl1y9g2m&zLQodlxDQe5t ze74A-pBpIlCOSp+vzs<1{?Jh<5)t`U7lpH47Ax0o_SFnzt-ale`H{M8h&qB)qshbx7Ad#HNB$| zo={%npyBI&{m}+3+ngQmW@l~dYovp+my{i|_PyEoYucnl>EfHm=~;&)!6SYGXW9S; zu#fmK+2v+_G46lfe~J+}-wMrzj+?*^#t`G>E$l*-E7%bPB)Ef578L#cU|%dTi4@hk zp;+bBv%g-&D%NlYIGgkRvGc3A&8QgDxkHez9M?flQx3A$cKc(&?EFW$uDMSdb(QMw9odi zQA?zO%QwiY&D&*2_|La;le8f+v*;YqftP=UX(~GO>fBxRS{^y4gbh*RyJXj3%v!%! zELfdXKw~e(B^eo_RBX;Th4TrEi|2p2@Hg*5bt%Y7ZIk$P-}GUj)gwz0gIBAGiFNn8 zU4&Na+V|69<~TqZyxqSPaeGkw<_`ynX{4vBxwIX_Ypq#9SqSJ=W^R4opKAeSa3L{m z&lHRtdQy{5Ggy~SFu34>`lJ%Zqqg`)p0E)ulwxhQ-;}L>tXPKb-xTPBQs}1)CSM*$ z)G0-&fr8_TI{4boZwExp&4Rt|u<&mI1_Iy+`yv2(?Zm>&!E#z5*xWy{v=^H#tjEA3 z;?O-=$gFu6kw*5=S@@t1PtJM?AR~Jb<+?`D@ni^f9@rf(6M@{G_~V?Cy-fQf^8)n? zQMliUqyBPjXiOCQo#z#uU#^qooR+z_tHzkiIsIG6rn#gWN}koO1iCdnJ2E?}15?Vb zHv1jpiRE-A-RvipUQ>D1lRSvmj z7W3Og%mVd(!g)KZzdxx03y^c4IMqbhs;z8!D&FY;i56b*oQ6$WJxRAsvOKW!wE>ua zD0mc=bW>_*_Ph03EUervAR2#dSHw8J{!GR_N!df0ZL;vK+=3WRYyZ#GgT>l0+k}~1qIqt zS6WmMZM)!rz7z_m`fK9CHVM8F$z&G%jWzFH!hm|FYpam-1QF?Z)lPOHi8}0f1o9EZ zDHf!)*@a?vnvbdJDr!`&Cqj=g-f;y=uFs7+Jzk$Lqc5IOB(A-BqFIgF5T*Qh4dUC& z&KPT!3?JZJ?!2FGI-p$Yz1pL2ZT@|G!_!$1J@*9lY>pk*)lpl#C(!j;vJ^FY@2K3n z2bIo|a*SE!HzHgWM{6~I(^a*s15DV0tUv$zES9Amg!xeS8?y}$1Z}K#^z*n0>1~He8ZPz~6(W>wyBjvX_I$UA!VL?CFEa)<61QoPZ6E_lJpjc$tmFIQ8ZC{iPDf zO2-9y&-i(=bBR|;{%~gM8=O_tg<9F|DLGA&TZU$Dmt&g50M3#7f)z&Uh;BRwc9Fuz z-1wDw3C{{c-~!Wkhp>&;jVmvmxQJZfG-RppOg1^@pFD4B;*!n~lLSmHhRBGUZW=wL zrq<~HsA?@Fl|25*Z_6NPzj7X+}j+I5Z=nZ2_bWFC7 zTuxY^a9H;EY7yk(wd>FO+r1&Q=A6pE#dPEy^vWSAqgg}SUq@acOCxOw#+d|Qm9XIz zRGFSu)D?W`_1iH$=?m+!uJ;FT$Ox9sW_Mi@heywtUNevsjY|GZ+9y&g$4FCA5uwfk% zf*2q%_Xk{=xlxR0V-lrZ<8c^ny0kflt5f{jx54mj|S>kwam*Tak1b3;( z5uPT_RKvI3-JN1xNUUV?slZ3MO>r6QL6oc6t-jxIO{GxTrzD(yK)QDPpLm+v`7|p} z2gy(VZGC&YNw^Sa`UGiI9uXm!9PVra7Ew3o^o&h~XSGDkY zs;^`*cxA6xHK0$Wic0L>UEZ->|DkX6j1#<+RIHQm=vtR9K&^UG7kBp zohssHdJ&9qvGa3a$c)-8t8?K+cH6&N!v~A?-<*cwix;^Kx->T5?74h9@7rrK!RqW( zo2vJoGt#1rN>*x0wCL^Iy~m|a9o+HOx%%|#GJ$IR^@H56PS~Nk&64x4VbME}59a@h zAqcjHo2qUpv4ru+gtljF5cq0UfGkddYadJBa9qH5nTqNu$*6Eyt0)uW)o4o zI;X)D{>#dI8(%wELz1GF@W7BU?iTh#pd^;0(7A|qgmkyuW5DgLce~io- ziyf8;ON`-an0(auAd<+A^E&OM70amakbMh9ou51y1A4-pKz;ftECew{C|lR<2EG2V zc_YNUU-=dDwpU#60DATW|2Y$&LhL{Md zgU?Q#<3)i(y#qZ1bzpAfA$a(p99$lv#>L?Q)GTy zvV36GhERupL#v>^msU5ZmKGe6Pb0Y50Z_*r_EQ}YYljZ+66G=_SknIB zZ29q((LiBZotu{WaHM14bGk|AaDkw7pRRF+J)Lu6k|cfbwnXs?-X|W_s!|@*zFqbI zKH(l_gt(*O6YGy(ey6N?m_zU{`f$GyG}a%6%QeTyYV_*9CTC!O*p|m9#!SnxQYjCr zx0?Pz4pbv$bbm($)?Vpu@0tzWHsS2>)v#t> z@)vmMMS@d6sl1*mp^|5P{sVa2Ydr|^bT4x;;m;G%!7jv|MnM$?)5Ax-e8U)PJP1|j zw%heI;oCzyygq;2y=EfJqsY192X~vsQkXUXIO-m*UbQ!I#`v`?SW-Wg`74otU4C1v*?+r{tKmsUFh+cJOFn%ei*x1dOd6 zFdTHO)IfMfuFw1>5}qFUpQ-y^y)mXc>I%0whfG<;p=IXi5i)%>S(gUE5DNjBWKBzr z_#Wcq8RL0%$M(|1pAfjAhgbM^y%{*VI1Cxpv0wt>7i8%;SsQ+%*i3Mo@%ohOIdc9n_pG$ewjs26kJ$SwQbo^Sk8@-{F@9Fe^jtAAGY004(QP$Jw zW%MMJ!r8%+p2x)wEYW>%pS&FodEgu=HP#p6`0Pp&o4ydp&i>(Z~^F0082|Xag}ZxCR2>ZQ5t; z>A|WQnDS?znrt%Ye7if=pzl|H131>3+~^IjMyPz5ZIm@Fg=5~D$N*x02W!5TwV`kb z5cs|uy{8RXJNs9M*y;%C*|n%;`^I*cHg&PuVYA{FO+N1V#OU2-1R1gU@ug@Xa?q>b ze*(Sl%OV@%(h7UJ-Bu0-x!o!4QqeLO#F)tNvHiyS;USp!I+M=xg@Z(rv47_0_;K4l zshut-0EL`c=&=BxhuXPiRDTm2%{M?W6#9@tfK~EMaZ8WoQZWLcVe@du#-RsW4+z}g zO%&Y$Psw`fY1m|z2k?BkJbNCMBPap;?iM?k=FSWB*Y9pWRVL?x;LPus(N-8_gAb^2 zM!(Sv0At)38Cm$o>ww`vVSsgov{ zCdYVS8Njokqj9l98H3CsY7CH3qo`^|-M;Kkwb$*2&=wdc*1-MVk+~=0au2!?|GVoi zlb*^0KS?Cd6dOGkZxX~LQMUMnNLwVqKjApVqAuG@J2V4|Fd>bG08(u4#?aCTUfwsl z{TWl42|bHA2xHp6o%d%^K-JUV6R+VEJtB_j^juRPb}G3*dpx1g1>G$4D|Q=s2G}3F z;M%u%O4iu*46HuCLsus<$^K?YHU&?^`|2hfnKp0+1Y(JBc(8|T9J{KMB=@c(b3ro2 zd}F1=?F9afZ~ia~4`SjA>gbccd%Z9QB@zWr+A5TT>sE|}xp#hA#&LC`+{fA1q~Mmx z+3>dUL=K{Nck=f3=8SQ@%l>15p%Xoytnks;MkrQJ`6T31H;fuO#pNAfE-KSZmMP3@ zdV?m2M1M4Ni5x`?cm$`5?d(F2Rn)Mc246oiYT~1vAZvcRa4>RjEnY z8NB%znB~)cz7NJ}j%6vQisQW~_;r>G41dCv^mugKaMV#j1*e|WaXQam%?@nx(d*kR z@V)Bo;iEq2(L+y3>yNCS^$`W~tUB=5o*d2ik0YLVGl&)hCY;~+g$9;+2nOIL&ClSa zTuN#y(f|?&^pdT#|Ez4cA^jTq_=Y?0|BCwVa5kW}eTrH&O080>)LunxYP43(*4|X@ zy@`aP_O8aBMb+LrYL6iH9yKCnjTi~R=Y7B5`2U<|Ki74x^W5h?g}(n)O**8@D0X7% zVv1o98ti#psHl7+4G@z!_b)r-6_a96mysLGA`sTw(Ba-7OH=r)+EA&MQ`L_4tX0x^ zh97RKX4$v-B12RoBIkh@0H=2|>nW{0opXR%ix!QX23G=kLL=*dp`Khm?uTVT%=5qU zl4gELxb+XDu+fPBS<+5c=0N?{hS8o(nA9d9b3JdK`8G~5DcxJQ00$!y=d99=`xY)w zp-=NHMv)Qjt9j(z87hEilFo(355}q1@Z61JoxzK+smK_6!asIS7%bE2S{&+M-m`xqaH!!UdGuQ{MHaAnI2l0j<#hiPzCyfQYWoGe0;pPvFm9 zT-J;f{>>*8e=-gaW$IrStoFN!%a~L;Qa~w)fv1KAARO8J#5#Sm8Z{j z#VBuH3O4+H@pkC~JCMTsw_Q%vgPKQz$H#I*U>;hwTpuL-h7cqpS2-lF(*F7RD~i67 zB&2SfG7B>msr15LAdW>s7Alqm5I~DQGk<7+a$^#JgrrLh9s~7$Xle9d(Mgo*vsD77 z{XEUQAQbTUUiSPIpf#1~#b0Qe-(P5Lc5fhIUulw)PBL~)2q*Ap5kw1*lb26_XnqN}@H)z34&U z?4Hgp4HD1g^PpCA;OR=)fDO?6y6cAq?_jC(#}EdCh`QU>IwX)KN;^qF`M~?}m)5JT zP`Yj~INK=K`7hKcie~x|80v(_XO498{ z%^s9ZU(A!qoHI=zrty!fwL9+QM|?owwFzMRf6~AS2FK|Vrouv>ZbLV&|7K8fNZY)u z_sZaM(dD5>N()A^cp|44v_qzt)7Vu!$_hUiHdi!+Gsi3aMT~4UHg=v|7Nr$)@50{9 z>sQQ{(kob4m;|9pD;r0~k%Nr~Vsm~KY04(B>;tCiYDmM}oAtAst`I3MB8-^1o2*4y zg=}#5@v$pYJIkkeVAjPefCS@EAtJ8tvw2n~bX5N#2M1`#1Ca#)q+jL=(#NqNRit|l zV;QlZ#8SMO5qsok2-sFZGbtrhPJ{>uIw=e`rw!G+gd*hp>*aCy>? zvFOe+_1UcHYR?BD$%7t)pjqZN4t<aVv#X#4^luROO`zvzKdla_cXG4rX=K-zCu|J>K`0jQkZn&>rh- z>q*zkKe)=0ROa|p#N4B4M6USBET+lU%s<_26PUl6swgZeP}E@(*;cNu1~k7XyBjLZ z`HpJ}_F3G%AAjI!fpx$zz!qTGfrip=ZgX!>06=%A<7x8awY>DVcI!75wXO&#Uzb9A zHpP!eJ}**?zDle*Ov-CgAC3N^=C%f#m_;69M2Pse-+jVicE?|p7pHyz$4(J<~(i=wYOGLEU<%oiQ19w`jb~5lv3X_mQZu-QAF5j zyURDVYTRjBr8W-84N##WY~6PKt5@Up{EN%>@?_At1##d*91dmXm79_9O;V`0J-&J- zpK)+*(;)3(T5-M#g*qaET^f{}zKnLz!3M-K{r>y{M~!|6dK$UU0{mKS1)jh089wp^ zYd{j+YOQw%d+yQ?e0FVr=dgLi!3zTw+BkM`_el7$gU;YJ$1KNg&gTayx7TlO%4d!M zt?uykNvryn@^{l4w$F`sbSjz%J*O15cln`|JisON88##nfPU9$(VI2@VJ)y4#^{%M z6js!13fnZP*!`ln;HMR^%EyNq@W#*DCvh1TYB6&#vZSlKwm19H~JQ6?WU;JO# z5kR7Ld^&MB&Ca1I>0t!MCA?GexWe&E#x3p=}c>M%Vwn0Sj)w5+(Zh1v781%P3 z*?dm@r{9L5rIzX@KJW$=;>v3tbcad25&#QagCiBE75^)48;W>{K&Dj_?+f*XXBZ!F zR_V>eQ`v_Q#P&x7ry?n1VXlqKT`eXnzX*Ztign-ZO&3fsm%QACV)MCjOiNwT=Rf@? zyE>F^p~Y9X(2UW~pQF3J5l>#Y@4~0|SZ<;CC`X;(%hUO7L*CnkziIFKcH-Xvw5TOh z`hM3OpEVQYrK*@}CPu^F?*}utYCbXE)Y)67QZjfd%Vop$A`N=Hdo30DIIr^(gHF1G zvq(BMeUX^Ne34-3H7~e>%PNPbHFdm}aWQ!^X#P(YL}d5S-T0_|l4n;p!5Gm?U+7fP z!jB{4W`p$yzKYNU-Cx{?4&c<=Xpg`J$C=E?Pll3-8jyKO;5-)-tLhVDbw&n{oQEfp zof$G!Uf&fSJbY-BLUn8LXFT7c=|_TU%MEA`XW4~ncv(2+JJ8ZUq^W_ev5BP!uL%Av z=w6fluf(qR<`3BpQd!vW)pW8Y%HvP2CAg_7n2!jK^-iTP%`tGDw?^{a6(7LAxz1Rv z3)Vtc$M>Et-r$@L&XwlS{{#* z%?2{~t{;8&ntME~&j1RJ1vVdO;f_^L8v1izz0`GA82%;8E0G;Q!Jbk=Rk*Q9ykP{9 zwvb)l!HhkuHYv7Ct~*nRc}1w4!c$`~1^wOja3=&Y)f{t1-=17-oH(8FS!4=SyXujR zcIH(75Xghz3@T(Jzoi37k;X zrbjpVDeqg4O?>>{{~ew0*i0`}sgF>o_H#p@!M32sD=a(I5fiV}V0=RFX)h@kwli7; z{v~k=mD0CJ@X^Ot(aifPRR8Z|g=rE&)N^HKn|fz(F`b91J~!2` zpdH(30GLb5bz4^RmU)Qg7O?xh9x>9j);4v{eWiVeBtoCjmo1|`ldGQ<_GkYnREV0? zsed4$`tejon3!}p!kRPMC4qh3`uXcD?cG!Wnq;f%-WdXr5n&=$7Hf3o7kgRFmrzTP za(2#kiBiBUD&q6^jT@>qc~U25YJpM&x~wo)d1K&e6S9=jH+B`JWUvQAqO;(17FZBK zcx^2vQ;a>m^3e;)2OBOjk*fw3<-QOGF4nJh-Fe7D@)QHwu-olV&mk**>sJ#6D_-mi z1iuSrns!P{xpKoTmeFUY_g+8@<#l$B09pU8vjyc5#dh9+T8)M76ckFg{#yX@SDV~_ z(eN_~_V>2%zB;6U?-2mK>NM_WQG4enWns>yR_=e-!J)2Xsl~^w{mOUq`;0#r6oN5}O5)y#~?c?S*h_@upl zQSy^#c-Szn|MpDkzu#dd+?fu+QO0NO2y=9U~R?6EJ(#tAM3y9Y}Pi`s}tCNwwa2 zq;(h27Sf=*EPTSC>bujBTN7ViPPcB#Ecj15jlExHvqY+ehUaeG>K1x~-ZQ!Nl=-kn zbP)|!kLykq(9nektRqYaa2aJ4Y+HX~@SiSv>0jRh`im5=!Js~^^?mSxJKTMHjY?v8 zVIE67<#Il@C2JLsypu8oPFN?4$Q&t=oadNY1q>5`q0I*^QX6R zD4HPWPxKb^tRKjS|8J1^U8ka6>G!fSg0%b(KS1{x<2i#afYzM<)w5L?N~eI>r8^bS zwB=5inr;qxZGSPSOpxdJUgs4XN6ekD1eco*;qL{MrcO!6N!%)#{81Sf_ZdZ0`s`&5J~>IzYFU(_%TMg&eCB69q)8it?8MkVAL;BV zxo%KgVZB&PE1{6*vo?tl;p6&BEidXAq~a!gR4^!UgbY4PvXoo}g@|oO-m(Et2NS!F zkxPjdsj0BVqIu_(Px80y`06F@sNN1iwwb6x_Vg18aeQURHJ&uTdSTCpvrO)&fEYq6 z3kicA_FqElr+57>tMvTaU`FZ;BtE3n-*3WeS*+rcB3msBs|q#%!*V=^&TH|tO#lug zbPPScgFy-h)yjm{HnbHr;gvzdYz}3F9Hr66nP~TxkIrmX8^Z`nJ)!Zys*x~i5yyiA zFG+l@ZEzN{bPSEKyJWqYPfKh0%D~e4Nnf9$+>x0>>jaPv0B}yxMjKK9dN#INB!6n$ z#~M#K9cC)sbjALErQN{AgfN~}r#G-nd^BSA!%)DPSJ#9DdyI8_|DY6uymG~$2jpi$ zQ>-1y;*M|Wxt4FZ0VYXZ%}P5%g)eAZQA2i3lr@%Rh9>Gi;cZ+?2|6M>ll z>J}}1wB{2?<>u6mTRIXu8b_BX{J-6><*dVT$eTBT8J{L&!+3C;BD1rvuYuhHF;8{8 zQ)^BjmNlgbTkeqPm6b2sPbI>@NHly0`qJ%m4~6m$k2 zIZ(#DZ)glNu@M>{^c+DeTglVV*KE3 zz`=sp7EzVg64RmB#$|Cuymg-H0)A)kf%y1%`aw98n5=6hg=p&P? z9q7RG#bI#wICqbtjv;#y(GF+nK1a}HbB-7tdu9GF$2Pgu_4T~DPkel(q8XK3CJq(1 zAC&RiyOk-5UhcMTr#5%4ji@2Unq*H7_EX#ugj1x}^sm_IViJ>6VtXUE;R+luu`SxS zid2!9y_hO<`fuf*arD<-?Ha_lOOseuPzM8$bU4?A*sC9cZMMek1n--73oL!8@)pjyO^GmWJ17DxbFwwZ?>PB5AxD)L!t0M6y6OJ=5Dsw^k3~)39Ki*1MN7*Gu^uS zcn2ap+}(4ZHAsif2>)KEH>p06lgOv6=0G_2N5}_XW_dM9l$k0lJwQQXB6!9yMal|@ zbXo@n?{+f2J1Zi(fb&EZvlPlPkN^fu8K=Oj}FISvK!kkR6w62xmiS0Lm;_ZMs)w*hs^uk@r zi!K5FkcuzOzxd}}b#6y?Y{2IK?54LDxNG%A1Hq!38nzu+3^^G z<9OWrZhVDE;@Z)L7>Oi}<6d6_9`57qhu@MG<&LdMm}#<#QEi@u&Rwx*`77q-=GEcA z5F^+3wRv~92WIm^XWqu4T34W-bOy5BHI>DC-7&le9XJIc-9a6loj73@iXV;nNy(qJ z_}?B;Rr^s#lI0NVq)>6Gt&Yoi$uQ7-F1?^sOvJTP^G;16O92yqCD%ml3T*6hMT^cD zRhluHrmM&l%HA}1HO(I6d}*G`{Da!T;rmwPC#YHqvN=t^<_i>b>q;Ga&Zq?e7X9hi z^?Kf3tyT`bv}nw;|Liab90mNtt3>fU=4x!t!~U%^>pt;8zx2nV9QVoSvRJMyNuDV4 zv5Vj@Ls|1FBE98xkWy@yx@M=zr+cT&=69&P=^Oe9ecMjl?YCGkkH3tAX6!->L<26a z-Kg!x>&h_wj#OmYG;#eU#N4-U&PK*y#A8;EmkrSyt!&*P^jcaJE-URVhK(k7!I#}7 zc=cQy|EzTJo#&*)%~(VeI)E)Fhz_~56ulIyB(s=2bG$Zhg}O%hcQ48ZpVFc$ty_g! z4u*znqi}Gr_df07jntKq-7VeVMQ z)(4M;)lp~vVqfa%Obd9n-rQ>an>tT`U`AzYOGZSDWm!PYkg=p9;0|orKEhTn=sgt0 zhEQj=P+%$H{P0mS#W^G^8rz;o_v)Z*!`XJw>E^K0rOCb_mN4MOJoyKdyMC7uIc9qs zcSVNQ;d+48Hzg}l)fE*^wjps=YV?!StX^Q@=F8I-e<4F+{+B)Oc60S=0(*9F(Hart!5pnRV_aE_nI zmVuGYkmwOX`_Pu(_Iy=PLlpa;@!Cpv8tCA_a?yVJ`_lSP840FezVboo0}!P7RvJ_R z%{uS@n$mvYl=vgv5%DPIfOfiRRw~*9b@9XND9E9zK|!HOJx+0-$jkGj_(bsap={g} zQgi#dC#hM3c>CmNhb(dN^QiHh$UML0pU2DRz+b5=D+ zsWOWdnM5vx4IeU1IiE;bL5t6G0A|xb+X}sS=8pMK%zk{f4%bmba?HMRt}ek7-rEj< z#fvb0@~Yr8mUaE@v77VUg8ua)b|$=-eH(N0^zd8^ZAeN-cw2_QKw=y(qF13Q6{n|f z|M!)oB>&Kr5_DKHr=^+*rB_gt7sZaMNyJ}&uajMfm8{TL@{0JBCfq;$D#C+yezLb; zd|T_|=f&VkKRy^BFvXaF=-a-5{Z`eS_5AaebP?Q=PG&*LD`(%8Pp%pH^}ee7-`+;_ zFL-A9o*_P$zCSMt-D2j$k$5#MG<@eFcOUf4^oNC|Q?dlH2houFlWYcmg=05|%bh7? zeM~}MtKI5_4Fr&Wj2)r15)|}*x_nSwq*UyI@@N`xST2oVpT5N!XHi{}D^t3LW z)QWYzln?}cv`F-@tpJ-bx;2s|w(^WsB^_*bQKh+#fV_AwFOu0j+L zhwf}0{96B>DmmoSin7%d_O_O{J?}3_-K{!xpZ7NQ_1O(piGa>BCsb~N8fz(%;B5`S z><96Y71j{(#eq3vk|K+edR73!{2M5dH}c1Qy|cIIhJzvK@RXPKN|HlJ7Jc}YZ)x@R z=6GiB+z>kK;_-@eC`_D*ELPO!BWtwUb{4TlSlBi^{-ZU3lRqhQOT4Oj1Jq$=W>0VM z+{dD6A_66!;&N;G?v>?NJnBa*+$P)Xf=(NM%N(uPBV1I>u+xMQdzMejPXd3a z9q)SU?37-g=>@v+(O*b`k6cy3-Gpik&WnP&pu)H1!R2pc?@srJhOS1qYmqM9$E}w4 z(b&5mLotm9<t93*u}%_?&I@<({Y~xI@y}YYbBk;1;BMyD z;^O|%)9HzryP2v{H^`S(=iy}m#Zv?v-Rx5NHb-kYv%5T}@YGaUER3yRC;>xehpD!es1gMDY)rLAZ4`DY_hw!C7jR>u(TKM-eB8GtSm3a zstZT$5maSzy-rWzwtu?^K)ymZW95bGe{|MtH1A7e^2Jj zh&aEAV%iw0dSO6u2A+JGRA_OB+bc^SPqbZ!3Txk_Z=2>rQN z=Vock1nN#SB$^R)M-Sle9ulB-9$_v3b(duYR-=9@OfkQ`+}vu!_ReUIg6erUr9` z7^=Hgn6q0LrwQ1a{$~BSfVntOrqCTWDg;%v-waLrPIGb1|1^KhHvi0K29+EG$LGB| zUTFD@uEmy}4Gw1v9*w+?J$S?KW>^EXx)N2+TC zhONu}Nda!+B~dT04W+#&CLTBJcxA6 zPcr?5?VaFqQp3@hM6^I-40PiJ{kS5$gGlOXz$JK?u_l-{sk z^&S$X))sE=9Q3;%q{FW@Czd1#hf#5VtC(ppQgOw7E`vkrTc^}|fQ-3!v_JhmiKM|HrA2=Bl&?)2e)`;lG^#ZViDV4_R$p6~Js? ztK4U6+^#q|xg*yn)6VP}v(xi9#8;AAr`&=Zn~=W#0?9ANmZ)LzXh=a~C+wtPXUDyM z6h@*TXZ5@<{^5>Hy!mSll$Etg)A9XMn_4$PVj>{!fBQm>(Uu>GWFg-A1U3%q- zIW{nU5#n6K@#^b}C`pGruWVi~g0^OSuGJqe-QckH;(U>ljsE?j&C@rLrKlj?dw~zF zSm$QbZSRUF!86E4BvL`}S%M4Jt+2-qE~L|xS~P;Wva@JQTSLutv&NZLtoo~^Vt0tb zmjFzeDM|3wz>BmVNP=3eCmeQOYTx*7sZ1kyw%Bu;z85%+ zq@9l@iwHik5aU-k`WKtEIk@&K@n2U<)!}T5MvHm-%|$QF;vQ0)G6^N?rpU-HIrwZR z;|I7qQ_QvKy}ZrK1%N&Zke^v|DL2$UYEX<&c;LkykuJR<52H7suV3J^j*J6JKh0PN z#Oy6qY&&6Fk5bo94sA$KmQvJsD9MwS`}qFif2tL-SS$0dpI?Zc(v;*oAHxCD4|MA- z4F(8{p5fONvZqT8@lF=nGL{2+4*D_s$B(k5}$UmeZ7|j zD(=(@Hiu`Ke7^e^)z#Ito@z{&pknX+4Hje$XR;()V40J6`k3|ScoU!Pabun5@9%mP zmE0H)8ujqF3@j`{ssH>D@QaMH5^8TCZ^LDO{!!%PNEn6MW7YyC+i#)^Ow8An7w4hu zJ@(nP%+vtDo!CBc0r?3jw%d0#ygUU24b7gQ#AL4HJ^wT?jFCKsgZ06I)s3?0qQi$N zB1!(9M3$G;5+Nl%L^iTl=&#ok5~E5*pOeBWrLW$koe8@$Zw6)W)1O4YY46?P5(SAV zQT%^;4ds0^Zq*?DWKH2F&`MIl^ zWEn%ensMHAjJ3`FI1qZl*{@K`N&MXJDJ!0e+qa*e+GM{4^Tk)bR+MV8-stG&VK7`i zKAqZPTO9O+%>d^;IPwo^(&- z+FY-X4}F7=lL%`%MHaXyLv>oz)~+?>bxYyv?uV!4Q$xcnTb0^<-wehR<%%U;Jo>Og9FXpA z7+m9CzO^|~+=lCrvnjn1kK-e#&g&3sd&NfXGTJ0kul{Ll{gzl81UqJ8_%IE*41!RmC`9Gbpt%HjA}7%@P?8(&foUCm1E*2&oP zA?!^}75N2RqeGh;addDgdKQg0I&z5<894GRqif|!!3NMzWJqa_F-WrD_LYmrp1Hn| z-7Lagf`8mNvVumy?6;R;ff`k9|FlT-ilx{F(5Q|&)E(*xCmJ>xaZjpw`2yF}9d;*_1R z_t7&i=K$3fV-{5>8-EF-Ja#@rS&T{rkI-8f{%WI`b)?cK3Er*wIuc1Bfos##&3)2p zP)wC7<6gKp`E7wy8J?h-et+SU-WxMo1qIc0l;u17=TaMHv%A&z!NcLz_iUq}^ALcRQGp zO3#doE5|#DE|A17N&RrT%=+<_Q}UAjR}>vMemq*pZZSq4keZc7wkj?Tyw0KDeUqAX zGZq}z9c5m3xA==aFv2W4<~sN*{{4?ULGuufMXW;sxyI+iSm?i7hO@%9UYV(+`Q>Nos%vF8g!Usd2P z;4~-_8`!v6@(tpz_4Q(RM26{pkU|)UyNr=ihw-ukPHw<UpU+AXw!RaEXpRZ`!! zYg8dc?5IoMJQ2hB>hz-+?AEJm77QYbCtHtF_p0^ms1x@`UMtAF;}i{5AxiVl9DDpj zl)*5)Ng<4^TDD4i$KlbhQ-E&f_bUF+KzD6OX^sBayL(UNNV{|$loE2{yD|2UlLV?J z@Ig(y`w&7yeCv-`?uUV^&4RXrHsy&k@i}adNm;XgZ!a@xnvjG)yI_LjRiUqV%gYIh zTK1D&S;x6J%jL!y86wNhlMbcxK=q;CDA?OTEGBAUdVZ$JYB=ElyA%2HUEC_MuhHw9 zfP)~1CR0x8cHDC6+A8>NSYxQ2z$vA2UJn>pzZdq@C^#Xoh zdqe|=^fm{HmPOP#EjbbH25nT$CZP%K7azkF(mG$3cnFnvV!sc|V%0fVJ$l8KpsRTu zO8L$dH*_-Z+K;9`{p&$Rca2+turcwk=8~cyK0rNk55^Im*gM#q=U-^i{<0)$3uHRn zH_J=aK6A*?VLE!3Hi&0;r$KN%3v1#-jxKH%pl+cXKmYXX5gm8@@y1#xCav0t9od(z z48bdZip}mIsrXig{8+&@W$YEwRGTr);Lw|2E0DvqPPPlK%Q*y-eRpGMtZQa*dHiOB zm&!{b3*PxxlCIhz1he8Qe_ituN*=VlqosmzZgl~c62oxde$5Fm7!q248t=D%7jc(T&EAIMN0uPq5-R!nvG8HJu)x# z2l7Bbq!k*ScO@_{>}1p$JUt%!O}$q309mlnN$TVTn`5E)<0cDkchxB5N9ij>^1C4R z#OSfF27Mj!AhRy0lnNE`7ddO(RS@~@s9$AV72Rat8_}SIGlyS`bO`b4OLVX-@+it2;l!x9Kc))(Q=DJL~4JFw^ z(QdVI!ny}MfWXZX+W7j09)ZfAZ3qAKqN*1(7zzgC2SM1%t1q&GJt^ZKz5~NjeW$5Z JrC|B>e*nH7H{}2T diff --git a/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md b/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md deleted file mode 100644 index e12c3f3444f..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/manage-docs-versions.md +++ /dev/null @@ -1,55 +0,0 @@ ---- -sidebar_position: 1 ---- - -# Manage Docs Versions - -Docusaurus can manage multiple versions of your docs. - -## Create a docs version - -Release a version 1.0 of your project: - -```bash -npm run docusaurus docs:version 1.0 -``` - -The `docs` folder is copied into `versioned_docs/version-1.0` and `versions.json` is created. - -Your docs now have 2 versions: - -- `1.0` at `http://localhost:3000/docs/` for the version 1.0 docs -- `current` at `http://localhost:3000/docs/next/` for the **upcoming, unreleased docs** - -## Add a Version Dropdown - -To navigate seamlessly across versions, add a version dropdown. - -Modify the `docusaurus.config.js` file: - -```js title="docusaurus.config.js" -module.exports = { - themeConfig: { - navbar: { - items: [ - // highlight-start - { - type: 'docsVersionDropdown', - }, - // highlight-end - ], - }, - }, -}; -``` - -The docs version dropdown appears in your navbar: - -![Docs Version Dropdown](./img/docsVersionDropdown.png) - -## Update an existing version - -It is possible to edit versioned docs in their respective folder: - -- `versioned_docs/version-1.0/hello.md` updates `http://localhost:3000/docs/hello` -- `docs/hello.md` updates `http://localhost:3000/docs/next/hello` diff --git a/docs/my-website/src/pages/tutorial-extras/translate-your-site.md b/docs/my-website/src/pages/tutorial-extras/translate-your-site.md deleted file mode 100644 index caeaffb0554..00000000000 --- a/docs/my-website/src/pages/tutorial-extras/translate-your-site.md +++ /dev/null @@ -1,88 +0,0 @@ ---- -sidebar_position: 2 ---- - -# Translate your site - -Let's translate `docs/intro.md` to French. - -## Configure i18n - -Modify `docusaurus.config.js` to add support for the `fr` locale: - -```js title="docusaurus.config.js" -module.exports = { - i18n: { - defaultLocale: 'en', - locales: ['en', 'fr'], - }, -}; -``` - -## Translate a doc - -Copy the `docs/intro.md` file to the `i18n/fr` folder: - -```bash -mkdir -p i18n/fr/docusaurus-plugin-content-docs/current/ - -cp docs/intro.md i18n/fr/docusaurus-plugin-content-docs/current/intro.md -``` - -Translate `i18n/fr/docusaurus-plugin-content-docs/current/intro.md` in French. - -## Start your localized site - -Start your site on the French locale: - -```bash -npm run start -- --locale fr -``` - -Your localized site is accessible at [http://localhost:3000/fr/](http://localhost:3000/fr/) and the `Getting Started` page is translated. - -:::caution - -In development, you can only use one locale at a same time. - -::: - -## Add a Locale Dropdown - -To navigate seamlessly across languages, add a locale dropdown. - -Modify the `docusaurus.config.js` file: - -```js title="docusaurus.config.js" -module.exports = { - themeConfig: { - navbar: { - items: [ - // highlight-start - { - type: 'localeDropdown', - }, - // highlight-end - ], - }, - }, -}; -``` - -The locale dropdown now appears in your navbar: - -![Locale Dropdown](./img/localeDropdown.png) - -## Build your localized site - -Build your site for a specific locale: - -```bash -npm run build -- --locale fr -``` - -Or build your site to include all the locales at once: - -```bash -npm run build -``` From 86350fe6d70dc62ecb5331a7df5f96a35b853305 Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Wed, 3 Dec 2025 03:27:04 -0300 Subject: [PATCH 14/24] docs: add Google ADK and Harbor to projects (#17352) Both frameworks integrate with LiteLLM: - Google ADK uses LiteLLM for model-agnostic agent building - Harbor uses LiteLLM for agent evaluation across providers --- docs/my-website/docs/projects/Google ADK.md | 21 ++++++++++++++++++ docs/my-website/docs/projects/Harbor.md | 24 +++++++++++++++++++++ docs/my-website/sidebars.js | 4 +++- 3 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 docs/my-website/docs/projects/Google ADK.md create mode 100644 docs/my-website/docs/projects/Harbor.md diff --git a/docs/my-website/docs/projects/Google ADK.md b/docs/my-website/docs/projects/Google ADK.md new file mode 100644 index 00000000000..25e910dcbad --- /dev/null +++ b/docs/my-website/docs/projects/Google ADK.md @@ -0,0 +1,21 @@ + +# Google ADK (Agent Development Kit) + +[Google ADK](https://github.com/google/adk-python) is an open-source, code-first Python framework for building, evaluating, and deploying sophisticated AI agents. While optimized for Gemini, ADK is model-agnostic and supports LiteLLM for using 100+ providers. + +```python +from google.adk.agents.llm_agent import Agent +from google.adk.models.lite_llm import LiteLlm + +root_agent = Agent( + model=LiteLlm(model="openai/gpt-4o"), # Or any LiteLLM-supported model + name="my_agent", + description="An agent using LiteLLM", + instruction="You are a helpful assistant.", + tools=[your_tools], +) +``` + +- [GitHub](https://github.com/google/adk-python) +- [Documentation](https://google.github.io/adk-docs) +- [LiteLLM Samples](https://github.com/google/adk-python/tree/main/contributing/samples/hello_world_litellm) diff --git a/docs/my-website/docs/projects/Harbor.md b/docs/my-website/docs/projects/Harbor.md new file mode 100644 index 00000000000..684dfa93720 --- /dev/null +++ b/docs/my-website/docs/projects/Harbor.md @@ -0,0 +1,24 @@ + +# Harbor + +[Harbor](https://github.com/laude-institute/harbor) is a framework from the creators of Terminal-Bench for evaluating and optimizing agents and language models. It uses LiteLLM to call 100+ LLM providers. + +```bash +# Install +pip install harbor + +# Run a benchmark with any LiteLLM-supported model +harbor run --dataset terminal-bench@2.0 \ + --agent claude-code \ + --model anthropic/claude-opus-4-1 \ + --n-concurrent 4 +``` + +Key features: +- Evaluate agents like Claude Code, OpenHands, Codex CLI +- Build and share benchmarks and environments +- Run experiments in parallel across cloud providers (Daytona, Modal) +- Generate rollouts for RL optimization + +- [GitHub](https://github.com/laude-institute/harbor) +- [Documentation](https://harborframework.com/docs) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 983816ed216..a9790547e84 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -820,10 +820,12 @@ const sidebars = { "Learn how to deploy + call models from different providers on LiteLLM", slug: "/project", }, - items: [ + items: [ "projects/smolagents", "projects/mini-swe-agent", "projects/openai-agents", + "projects/Google ADK", + "projects/Harbor", "projects/Docq.AI", "projects/PDL", "projects/OpenInterpreter", From c173a4a27594b0a435f58a0be7633514bbeee440 Mon Sep 17 00:00:00 2001 From: Fabian Reinold <32450519+freinold@users.noreply.github.com> Date: Wed, 3 Dec 2025 07:30:54 +0100 Subject: [PATCH 15/24] Helm Chart: add ingress-only labels (#17348) * feat(helm): add ingress-only labels * feat(helm): add ingress configuration tests * chore(helm): bump chart version --- deploy/charts/litellm-helm/Chart.yaml | 4 +- deploy/charts/litellm-helm/README.md | 125 +++++++++--------- .../litellm-helm/templates/ingress.yaml | 3 + .../litellm-helm/tests/ingress_tests.yaml | 45 +++++++ deploy/charts/litellm-helm/values.yaml | 45 ++++--- 5 files changed, 140 insertions(+), 82 deletions(-) create mode 100644 deploy/charts/litellm-helm/tests/ingress_tests.yaml diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index eedadebaa8e..7f14af7db5d 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.8 +version: 0.4.9 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to @@ -33,5 +33,5 @@ dependencies: condition: db.deployStandalone - name: redis version: ">=18.0.0" - repository: oci://registry-1.docker.io/bitnamicharts + repository: oci://registry-1.docker.io/bitnamicharts condition: redis.enabled diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index 352c3e9ddff..6fdc423a177 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -10,46 +10,48 @@ - Helm 3.8.0+ If `db.deployStandalone` is used: + - PV provisioner support in the underlying infrastructure If `db.useStackgresOperator` is used (not yet implemented): -- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. + +- The Stackgres Operator must already be installed in the Kubernetes Cluster. This chart will **not** install the operator if it is missing. ## Parameters ### LiteLLM Proxy Deployment Settings -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | -| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | -| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | -| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | -| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | -| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | -| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | -| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | -| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | -| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | -| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | -| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | -| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | -| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | -| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | -| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | -| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | -| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | -| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. -| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | -| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | -| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | -| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | +| Name | Description | Value | +| --------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------- | +| `replicaCount` | The number of LiteLLM Proxy pods to be deployed | `1` | +| `masterkeySecretName` | The name of the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use the generated secret name. | N/A | +| `masterkeySecretKey` | The key within the Kubernetes Secret that contains the Master API Key for LiteLLM. If not specified, use `masterkey` as the key. | N/A | +| `masterkey` | The Master API Key for LiteLLM. If not specified, a random key in the `sk-...` format is generated. | N/A | +| `environmentSecrets` | An optional array of Secret object names. The keys and values in these secrets will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | +| `environmentConfigMaps` | An optional array of ConfigMap object names. The keys and values in these configmaps will be presented to the LiteLLM proxy pod as environment variables. See below for an example Secret object. | `[]` | +| `image.repository` | LiteLLM Proxy image repository | `ghcr.io/berriai/litellm` | +| `image.pullPolicy` | LiteLLM Proxy image pull policy | `IfNotPresent` | +| `image.tag` | Overrides the image tag whose default the latest version of LiteLLM at the time this chart was published. | `""` | +| `imagePullSecrets` | Registry credentials for the LiteLLM and initContainer images. | `[]` | +| `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | +| `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | +| `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | +| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | +| `ingress.labels` | Additional labels for the Ingress resource | `{}` | +| `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | +| `proxyConfigMap.create` | When `true`, render a ConfigMap from `.Values.proxy_config` and mount it. | `true` | +| `proxyConfigMap.name` | When `create=false`, name of the existing ConfigMap to mount. | `""` | +| `proxyConfigMap.key` | Key in the ConfigMap that contains the proxy config file. | `"config.yaml"` | +| `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. Rendered into the ConfigMap’s `config.yaml` only when `proxyConfigMap.create=true`. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | `N/A` | +| `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | +| `pdb.enabled` | Enable a PodDisruptionBudget for the LiteLLM proxy Deployment | `false` | +| `pdb.minAvailable` | Minimum number/percentage of pods that must be available during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.maxUnavailable` | Maximum number/percentage of pods that can be unavailable during **voluntary** disruptions (choose **one** of minAvailable/maxUnavailable) | `null` | +| `pdb.annotations` | Extra metadata annotations to add to the PDB | `{}` | +| `pdb.labels` | Extra metadata labels to add to the PDB | `{}` | #### Example `proxy_config` ConfigMap from values (default): - ``` proxyConfigMap: create: true @@ -67,7 +69,6 @@ proxy_config: #### Example using existing `proxyConfigMap` instead of creating it: - ``` proxyConfigMap: create: false @@ -77,8 +78,7 @@ proxyConfigMap: # proxy_config is ignored in this mode ``` -#### Example `environmentSecrets` Secret - +#### Example `environmentSecrets` Secret ``` apiVersion: v1 @@ -91,21 +91,23 @@ type: Opaque ``` ### Database Settings -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | -| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | -| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | -| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | -| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | -| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | -| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | -| `db.useStackgresOperator` | Not yet implemented. | `false` | -| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | -| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | -| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | + +| Name | Description | Value | +| ------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ | +| `db.useExisting` | Use an existing Postgres database. A Kubernetes Secret object must exist that contains credentials for connecting to the database. An example secret object definition is provided below. | `false` | +| `db.endpoint` | If `db.useExisting` is `true`, this is the IP, Hostname or Service Name of the Postgres server to connect to. | `localhost` | +| `db.database` | If `db.useExisting` is `true`, the name of the existing database to connect to. | `litellm` | +| `db.url` | If `db.useExisting` is `true`, the connection url of the existing database to connect to can be overwritten with this value. | `postgresql://$(DATABASE_USERNAME):$(DATABASE_PASSWORD)@$(DATABASE_HOST)/$(DATABASE_NAME)` | +| `db.secret.name` | If `db.useExisting` is `true`, the name of the Kubernetes Secret that contains credentials. | `postgres` | +| `db.secret.usernameKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the username for authenticating with the Postgres instance. | `username` | +| `db.secret.passwordKey` | If `db.useExisting` is `true`, the name of the key within the Kubernetes Secret that holds the password associates with the above user. | `password` | +| `db.useStackgresOperator` | Not yet implemented. | `false` | +| `db.deployStandalone` | Deploy a standalone, single instance deployment of Postgres, using the Bitnami postgresql chart. This is useful for getting started but doesn't provide HA or (by default) data backups. | `true` | +| `postgresql.*` | If `db.deployStandalone` is `true`, configuration passed to the Bitnami postgresql chart. See the [Bitnami Documentation](https://github.com/bitnami/charts/tree/main/bitnami/postgresql) for full configuration details. See [values.yaml](./values.yaml) for the default configuration. | See [values.yaml](./values.yaml) | +| `postgresql.auth.*` | If `db.deployStandalone` is `true`, care should be taken to ensure the default `password` and `postgres-password` values are **NOT** used. | `NoTaGrEaTpAsSwOrD` | #### Example Postgres `db.useExisting` Secret + ```yaml apiVersion: v1 kind: Secret @@ -143,7 +145,7 @@ metadata: name: litellm-env-secret type: Opaque data: - SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded + SOME_PASSWORD: cDZbUGVXeU5e0ZW # base64 encoded ANOTHER_PASSWORD: AAZbUGVXeU5e0ZB # base64 encoded ``` @@ -153,23 +155,23 @@ Source: [GitHub Gist from troyharvey](https://gist.github.com/troyharvey/4506472 The migration job supports both ArgoCD and Helm hooks to ensure database migrations run at the appropriate time during deployments. -| Name | Description | Value | -| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----- | -| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | -| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | -| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | -| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | -| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | -| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | -| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | -| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | - +| Name | Description | Value | +| -------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------- | +| `migrationJob.enabled` | Enable or disable the schema migration Job | `true` | +| `migrationJob.backoffLimit` | Backoff limit for Job restarts | `4` | +| `migrationJob.ttlSecondsAfterFinished` | TTL for completed migration jobs | `120` | +| `migrationJob.annotations` | Additional annotations for the migration job pod | `{}` | +| `migrationJob.extraContainers` | Additional containers to run alongside the migration job | `[]` | +| `migrationJob.hooks.argocd.enabled` | Enable ArgoCD hooks for the migration job (uses PreSync hook with BeforeHookCreation delete policy) | `true` | +| `migrationJob.hooks.helm.enabled` | Enable Helm hooks for the migration job (uses pre-install,pre-upgrade hooks with before-hook-creation delete policy) | `false` | +| `migrationJob.hooks.helm.weight` | Helm hook execution order (lower weights executed first). Optional - defaults to "1" if not specified. | N/A | ## Accessing the Admin UI + When browsing to the URL published per the settings in `ingress.*`, you will -be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal +be prompted for **Admin Configuration**. The **Proxy Endpoint** is the internal (from the `litellm` pod's perspective) URL published by the `-litellm` -Kubernetes Service. If the deployment uses the default settings for this +Kubernetes Service. If the deployment uses the default settings for this service, the **Proxy Endpoint** should be set to `http://-litellm:4000`. The **Proxy Key** is the value specified for `masterkey` or, if a `masterkey` @@ -181,7 +183,8 @@ kubectl -n litellm get secret -litellm-masterkey -o jsonpath="{.data.ma ``` ## Admin UI Limitations -At the time of writing, the Admin UI is unable to add models. This is because + +At the time of writing, the Admin UI is unable to add models. This is because it would need to update the `config.yaml` file which is a exposed ConfigMap, and -therefore, read-only. This is a limitation of this helm chart, not the Admin UI +therefore, read-only. This is a limitation of this helm chart, not the Admin UI itself. diff --git a/deploy/charts/litellm-helm/templates/ingress.yaml b/deploy/charts/litellm-helm/templates/ingress.yaml index 09e8d715ab8..ea9ffcbb54c 100644 --- a/deploy/charts/litellm-helm/templates/ingress.yaml +++ b/deploy/charts/litellm-helm/templates/ingress.yaml @@ -18,6 +18,9 @@ metadata: name: {{ $fullName }} labels: {{- include "litellm.labels" . | nindent 4 }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.ingress.annotations }} annotations: {{- toYaml . | nindent 4 }} diff --git a/deploy/charts/litellm-helm/tests/ingress_tests.yaml b/deploy/charts/litellm-helm/tests/ingress_tests.yaml new file mode 100644 index 00000000000..aad6ecfcee8 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/ingress_tests.yaml @@ -0,0 +1,45 @@ +suite: Ingress Configuration Tests +templates: + - ingress.yaml +tests: + - it: should not create Ingress by default + asserts: + - hasDocuments: + count: 0 + + - it: should create Ingress when enabled + set: + ingress.enabled: true + asserts: + - hasDocuments: + count: 1 + - isKind: + of: Ingress + + - it: should add custom labels + set: + ingress.enabled: true + ingress.labels: + custom-label: "true" + another-label: "value" + asserts: + - isKind: + of: Ingress + - equal: + path: metadata.labels.custom-label + value: "true" + - equal: + path: metadata.labels.another-label + value: "value" + + - it: should add annotations + set: + ingress.enabled: true + ingress.annotations: + kubernetes.io/ingress.class: "nginx" + asserts: + - isKind: + of: Ingress + - equal: + path: metadata.annotations["kubernetes.io/ingress.class"] + value: "nginx" diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index acb8c9ca32f..35021157826 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -35,7 +35,8 @@ podAnnotations: {} podLabels: {} terminationGracePeriodSeconds: 90 -topologySpreadConstraints: [] +topologySpreadConstraints: + [] # - maxSkew: 1 # topologyKey: kubernetes.io/hostname # whenUnsatisfiable: DoNotSchedule @@ -46,7 +47,8 @@ topologySpreadConstraints: [] # At the time of writing, the litellm docker image requires write access to the # filesystem on startup so that prisma can install some dependencies. podSecurityContext: {} -securityContext: {} +securityContext: + {} # capabilities: # drop: # - ALL @@ -57,13 +59,15 @@ securityContext: {} # A list of Kubernetes Secret objects that will be exported to the LiteLLM proxy # pod as environment variables. These secrets can then be referenced in the # configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentSecrets: [] +environmentSecrets: + [] # - litellm-env-secret # A list of Kubernetes ConfigMap objects that will be exported to the LiteLLM proxy # pod as environment variables. The ConfigMap kv-pairs can then be referenced in the # configuration file (or "litellm" ConfigMap) with `os.environ/` -environmentConfigMaps: [] +environmentConfigMaps: + [] # - litellm-env-configmap service: @@ -82,7 +86,9 @@ separateHealthPort: 8081 ingress: enabled: false className: "nginx" - annotations: {} + labels: {} + annotations: + {} # kubernetes.io/ingress.class: nginx # kubernetes.io/tls-acme: "true" hosts: @@ -129,7 +135,8 @@ proxy_config: general_settings: master_key: os.environ/PROXY_MASTER_KEY -resources: {} +resources: + {} # We usually recommend not to specify default resources and to leave this as a conscious # choice for the user. This also increases chances charts run on environments with little # resources, such as Minikube. If you do want to specify resources, uncomment the following @@ -231,7 +238,7 @@ migrationJob: # cpu: 100m # memory: 100Mi extraContainers: [] - + # Hook configuration hooks: argocd: @@ -240,30 +247,30 @@ migrationJob: enabled: false # Additional environment variables to be added to the deployment as a map of key-value pairs -envVars: { - # USE_DDTRACE: "true" -} +envVars: {} +# USE_DDTRACE: "true" # Additional environment variables to be added to the deployment as a list of k8s env vars -extraEnvVars: { - # - name: EXTRA_ENV_VAR - # value: EXTRA_ENV_VAR_VALUE -} +extraEnvVars: {} +# - name: EXTRA_ENV_VAR +# value: EXTRA_ENV_VAR_VALUE # Pod Disruption Budget pdb: enabled: false # Set exactly one of the following. If both are set, minAvailable takes precedence. - minAvailable: null # e.g. "50%" or 1 - maxUnavailable: null # e.g. 1 or "20%" + minAvailable: null # e.g. "50%" or 1 + maxUnavailable: null # e.g. 1 or "20%" annotations: {} labels: {} serviceMonitor: enabled: false - labels: {} + labels: + {} # test: test - annotations: {} + annotations: + {} # kubernetes.io/test: test interval: 15s scrapeTimeout: 10s @@ -273,4 +280,4 @@ serviceMonitor: # action: replace namespaceSelector: matchNames: [] - # - test-namespace \ No newline at end of file + # - test-namespace From 1ac2655b17f006f738632c27194153b93e9faa0c Mon Sep 17 00:00:00 2001 From: rioiart Date: Wed, 3 Dec 2025 07:46:03 +0100 Subject: [PATCH 16/24] Fix/organization max budget not enforced (#17334) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test: add failing tests for organization budget enforcement bug Add comprehensive tests exposing that organization-level budgets are retrieved but never enforced during request authentication. Tests verify: 1. Basic org budget exceeded scenario (team under budget, org over) 2. Multiple teams collectively exceeding org budget 3. Organization budget fields exist but are never checked 4. Inconsistency between team budget enforcement (works) and org (doesn't) Tests intentionally fail to document the bug. Will be fixed in next commit. Related to organization_max_budget not being enforced in auth_checks.py * fix: enforce organization budget in auth checks Add organization budget enforcement to common_checks() in auth_checks.py. Previously, organization_max_budget was retrieved from DB but never checked, allowing teams to collectively exceed their organization's budget limit. Changes: - Add _organization_max_budget_check() function following team budget pattern - Call org budget check after team budget check in common_checks() - Add "organization_budget" to budget_alerts type literals - Update tests to verify org budget is enforced Budget hierarchy is now properly enforced: Organization Budget (hard ceiling) └─ Team Budget (sub-allocation) └─ Team Member Budget (per-user within team) └─ Key Budget (per-key) 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude * fix: add organization_id to budget alerts, fix enum comparison and linting of newly added code - Add organization_id field to CallInfo class for better alert context - Include organization_id in budget alerts (token, soft, team, org) - Fix event_group enum comparison (was comparing enum to string) - Add OrganizationBudgetAlert class for organization budget alerting - Add organization_budget to test parameterizations - Apply Black formatting to slack_alerting.py --------- Co-authored-by: Claude --- .../SlackAlerting/budget_alert_types.py | 10 + .../SlackAlerting/slack_alerting.py | 19 +- litellm/proxy/_types.py | 1 + litellm/proxy/auth/auth_checks.py | 70 ++++ litellm/proxy/utils.py | 1 + tests/logging_callback_tests/test_alerting.py | 2 + .../test_organization_budget_enforcement.py | 344 ++++++++++++++++++ 7 files changed, 441 insertions(+), 6 deletions(-) create mode 100644 tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py diff --git a/litellm/integrations/SlackAlerting/budget_alert_types.py b/litellm/integrations/SlackAlerting/budget_alert_types.py index 1e9ad286e37..dadfef3fc40 100644 --- a/litellm/integrations/SlackAlerting/budget_alert_types.py +++ b/litellm/integrations/SlackAlerting/budget_alert_types.py @@ -50,6 +50,14 @@ class TeamBudgetAlert(BaseBudgetAlertType): return user_info.team_id or "default_id" +class OrganizationBudgetAlert(BaseBudgetAlertType): + def get_event_message(self) -> str: + return "Organization Budget: " + + def get_id(self, user_info: CallInfo) -> str: + return user_info.organization_id or "default_id" + + class TokenBudgetAlert(BaseBudgetAlertType): def get_event_message(self) -> str: return "Key Budget: " @@ -72,6 +80,7 @@ def get_budget_alert_type( "soft_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], @@ -83,6 +92,7 @@ def get_budget_alert_type( "soft_budget": SoftBudgetAlert(), "user_budget": UserBudgetAlert(), "team_budget": TeamBudgetAlert(), + "organization_budget": OrganizationBudgetAlert(), "token_budget": TokenBudgetAlert(), "projected_limit_exceeded": ProjectedLimitExceededAlert(), } diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py index 3efe5873786..0e691e2c43f 100644 --- a/litellm/integrations/SlackAlerting/slack_alerting.py +++ b/litellm/integrations/SlackAlerting/slack_alerting.py @@ -134,19 +134,25 @@ class SlackAlerting(CustomBatchLogger): if llm_router is not None: self.llm_router = llm_router - def _prepare_outage_value_for_cache(self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel]) -> dict: + def _prepare_outage_value_for_cache( + self, outage_value: Union[dict, ProviderRegionOutageModel, OutageModel] + ) -> dict: """ Helper method to prepare outage value for Redis caching. Converts set objects to lists for JSON serialization. """ # Convert to dict for processing cache_value = dict(outage_value) - - if "deployment_ids" in cache_value and isinstance(cache_value["deployment_ids"], set): + + if "deployment_ids" in cache_value and isinstance( + cache_value["deployment_ids"], set + ): cache_value["deployment_ids"] = list(cache_value["deployment_ids"]) return cache_value - def _restore_outage_value_from_cache(self, outage_value: Optional[dict]) -> Optional[dict]: + def _restore_outage_value_from_cache( + self, outage_value: Optional[dict] + ) -> Optional[dict]: """ Helper method to restore outage value after retrieving from cache. Converts list objects back to sets for proper handling. @@ -528,6 +534,7 @@ class SlackAlerting(CustomBatchLogger): "soft_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], @@ -1338,7 +1345,7 @@ Model Info: subject=email_event["subject"], html=email_event["html"], ) - if webhook_event.event_group == "team": + if webhook_event.event_group == Litellm_EntityType.TEAM: from litellm.integrations.email_alerting import send_team_budget_alert await send_team_budget_alert(webhook_event=webhook_event) @@ -1399,7 +1406,7 @@ Model Info: current_time = datetime.now().strftime("%H:%M:%S") _proxy_base_url = os.getenv("PROXY_BASE_URL", None) # Use .name if it's an enum, otherwise use as is - alert_type_name = getattr(alert_type, 'name', alert_type) + alert_type_name = getattr(alert_type, "name", alert_type) alert_type_formatted = f"Alert type: `{alert_type_name}`" if alert_type == "daily_reports" or alert_type == "new_model_added": formatted_message = alert_type_formatted + message diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7e7d4049818..53a8627bc8f 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2444,6 +2444,7 @@ class CallInfo(LiteLLMPydanticObjectBase): user_id: Optional[str] = None team_id: Optional[str] = None team_alias: Optional[str] = None + organization_id: Optional[str] = None user_email: Optional[str] = None key_alias: Optional[str] = None projected_exceeded_date: Optional[str] = None diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c9774b18b88..45b0752d4cd 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -143,6 +143,14 @@ async def common_checks( valid_token=valid_token, ) + # 3.1. If organization is in budget + await _organization_max_budget_check( + valid_token=valid_token, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + await _tag_max_budget_check( request_body=request_body, prisma_client=prisma_client, @@ -1893,6 +1901,7 @@ async def _virtual_key_max_budget_check( max_budget=valid_token.max_budget, user_id=valid_token.user_id, team_id=valid_token.team_id, + organization_id=valid_token.org_id, user_email=user_email, key_alias=valid_token.key_alias, event_group=Litellm_EntityType.KEY, @@ -1939,6 +1948,7 @@ async def _virtual_key_soft_budget_check( user_id=valid_token.user_id, team_id=valid_token.team_id, team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, user_email=None, key_alias=valid_token.key_alias, event_group=Litellm_EntityType.KEY, @@ -1977,6 +1987,7 @@ async def _team_max_budget_check( user_id=valid_token.user_id, team_id=valid_token.team_id, team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, event_group=Litellm_EntityType.TEAM, ) asyncio.create_task( @@ -1993,6 +2004,65 @@ async def _team_max_budget_check( ) +async def _organization_max_budget_check( + valid_token: Optional[UserAPIKeyAuth], + prisma_client: Optional[PrismaClient], + user_api_key_cache: DualCache, + proxy_logging_obj: ProxyLogging, +): + """ + Check if the organization is over its max budget. + + Raises: + BudgetExceededError if the organization is over its max budget. + Triggers a budget alert if the organization is over its max budget. + """ + # Only check if token has organization info and organization_max_budget is set + if ( + valid_token is None + or valid_token.org_id is None + or valid_token.organization_max_budget is None + or valid_token.organization_max_budget <= 0 + ): + return + + # Get organization object to check current spend + if prisma_client is not None: + org_table = await get_org_object( + org_id=valid_token.org_id, + prisma_client=prisma_client, + user_api_key_cache=user_api_key_cache, + ) + + if ( + org_table is not None + and org_table.spend >= valid_token.organization_max_budget + ): + # Trigger budget alert + call_info = CallInfo( + token=valid_token.token, + spend=org_table.spend, + max_budget=valid_token.organization_max_budget, + user_id=valid_token.user_id, + team_id=valid_token.team_id, + team_alias=valid_token.team_alias, + organization_id=valid_token.org_id, + event_group=Litellm_EntityType.ORGANIZATION, + ) + asyncio.create_task( + proxy_logging_obj.budget_alerts( + type="organization_budget", + user_info=call_info, + ) + ) + + raise litellm.BudgetExceededError( + current_cost=org_table.spend, + max_budget=valid_token.organization_max_budget, + message=f"Budget has been exceeded! Organization={valid_token.org_id} Current cost: {org_table.spend}, Max budget: {valid_token.organization_max_budget}", + ) + + async def _tag_max_budget_check( request_body: dict, prisma_client: Optional[PrismaClient], diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 9594a55962c..3b746b757ec 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1072,6 +1072,7 @@ class ProxyLogging: "user_budget", "soft_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], diff --git a/tests/logging_callback_tests/test_alerting.py b/tests/logging_callback_tests/test_alerting.py index b9ecfaeb3f0..ac7f5cd6aa1 100644 --- a/tests/logging_callback_tests/test_alerting.py +++ b/tests/logging_callback_tests/test_alerting.py @@ -477,6 +477,7 @@ async def test_send_daily_reports_all_zero_or_none(): "token_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], @@ -514,6 +515,7 @@ async def test_send_token_budget_crossed_alerts(alerting_type): "token_budget", "user_budget", "team_budget", + "organization_budget", "proxy_budget", "projected_limit_exceeded", ], diff --git a/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py b/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py new file mode 100644 index 00000000000..9c2adca9cd3 --- /dev/null +++ b/tests/test_litellm/proxy/auth/test_organization_budget_enforcement.py @@ -0,0 +1,344 @@ +""" +Tests for organization budget enforcement. + +These tests verify that organization-level budgets are properly enforced during +request authentication. When an organization's spend exceeds its max_budget, +requests should fail with BudgetExceededError. + +This prevents teams within an organization from collectively exceeding the +organization's budget limit. +""" + +import asyncio +import os +import sys +from typing import Optional +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +sys.path.insert(0, os.path.abspath("../../../")) + +import litellm +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_OrganizationTable, + LiteLLM_TeamTable, + UserAPIKeyAuth, +) +from litellm.proxy.auth.auth_checks import common_checks +from litellm.proxy.utils import ProxyLogging + + +@pytest.mark.asyncio +async def test_organization_budget_exceeded_blocks_request(): + """ + Bug: Organization budget is retrieved but NEVER enforced. + + When organization spend >= organization_max_budget, requests should fail + with BudgetExceededError. Currently this passes because no check exists. + """ + org_id = "test-org-budget-exceeded" + + # Organization with max_budget of 100, but spend is 150 + org_object = LiteLLM_OrganizationTable( + organization_id=org_id, + budget_id="org-budget-1", + spend=150.0, # Over budget! + models=["gpt-4"], + created_by="test", + updated_by="test", + litellm_budget_table=LiteLLM_BudgetTable( + max_budget=100.0, # Budget is 100 + ), + ) + + # Team within the organization (team itself is under budget) + team_object = LiteLLM_TeamTable( + team_id="test-team-1", + organization_id=org_id, + max_budget=50.0, # Team budget is 50 + spend=10.0, # Team spend is only 10 - under budget + models=["gpt-4"], + ) + + # Valid token with organization info + valid_token = UserAPIKeyAuth( + token="sk-test-123", + team_id="test-team-1", + org_id=org_id, + organization_max_budget=100.0, # This is set but never checked! + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.budget_alerts = AsyncMock() + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + with patch("litellm.proxy.proxy_server.user_api_key_cache") as mock_cache: + with patch("litellm.proxy.auth.auth_checks.get_org_object", new_callable=AsyncMock) as mock_get_org: + mock_get_org.return_value = org_object + + # BUG: This should raise BudgetExceededError but currently passes + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body={"model": "gpt-4"}, + team_object=team_object, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging, + valid_token=valid_token, + request=mock_request, + ) + + assert "Organization" in str(exc_info.value.message) + assert exc_info.value.current_cost == 150.0 + assert exc_info.value.max_budget == 100.0 + + +@pytest.mark.asyncio +async def test_multiple_teams_exceed_organization_budget(): + """ + Test that organization budget is enforced even when individual teams are under budget. + + Scenario: + - Organization max_budget = $5000, spend = $5000 (at limit) + - Team A spend = $1500 (under team budget of $2000) + - Request via Team A should FAIL because org is at budget limit + + Expected: Request fails with BudgetExceededError + """ + org_id = "multi-team-org" + + # Organization at budget limit + org_object = LiteLLM_OrganizationTable( + organization_id=org_id, + budget_id="org-budget-2", + spend=5000.0, # At $5000 limit + models=["gpt-4"], + created_by="test", + updated_by="test", + litellm_budget_table=LiteLLM_BudgetTable( + max_budget=5000.0, # Org budget is $5000 + ), + ) + + # Team A - under its own budget, but org is almost at limit + team_a = LiteLLM_TeamTable( + team_id="team-a", + organization_id=org_id, + max_budget=2000.0, + spend=1500.0, # Team A has spent $1500 of its $2000 budget + models=["gpt-4"], + ) + + valid_token = UserAPIKeyAuth( + token="sk-team-a-key", + team_id="team-a", + org_id=org_id, + organization_max_budget=5000.0, # Set but never enforced + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.budget_alerts = AsyncMock() + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + with patch("litellm.proxy.proxy_server.user_api_key_cache") as mock_cache: + with patch("litellm.proxy.auth.auth_checks.get_org_object", new_callable=AsyncMock) as mock_get_org: + mock_get_org.return_value = org_object + + # Org is at budget limit, should raise BudgetExceededError + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body={"model": "gpt-4"}, + team_object=team_a, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging, + valid_token=valid_token, + request=mock_request, + ) + + # Verify the error message mentions organization + assert "Organization" in str(exc_info.value.message) + assert exc_info.value.current_cost == 5000.0 + assert exc_info.value.max_budget == 5000.0 + + +@pytest.mark.asyncio +async def test_organization_budget_fields_are_checked(): + """ + Verify that organization_max_budget is populated in UserAPIKeyAuth + and BudgetExceededError is raised when organization is over budget. + """ + # Token has org budget info + valid_token = UserAPIKeyAuth( + token="sk-test", + team_id="test-team", + org_id="test-org", + organization_max_budget=100.0, # Budget is $100 + ) + + # Verify the field exists and is set + assert valid_token.organization_max_budget == 100.0 + assert valid_token.org_id == "test-org" + + team_object = LiteLLM_TeamTable( + team_id="test-team", + organization_id="test-org", + max_budget=None, + spend=0.0, + models=["gpt-4"], + ) + + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.budget_alerts = AsyncMock() + + # Organization is over budget + org_over_budget = LiteLLM_OrganizationTable( + organization_id="test-org", + budget_id="budget-1", + spend=150.0, # Over $100 budget + models=["gpt-4"], + created_by="test", + updated_by="test", + litellm_budget_table=LiteLLM_BudgetTable(max_budget=100.0), + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + with patch("litellm.proxy.proxy_server.user_api_key_cache") as mock_cache: + with patch("litellm.proxy.auth.auth_checks.get_org_object", new_callable=AsyncMock) as mock_get_org: + mock_get_org.return_value = org_over_budget + + # Organization is over budget, should raise BudgetExceededError + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body={"model": "gpt-4"}, + team_object=team_object, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging, + valid_token=valid_token, + request=mock_request, + ) + + assert exc_info.value.current_cost == 150.0 + assert exc_info.value.max_budget == 100.0 + + +@pytest.mark.asyncio +async def test_both_team_and_org_budget_enforced(): + """ + Verify that both team budget and organization budget are enforced consistently. + + This test verifies: + 1. Team over budget raises BudgetExceededError + 2. Organization over budget also raises BudgetExceededError + """ + mock_request = MagicMock() + mock_request.url.path = "/v1/chat/completions" + + mock_proxy_logging = MagicMock(spec=ProxyLogging) + mock_proxy_logging.budget_alerts = AsyncMock() + + # Scenario A: Team over budget - should raise BudgetExceededError + team_over_budget = LiteLLM_TeamTable( + team_id="team-over", + max_budget=100.0, + spend=150.0, # Over budget + models=["gpt-4"], + ) + + valid_token_team = UserAPIKeyAuth( + token="sk-team-test", + team_id="team-over", + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + with patch("litellm.proxy.proxy_server.user_api_key_cache") as mock_cache: + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body={"model": "gpt-4"}, + team_object=team_over_budget, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging, + valid_token=valid_token_team, + request=mock_request, + ) + assert "Team" in str(exc_info.value.message) + + # Scenario B: Org over budget - should also raise BudgetExceededError + org_over_budget = LiteLLM_OrganizationTable( + organization_id="org-over", + budget_id="budget-1", + spend=150.0, # Over $100 budget + models=["gpt-4"], + created_by="test", + updated_by="test", + litellm_budget_table=LiteLLM_BudgetTable(max_budget=100.0), + ) + + team_under_budget = LiteLLM_TeamTable( + team_id="team-under", + organization_id="org-over", + max_budget=50.0, + spend=10.0, # Team is fine + models=["gpt-4"], + ) + + valid_token_org = UserAPIKeyAuth( + token="sk-org-test", + team_id="team-under", + org_id="org-over", + organization_max_budget=100.0, + ) + + with patch("litellm.proxy.proxy_server.prisma_client") as mock_prisma: + with patch("litellm.proxy.proxy_server.user_api_key_cache") as mock_cache: + with patch("litellm.proxy.auth.auth_checks.get_org_object", new_callable=AsyncMock) as mock_get_org: + mock_get_org.return_value = org_over_budget + + # Organization is over budget, should raise BudgetExceededError + with pytest.raises(litellm.BudgetExceededError) as exc_info: + await common_checks( + request_body={"model": "gpt-4"}, + team_object=team_under_budget, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings={}, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=mock_proxy_logging, + valid_token=valid_token_org, + request=mock_request, + ) + + assert "Organization" in str(exc_info.value.message) + assert exc_info.value.current_cost == 150.0 + assert exc_info.value.max_budget == 100.0 From 54e29e7828f698690963eafac8148aeaf466262f Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 12:19:21 +0530 Subject: [PATCH 17/24] Enforce support of enforce_user_param to openai post endpoints --- litellm/proxy/auth/auth_checks.py | 13 +- .../proxy/test_enforce_user_param.py | 438 ++++++++++++++++++ 2 files changed, 450 insertions(+), 1 deletion(-) create mode 100644 tests/test_litellm/proxy/test_enforce_user_param.py diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index c9774b18b88..92cffe0ea82 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -182,7 +182,18 @@ async def common_checks( general_settings.get("enforce_user_param", None) is not None and general_settings["enforce_user_param"] is True ): - if RouteChecks.is_llm_api_route(route=route) and "user" not in request_body: + # Get HTTP method from request + http_method = request.method if hasattr(request, 'method') else None + + # Check if it's a POST request and if it's an OpenAI route but not MCP + is_post_method = http_method and http_method.upper() == "POST" + is_openai_route = RouteChecks.is_llm_api_route(route=route) + is_mcp_route = route in LiteLLMRoutes.mcp_routes.value or RouteChecks.check_route_access( + route=route, allowed_routes=LiteLLMRoutes.mcp_routes.value + ) + + # Enforce user param only for POST requests on OpenAI routes (excluding MCP routes) + if is_post_method and is_openai_route and not is_mcp_route and "user" not in request_body: raise Exception( f"'user' param not passed in. 'enforce_user_param'={general_settings['enforce_user_param']}" ) diff --git a/tests/test_litellm/proxy/test_enforce_user_param.py b/tests/test_litellm/proxy/test_enforce_user_param.py new file mode 100644 index 00000000000..5d9369d61bb --- /dev/null +++ b/tests/test_litellm/proxy/test_enforce_user_param.py @@ -0,0 +1,438 @@ +""" +Tests for enforce_user_param feature with POST/GET method filtering and MCP route exclusion. + +Tests verify that: +1. enforce_user_param only applies to POST requests +2. GET requests like /v1/models are not affected +3. MCP routes are excluded from enforcement +4. POST requests to completion endpoints still require user param when enforced +""" + +import asyncio +from unittest.mock import AsyncMock, MagicMock, PropertyMock, patch + +import pytest +from fastapi import Request + +from litellm.proxy._types import LiteLLMRoutes, UserAPIKeyAuth +from litellm.proxy.auth.auth_checks import common_checks +from litellm.proxy.auth.route_checks import RouteChecks + + +class MockRequest: + """Mock FastAPI Request object""" + def __init__(self, method: str = "POST"): + self.method = method + + +def get_mock_user_token(): + """Create a mock UserAPIKeyAuth token for testing""" + return UserAPIKeyAuth( + api_key="test-key", + user_id="test-user", + team_id="test-team", + org_id="test-org", + models=["*"], + metadata={} + ) + + +class TestEnforceUserParamPostGetFiltering: + """Test POST/GET method filtering for enforce_user_param""" + + @pytest.mark.asyncio + async def test_post_completion_without_user_param_should_fail(self): + """POST to /v1/chat/completions without user param should raise error""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + with pytest.raises(Exception) as exc_info: + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert "user" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_post_completion_with_user_param_should_pass(self): + """POST to /v1/chat/completions with user param should pass""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "user": "user123" + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_get_models_without_user_param_should_pass(self): + """GET to /v1/models without user param should NOT raise error""" + request = MockRequest(method="GET") + general_settings = {"enforce_user_param": True} + request_body = {} # GET requests typically don't have body + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise exception + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/models", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_get_files_without_user_param_should_pass(self): + """GET to /v1/files without user param should NOT raise error""" + request = MockRequest(method="GET") + general_settings = {"enforce_user_param": True} + request_body = {} + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/files", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_post_embeddings_without_user_param_should_fail(self): + """POST to /v1/embeddings without user param should raise error""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "text-embedding-ada-002", + "input": "test" + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + with pytest.raises(Exception) as exc_info: + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/embeddings", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert "user" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_post_embeddings_with_user_param_should_pass(self): + """POST to /v1/embeddings with user param should pass""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "text-embedding-ada-002", + "input": "test", + "user": "user123" + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/embeddings", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + +class TestEnforceUserParamMCPExclusion: + """Test MCP route exclusion from enforce_user_param""" + + @pytest.mark.asyncio + async def test_mcp_route_without_user_param_should_pass(self): + """POST to MCP route without user param should NOT raise error""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = {"action": "list_tools"} + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise exception for MCP routes + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/mcp/tools/list", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_mcp_root_route_without_user_param_should_pass(self): + """POST to /mcp without user param should NOT raise error""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": True} + request_body = {"data": "test"} + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/mcp", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + +class TestEnforceUserParamDisabled: + """Test behavior when enforce_user_param is disabled""" + + @pytest.mark.asyncio + async def test_post_without_user_param_when_disabled_should_pass(self): + """POST without user param when enforce_user_param=False should pass""" + request = MockRequest(method="POST") + general_settings = {"enforce_user_param": False} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_post_without_user_param_when_not_set_should_pass(self): + """POST without user param when enforce_user_param not set should pass""" + request = MockRequest(method="POST") + general_settings = {} # enforce_user_param not set + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + +class TestEnforceUserParamEdgeCases: + """Test edge cases for enforce_user_param""" + + @pytest.mark.asyncio + async def test_request_without_method_attribute_should_pass(self): + """Request without method attribute should not raise error""" + request = MagicMock() + del request.method # Remove method attribute + request.__hasattr__ = MagicMock(return_value=False) + + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise error even without method + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_case_insensitive_http_method(self): + """HTTP method comparison should be case-insensitive""" + request = MockRequest(method="post") # lowercase + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + with pytest.raises(Exception) as exc_info: + await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert "user" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_put_method_should_not_enforce_user_param(self): + """PUT method should not enforce user param (only POST)""" + request = MockRequest(method="PUT") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise for PUT method + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + @pytest.mark.asyncio + async def test_patch_method_should_not_enforce_user_param(self): + """PATCH method should not enforce user param (only POST)""" + request = MockRequest(method="PATCH") + general_settings = {"enforce_user_param": True} + request_body = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}] + } + + with patch('litellm.proxy.auth.auth_checks._is_api_route_allowed', new_callable=AsyncMock, return_value=True): + # Should not raise for PATCH method + result = await common_checks( + request_body=request_body, + team_object=None, + user_object=None, + end_user_object=None, + global_proxy_spend=None, + general_settings=general_settings, + route="/v1/chat/completions", + llm_router=None, + proxy_logging_obj=MagicMock(), + valid_token=get_mock_user_token(), + request=request, + ) + + assert result is True + + +if __name__ == "__main__": + # Run tests with: pytest tests/test_litellm/proxy/test_enforce_user_param.py -v + pytest.main([__file__, "-v"]) From 74ba18df55906165e5796a5408cbfd1fc8047604 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 2 Dec 2025 22:50:13 -0800 Subject: [PATCH 18/24] Litellm chainguard fixes 12 02 2025 p1 (#17406) * build: update dockerfile non root * build: update build * build: update non root * build: dockerfile fixes * build: ensure dockerfile + dockerfile.database also work --- Dockerfile | 15 +++++---------- docker/Dockerfile.database | 16 +++++++++------- docker/Dockerfile.non_root | 8 +++++--- 3 files changed, 19 insertions(+), 20 deletions(-) diff --git a/Dockerfile b/Dockerfile index f75706805e0..d8397ec4811 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -12,11 +12,9 @@ WORKDIR /app USER root # Install build dependencies -RUN apk add --no-cache gcc python3-dev openssl openssl-dev +RUN apk add --no-cache bash gcc py3-pip python3 python3-dev openssl openssl-dev - -RUN pip install --upgrade pip>=24.3.1 && \ - pip install build +RUN python -m pip install build # Copy the current directory contents into the container at /app COPY . . @@ -48,10 +46,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl tzdata nodejs npm - -# Upgrade pip to fix CVE-2025-8869 -RUN pip install --upgrade pip>=24.3.1 +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/docker/Dockerfile.database b/docker/Dockerfile.database index 09b5265191b..0e804cbfd12 100644 --- a/docker/Dockerfile.database +++ b/docker/Dockerfile.database @@ -1,8 +1,8 @@ # Base image for building -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base # Runtime image -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # Builder stage FROM $LITELLM_BUILD_IMAGE AS builder @@ -13,13 +13,15 @@ USER root # Install build dependencies RUN apk add --no-cache \ - build-base \ + bash \ + gcc \ + py3-pip \ + python3 \ python3-dev \ + openssl \ openssl-dev - -RUN pip install --upgrade pip && \ - pip install build +RUN python -m pip install build # Copy the current directory contents into the container at /app COPY . . @@ -46,7 +48,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl +RUN apk add --no-cache bash openssl tzdata nodejs npm python3 py3-pip WORKDIR /app # Copy the current directory contents into the container at /app diff --git a/docker/Dockerfile.non_root b/docker/Dockerfile.non_root index 8b66a367eeb..cd1633e319c 100644 --- a/docker/Dockerfile.non_root +++ b/docker/Dockerfile.non_root @@ -1,6 +1,6 @@ # Base images -ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/python:latest-dev -ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/python:latest-dev +ARG LITELLM_BUILD_IMAGE=cgr.dev/chainguard/wolfi-base +ARG LITELLM_RUNTIME_IMAGE=cgr.dev/chainguard/wolfi-base # ----------------- # Builder Stage @@ -11,6 +11,8 @@ WORKDIR /app # Install build dependencies including Node.js for UI build USER root RUN apk add --no-cache \ + python3 \ + py3-pip \ clang \ llvm \ lld \ @@ -71,7 +73,7 @@ WORKDIR /app # Install runtime dependencies USER root RUN apk upgrade --no-cache && \ - apk add --no-cache bash libstdc++ ca-certificates openssl supervisor + apk add --no-cache python3 py3-pip bash openssl tzdata nodejs npm supervisor # Copy only necessary artifacts from builder stage for runtime COPY . . From 8edcc4ecc3fc8ca56447e039871176622cf11aba Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Tue, 2 Dec 2025 22:52:09 -0800 Subject: [PATCH 19/24] Guardrails API - add streaming support (#17400) * fix(initial-commit): adding a way to get the right response type based on the api route * feat(unified_guardrail.py): support streaming guardrails * test: update tests * fix: fix linting errors * test: update tests --- litellm/batches/main.py | 2 - litellm/constants.py | 18 +- litellm/integrations/custom_guardrail.py | 1 - litellm/litellm_core_utils/README.md | 1 + .../api_route_to_call_types.py | 38 ++ .../guardrail_translation/base_translation.py | 14 + .../chat/guardrail_translation/handler.py | 98 +++++- ...odel_prices_and_context_window_backup.json | 65 ++++ .../unified_guardrail/unified_guardrail.py | 110 +++++- litellm/proxy/utils.py | 21 +- litellm/types/utils.py | 327 +++++++++++++++++- .../test_apply_guardrail_endpoint.py | 116 ++++--- .../test_bedrock_apply_guardrail.py | 161 +++++---- .../rerank/test_rerank_guardrail_handler.py | 72 ++-- .../test_text_completion_guardrail_handler.py | 48 ++- ...test_image_generation_guardrail_handler.py | 30 +- ...test_openai_responses_guardrail_handler.py | 14 +- .../test_text_to_speech_guardrail_handler.py | 73 ++-- ...t_audio_transcription_guardrail_handler.py | 80 +++-- .../content_filter/test_content_filter.py | 173 +++++---- .../guardrail_hooks/test_presidio.py | 51 +-- 21 files changed, 1134 insertions(+), 379 deletions(-) create mode 100644 litellm/litellm_core_utils/api_route_to_call_types.py diff --git a/litellm/batches/main.py b/litellm/batches/main.py index 353b1e25698..b99f4a628dc 100644 --- a/litellm/batches/main.py +++ b/litellm/batches/main.py @@ -18,8 +18,6 @@ from typing import Any, Coroutine, Dict, Literal, Optional, Union, cast import httpx from openai.types.batch import BatchRequestCounts -from openai.types.batch import Metadata -from openai.types.batch import Metadata as OpenAIBatchMetadata import litellm from litellm._logging import verbose_logger diff --git a/litellm/constants.py b/litellm/constants.py index db617a2e475..57029cd2a8c 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -262,7 +262,9 @@ TOGETHER_AI_EMBEDDING_350_M = int(os.getenv("TOGETHER_AI_EMBEDDING_350_M", 350)) QDRANT_SCALAR_QUANTILE = float(os.getenv("QDRANT_SCALAR_QUANTILE", 0.99)) QDRANT_VECTOR_SIZE = int(os.getenv("QDRANT_VECTOR_SIZE", 1536)) CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0.02)) -AUDIO_SPEECH_CHUNK_SIZE = 8192 # chunk_size for audio speech streaming. Balance between latency and memory usage +AUDIO_SPEECH_CHUNK_SIZE = int( + os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192) +) # chunk_size for audio speech streaming. Balance between latency and memory usage MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int( os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512) ) @@ -285,10 +287,16 @@ REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" MAX_LANGFUSE_INITIALIZED_CLIENTS = int( os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 50) ) -LOGGING_WORKER_CONCURRENCY = int(os.getenv("LOGGING_WORKER_CONCURRENCY", 100)) # Must be above 0 +LOGGING_WORKER_CONCURRENCY = int( + os.getenv("LOGGING_WORKER_CONCURRENCY", 100) +) # Must be above 0 LOGGING_WORKER_MAX_QUEUE_SIZE = int(os.getenv("LOGGING_WORKER_MAX_QUEUE_SIZE", 50_000)) -LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float(os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0)) -LOGGING_WORKER_CLEAR_PERCENTAGE = int(os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50)) # Percentage of queue to clear (default: 50%) +LOGGING_WORKER_MAX_TIME_PER_COROUTINE = float( + os.getenv("LOGGING_WORKER_MAX_TIME_PER_COROUTINE", 20.0) +) +LOGGING_WORKER_CLEAR_PERCENTAGE = int( + os.getenv("LOGGING_WORKER_CLEAR_PERCENTAGE", 50) +) # Percentage of queue to clear (default: 50%) MAX_ITERATIONS_TO_CLEAR_QUEUE = int(os.getenv("MAX_ITERATIONS_TO_CLEAR_QUEUE", 200)) MAX_TIME_TO_CLEAR_QUEUE = float(os.getenv("MAX_TIME_TO_CLEAR_QUEUE", 5.0)) LOGGING_WORKER_AGGRESSIVE_CLEAR_COOLDOWN_SECONDS = float( @@ -866,7 +874,7 @@ BEDROCK_INVOKE_PROVIDERS_LITERAL = Literal[ "deepseek_r1", "qwen3", "twelvelabs", - "openai" + "openai", ] BEDROCK_EMBEDDING_PROVIDERS_LITERAL = Literal[ diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 7f74f5d2157..507a754a7e0 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -20,7 +20,6 @@ from litellm.types.guardrails import ( GuardrailEventHooks, LitellmParams, Mode, - PiiEntityType, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel diff --git a/litellm/litellm_core_utils/README.md b/litellm/litellm_core_utils/README.md index 6494041291b..b61c8982762 100644 --- a/litellm/litellm_core_utils/README.md +++ b/litellm/litellm_core_utils/README.md @@ -9,4 +9,5 @@ Core files: - `default_encoding.py`: code for loading the default encoding (tiktoken) - `get_llm_provider_logic.py`: code for inferring the LLM provider from a given model name. - `duration_parser.py`: code for parsing durations - e.g. "1d", "1mo", "10s" +- `api_route_to_call_types.py`: mapping of API routes to their corresponding CallTypes (e.g., `/chat/completions` -> [acompletion, completion]) diff --git a/litellm/litellm_core_utils/api_route_to_call_types.py b/litellm/litellm_core_utils/api_route_to_call_types.py new file mode 100644 index 00000000000..35f83de1dd7 --- /dev/null +++ b/litellm/litellm_core_utils/api_route_to_call_types.py @@ -0,0 +1,38 @@ +""" +Dictionary mapping API routes to their corresponding CallTypes in LiteLLM. + +This dictionary maps each API endpoint to the CallTypes that can be used for that route. +Each route can have both async (prefixed with 'a') and sync call types. +""" + +from litellm.types.utils import API_ROUTE_TO_CALL_TYPES, CallTypes + + +def get_call_types_for_route(route: str) -> list: + """ + Get the list of CallTypes for a given API route. + + Args: + route: API route path (e.g., "/chat/completions") + + Returns: + List of CallTypes for that route, or empty list if route not found + """ + return API_ROUTE_TO_CALL_TYPES.get(route, []) + + +def get_routes_for_call_type(call_type: CallTypes) -> list: + """ + Get all routes that use a specific CallType. + + Args: + call_type: The CallType to search for + + Returns: + List of routes that use this CallType + """ + routes = [] + for route, types in API_ROUTE_TO_CALL_TYPES.items(): + if call_type in types: + routes.append(route) + return routes diff --git a/litellm/llms/base_llm/guardrail_translation/base_translation.py b/litellm/llms/base_llm/guardrail_translation/base_translation.py index 5acbf4e9f4f..c1ea3311bd8 100644 --- a/litellm/llms/base_llm/guardrail_translation/base_translation.py +++ b/litellm/llms/base_llm/guardrail_translation/base_translation.py @@ -84,3 +84,17 @@ class BaseTranslation(ABC): user_api_key_dict: User API key metadata (passed separately since response doesn't contain it) """ pass + + async def process_output_streaming_response( + self, + response: Any, + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: Optional["LiteLLMLoggingObj"] = None, + user_api_key_dict: Optional["UserAPIKeyAuth"] = None, + ) -> Any: + """ + Process output streaming response with guardrails. + + Optional to override in subclasses. + """ + return response diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 0abc94012ee..29fb12a6f74 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -14,16 +14,16 @@ Pattern Overview: This pattern can be replicated for other message formats (e.g., Anthropic). """ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, cast +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import litellm from litellm._logging import verbose_proxy_logger from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation -from litellm.types.utils import Choices +from litellm.types.utils import Choices, StreamingChoices if TYPE_CHECKING: from litellm.integrations.custom_guardrail import CustomGuardrail - from litellm.types.utils import ModelResponse + from litellm.types.utils import ModelResponse, ModelResponseStream class OpenAIChatCompletionsHandler(BaseTranslation): @@ -241,21 +241,79 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return response - def _has_text_content(self, response: "ModelResponse") -> bool: + async def process_output_streaming_response( + self, + response: "ModelResponseStream", + guardrail_to_apply: "CustomGuardrail", + litellm_logging_obj: Optional[Any] = None, + user_api_key_dict: Optional[Any] = None, + ) -> Any: + """ + Process output streaming response by applying guardrails to text content. + + Args: + response: LiteLLM ModelResponseStream object + guardrail_to_apply: The guardrail instance to apply + litellm_logging_obj: Optional logging object + user_api_key_dict: User API key metadata to pass to guardrails + + Returns: + Modified response with guardrail applied to content + + Response Format Support: + - String content: choice.message.content = "text here" + - List content: choice.message.content = [{"type": "text", "text": "text here"}, ...] + """ + + # Step 0: Check if response has any text content to process + if not self._has_text_content(response): + return response + + texts_to_check: List[str] = [] + images_to_check: List[str] = [] + task_mappings: List[Tuple[int, Optional[int]]] = [] + # Track (choice_index, content_index) for each text + + # Step 1: Extract all text content and images from response choices + for choice_idx, choice in enumerate(response.choices): + + self._extract_output_text_and_images( + choice=choice, + choice_idx=choice_idx, + texts_to_check=texts_to_check, + images_to_check=images_to_check, + task_mappings=task_mappings, + ) + + def _has_text_content( + self, response: Union["ModelResponse", "ModelResponseStream"] + ) -> bool: """ Check if response has any text content to process. Override this method to customize text content detection. """ - for choice in response.choices: - if isinstance(choice, litellm.Choices): - if choice.message.content and isinstance(choice.message.content, str): - return True + from litellm.types.utils import ModelResponse, ModelResponseStream + + if isinstance(response, ModelResponse): + for choice in response.choices: + if isinstance(choice, litellm.Choices): + if choice.message.content and isinstance( + choice.message.content, str + ): + return True + elif isinstance(response, ModelResponseStream): + for choice in response.choices: + if isinstance(choice, litellm.Choices): + if choice.message.content and isinstance( + choice.message.content, str + ): + return True return False def _extract_output_text_and_images( self, - choice: Any, + choice: Union[Choices, StreamingChoices], choice_idx: int, texts_to_check: List[str], images_to_check: List[str], @@ -266,21 +324,29 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Override this method to customize text/image extraction logic. """ - if not isinstance(choice, litellm.Choices): - return - verbose_proxy_logger.debug( "OpenAI Chat Completions: Processing choice: %s", choice ) - if choice.message.content and isinstance(choice.message.content, str): + # Determine content source based on choice type + content = None + if isinstance(choice, litellm.Choices): + content = choice.message.content + elif isinstance(choice, litellm.StreamingChoices): + content = choice.delta.content + else: + # Unknown choice type, skip processing + return + + # Process content if it exists + if content and isinstance(content, str): # Simple string content - texts_to_check.append(choice.message.content) + texts_to_check.append(content) task_mappings.append((choice_idx, None)) - elif choice.message.content and isinstance(choice.message.content, list): + elif content and isinstance(content, list): # List content (e.g., multimodal response) - for content_idx, content_item in enumerate(choice.message.content): + for content_idx, content_item in enumerate(content): # Extract text content_text = content_item.get("text") if content_text: diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 464f9c185f9..0fc97ce7b0a 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -269,6 +269,71 @@ "supports_response_schema": true, "supports_vision": true }, + "amazon.nova-2-lite-v1:0": { + "input_cost_per_token": 3e-07, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-06, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_video_input": true, + "supports_vision": true + }, + "apac.amazon.nova-2-lite-v1:0": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_video_input": true, + "supports_vision": true + }, + "eu.amazon.nova-2-lite-v1:0": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_video_input": true, + "supports_vision": true + }, + "us.amazon.nova-2-lite-v1:0": { + "input_cost_per_token": 6e-08, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 1000000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.75e-06, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_video_input": true, + "supports_vision": true + }, + "amazon.nova-micro-v1:0": { "input_cost_per_token": 3.5e-08, "litellm_provider": "bedrock_converse", diff --git a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py index aeae19a8270..0f05696af42 100644 --- a/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py +++ b/litellm/proxy/guardrails/guardrail_hooks/unified_guardrail/unified_guardrail.py @@ -13,6 +13,7 @@ from litellm.caching.caching import DualCache from litellm.cost_calculator import _infer_call_type from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.api_route_to_call_types import get_call_types_for_route from litellm.llms import load_guardrail_translation_mappings from litellm.proxy._types import UserAPIKeyAuth from litellm.types.guardrails import GuardrailEventHooks @@ -176,6 +177,113 @@ class UnifiedLLMGuardrails(CustomLogger): See Aim guardrail implementation for an example - https://github.com/BerriAI/litellm/blob/d0e022cfacb8e9ebc5409bb652059b6fd97b45c0/litellm/proxy/guardrails/guardrail_hooks/aim.py#L168 Triggered by mode: 'post_call' + + Supports sampling_rate parameter to control how often chunks are processed. + sampling_rate=1 means every chunk, sampling_rate=5 means every 5th chunk, etc. """ + + global endpoint_guardrail_translation_mappings + from litellm.proxy.common_utils.callback_utils import ( + add_guardrail_to_applied_guardrails_header, + ) + + guardrail_to_apply: CustomGuardrail = request_data.pop( + "guardrail_to_apply", None + ) + + # Get sampling rate from guardrail config or optional_params, default to 5 + sampling_rate = 5 + if guardrail_to_apply is not None: + # Check guardrail config first + guardrail_config = getattr(guardrail_to_apply, "guardrail_config", {}) + sampling_rate = guardrail_config.get( + "streaming_sampling_rate", sampling_rate + ) + + # Also check optional_params as fallback + sampling_rate = self.optional_params.get( + "streaming_sampling_rate", sampling_rate + ) + + if guardrail_to_apply is None: + async for item in response: + yield item + return + + event_type: GuardrailEventHooks = GuardrailEventHooks.post_call + if ( + guardrail_to_apply.should_run_guardrail( + data=request_data, event_type=event_type + ) + is not True + ): + verbose_proxy_logger.debug( + "UnifiedLLMGuardrails: Post-call streaming scanning disabled for %s", + guardrail_to_apply.guardrail_name, + ) + async for item in response: + yield item + return + + # Initialize translation mappings if needed + if endpoint_guardrail_translation_mappings is None: + endpoint_guardrail_translation_mappings = ( + load_guardrail_translation_mappings() + ) + + # Infer call type from first chunk + call_type = None + chunk_counter = 0 + async for item in response: - yield item + chunk_counter += 1 + + # Infer call type from first chunk if not already done + if call_type is None and user_api_key_dict.request_route is not None: + call_types = get_call_types_for_route(user_api_key_dict.request_route) + if call_types is not None: + call_type = call_types[0] + + # If call type not supported, just pass through all chunks + if ( + call_type is None + or CallTypes(call_type) + not in endpoint_guardrail_translation_mappings + ): + yield item + async for remaining_item in response: + yield remaining_item + return + + # Process chunk based on sampling rate + if chunk_counter % sampling_rate == 0: + verbose_proxy_logger.debug( + "Processing streaming chunk %s (sampling_rate=%s) with guardrail %s", + chunk_counter, + sampling_rate, + guardrail_to_apply.guardrail_name, + ) + + endpoint_translation = endpoint_guardrail_translation_mappings[ + CallTypes(call_type) + ]() + + processed_item = ( + await endpoint_translation.process_output_streaming_response( + response=item, + guardrail_to_apply=guardrail_to_apply, + litellm_logging_obj=request_data.get("litellm_logging_obj"), + user_api_key_dict=user_api_key_dict, + ) + ) + + # Add guardrail to applied guardrails header (only once, on first processed chunk) + if chunk_counter == sampling_rate: + add_guardrail_to_applied_guardrails_header( + request_data=request_data, + guardrail_name=guardrail_to_apply.guardrail_name, + ) + + yield processed_item + else: + yield item diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3b746b757ec..f0dccfae716 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -1560,6 +1560,7 @@ class ProxyLogging: Covers: 1. /chat/completions """ + for callback in litellm.callbacks: _callback: Optional[CustomLogger] = None if isinstance(callback, str): @@ -1574,11 +1575,21 @@ class ProxyLogging: ) or _callback.should_run_guardrail( data=request_data, event_type=GuardrailEventHooks.post_call ): - response = _callback.async_post_call_streaming_iterator_hook( - user_api_key_dict=user_api_key_dict, - response=response, - request_data=request_data, - ) + if "apply_guardrail" in type(callback).__dict__: + request_data["guardrail_to_apply"] = callback + response = ( + unified_guardrail.async_post_call_streaming_iterator_hook( + user_api_key_dict=user_api_key_dict, + request_data=request_data, + response=response, + ) + ) + else: + response = _callback.async_post_call_streaming_iterator_hook( + user_api_key_dict=user_api_key_dict, + response=response, + request_data=request_data, + ) return response def _init_response_taking_too_long_task(self, data: Optional[dict] = None): diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 5a582194147..cf3b1480e6a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -401,6 +401,328 @@ CallTypesLiteral = Literal[ "responses", ] +# Mapping of API routes to their corresponding call types +API_ROUTE_TO_CALL_TYPES = { + # Chat Completions + "/chat/completions": [CallTypes.acompletion, CallTypes.completion], + "/v1/chat/completions": [CallTypes.acompletion, CallTypes.completion], + "/engines/{model}/chat/completions": [CallTypes.acompletion, CallTypes.completion], + "/openai/deployments/{model}/chat/completions": [ + CallTypes.acompletion, + CallTypes.completion, + ], + # Text Completions + "/completions": [CallTypes.atext_completion, CallTypes.text_completion], + "/v1/completions": [CallTypes.atext_completion, CallTypes.text_completion], + "/engines/{model}/completions": [ + CallTypes.atext_completion, + CallTypes.text_completion, + ], + "/openai/deployments/{model}/completions": [ + CallTypes.atext_completion, + CallTypes.text_completion, + ], + # Embeddings + "/embeddings": [CallTypes.aembedding, CallTypes.embedding], + "/v1/embeddings": [CallTypes.aembedding, CallTypes.embedding], + "/engines/{model}/embeddings": [CallTypes.aembedding, CallTypes.embedding], + "/openai/deployments/{model}/embeddings": [ + CallTypes.aembedding, + CallTypes.embedding, + ], + # Image Generation + "/images/generations": [CallTypes.aimage_generation, CallTypes.image_generation], + "/v1/images/generations": [CallTypes.aimage_generation, CallTypes.image_generation], + "/engines/{model}/images/generations": [ + CallTypes.aimage_generation, + CallTypes.image_generation, + ], + "/openai/deployments/{model}/images/generations": [ + CallTypes.aimage_generation, + CallTypes.image_generation, + ], + # Image Edits + "/images/edits": [CallTypes.aimage_edit, CallTypes.image_edit], + "/v1/images/edits": [CallTypes.aimage_edit, CallTypes.image_edit], + # Audio Transcriptions + "/audio/transcriptions": [CallTypes.atranscription, CallTypes.transcription], + "/v1/audio/transcriptions": [CallTypes.atranscription, CallTypes.transcription], + # Audio Speech + "/audio/speech": [CallTypes.aspeech, CallTypes.speech], + "/v1/audio/speech": [CallTypes.aspeech, CallTypes.speech], + # Moderations + "/moderations": [CallTypes.amoderation, CallTypes.moderation], + "/v1/moderations": [CallTypes.amoderation, CallTypes.moderation], + # Rerank + "/rerank": [CallTypes.arerank, CallTypes.rerank], + "/v1/rerank": [CallTypes.arerank, CallTypes.rerank], + "/v2/rerank": [CallTypes.arerank, CallTypes.rerank], + # Search + "/search": [CallTypes.asearch, CallTypes.search], + "/v1/search": [CallTypes.asearch, CallTypes.search], + # Batches + "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], + "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], + "/batches/{batch_id}": [CallTypes.aretrieve_batch, CallTypes.retrieve_batch], + "/v1/batches/{batch_id}": [CallTypes.aretrieve_batch, CallTypes.retrieve_batch], + # Files + "/files": [ + CallTypes.acreate_file, + CallTypes.create_file, + CallTypes.afile_list, + CallTypes.file_list, + ], + "/v1/files": [ + CallTypes.acreate_file, + CallTypes.create_file, + CallTypes.afile_list, + CallTypes.file_list, + ], + "/files/{file_id}": [ + CallTypes.afile_retrieve, + CallTypes.file_retrieve, + CallTypes.afile_delete, + CallTypes.file_delete, + ], + "/v1/files/{file_id}": [ + CallTypes.afile_retrieve, + CallTypes.file_retrieve, + CallTypes.afile_delete, + CallTypes.file_delete, + ], + "/files/{file_id}/content": [CallTypes.afile_content, CallTypes.file_content], + "/v1/files/{file_id}/content": [CallTypes.afile_content, CallTypes.file_content], + # Assistants + "/assistants": [ + CallTypes.aget_assistants, + CallTypes.get_assistants, + CallTypes.acreate_assistants, + CallTypes.create_assistants, + ], + "/v1/assistants": [ + CallTypes.aget_assistants, + CallTypes.get_assistants, + CallTypes.acreate_assistants, + CallTypes.create_assistants, + ], + "/assistants/{assistant_id}": [ + CallTypes.adelete_assistant, + CallTypes.delete_assistant, + ], + "/v1/assistants/{assistant_id}": [ + CallTypes.adelete_assistant, + CallTypes.delete_assistant, + ], + # Threads + "/threads": [CallTypes.acreate_thread, CallTypes.create_thread], + "/v1/threads": [CallTypes.acreate_thread, CallTypes.create_thread], + "/threads/{thread_id}": [CallTypes.aget_thread, CallTypes.get_thread], + "/v1/threads/{thread_id}": [CallTypes.aget_thread, CallTypes.get_thread], + # Thread Messages + "/threads/{thread_id}/messages": [ + CallTypes.a_add_message, + CallTypes.add_message, + CallTypes.aget_messages, + CallTypes.get_messages, + ], + "/v1/threads/{thread_id}/messages": [ + CallTypes.a_add_message, + CallTypes.add_message, + CallTypes.aget_messages, + CallTypes.get_messages, + ], + # Thread Runs + "/threads/{thread_id}/runs": [ + CallTypes.arun_thread, + CallTypes.run_thread, + CallTypes.arun_thread_stream, + CallTypes.run_thread_stream, + ], + "/v1/threads/{thread_id}/runs": [ + CallTypes.arun_thread, + CallTypes.run_thread, + CallTypes.arun_thread_stream, + CallTypes.run_thread_stream, + ], + # Fine-tuning Jobs + "/fine_tuning/jobs": [ + CallTypes.acreate_fine_tuning_job, + CallTypes.create_fine_tuning_job, + CallTypes.alist_fine_tuning_jobs, + CallTypes.list_fine_tuning_jobs, + ], + "/v1/fine_tuning/jobs": [ + CallTypes.acreate_fine_tuning_job, + CallTypes.create_fine_tuning_job, + CallTypes.alist_fine_tuning_jobs, + CallTypes.list_fine_tuning_jobs, + ], + "/fine_tuning/jobs/{fine_tuning_job_id}": [ + CallTypes.aretrieve_fine_tuning_job, + CallTypes.retrieve_fine_tuning_job, + ], + "/v1/fine_tuning/jobs/{fine_tuning_job_id}": [ + CallTypes.aretrieve_fine_tuning_job, + CallTypes.retrieve_fine_tuning_job, + ], + "/fine_tuning/jobs/{fine_tuning_job_id}/cancel": [ + CallTypes.acancel_fine_tuning_job, + CallTypes.cancel_fine_tuning_job, + ], + "/v1/fine_tuning/jobs/{fine_tuning_job_id}/cancel": [ + CallTypes.acancel_fine_tuning_job, + CallTypes.cancel_fine_tuning_job, + ], + # Video Generation + "/videos": [ + CallTypes.acreate_video, + CallTypes.create_video, + CallTypes.avideo_list, + CallTypes.video_list, + ], + "/v1/videos": [ + CallTypes.acreate_video, + CallTypes.create_video, + CallTypes.avideo_list, + CallTypes.video_list, + ], + "/videos/{video_id}": [ + CallTypes.avideo_retrieve, + CallTypes.video_retrieve, + CallTypes.avideo_delete, + CallTypes.video_delete, + ], + "/v1/videos/{video_id}": [ + CallTypes.avideo_retrieve, + CallTypes.video_retrieve, + CallTypes.avideo_delete, + CallTypes.video_delete, + ], + "/videos/{video_id}/content": [CallTypes.avideo_content, CallTypes.video_content], + "/v1/videos/{video_id}/content": [ + CallTypes.avideo_content, + CallTypes.video_content, + ], + "/videos/{video_id}/remix": [CallTypes.avideo_remix, CallTypes.video_remix], + "/v1/videos/{video_id}/remix": [CallTypes.avideo_remix, CallTypes.video_remix], + # Vector Stores + "/vector_stores": [CallTypes.avector_store_create, CallTypes.vector_store_create], + "/v1/vector_stores": [ + CallTypes.avector_store_create, + CallTypes.vector_store_create, + ], + "/vector_stores/{vector_store_id}/search": [ + CallTypes.avector_store_search, + CallTypes.vector_store_search, + ], + "/v1/vector_stores/{vector_store_id}/search": [ + CallTypes.avector_store_search, + CallTypes.vector_store_search, + ], + "/vector_stores/{vector_store_id}/files": [ + CallTypes.avector_store_file_create, + CallTypes.vector_store_file_create, + CallTypes.avector_store_file_list, + CallTypes.vector_store_file_list, + ], + "/v1/vector_stores/{vector_store_id}/files": [ + CallTypes.avector_store_file_create, + CallTypes.vector_store_file_create, + CallTypes.avector_store_file_list, + CallTypes.vector_store_file_list, + ], + "/vector_stores/{vector_store_id}/files/{file_id}": [ + CallTypes.avector_store_file_retrieve, + CallTypes.vector_store_file_retrieve, + CallTypes.avector_store_file_delete, + CallTypes.vector_store_file_delete, + ], + "/v1/vector_stores/{vector_store_id}/files/{file_id}": [ + CallTypes.avector_store_file_retrieve, + CallTypes.vector_store_file_retrieve, + CallTypes.avector_store_file_delete, + CallTypes.vector_store_file_delete, + ], + "/vector_stores/{vector_store_id}/files/{file_id}/content": [ + CallTypes.avector_store_file_content, + CallTypes.vector_store_file_content, + ], + "/v1/vector_stores/{vector_store_id}/files/{file_id}/content": [ + CallTypes.avector_store_file_content, + CallTypes.vector_store_file_content, + ], + "/vector_stores/{vector_store_id}/files/{file_id}/update": [ + CallTypes.avector_store_file_update, + CallTypes.vector_store_file_update, + ], + "/v1/vector_stores/{vector_store_id}/files/{file_id}/update": [ + CallTypes.avector_store_file_update, + CallTypes.vector_store_file_update, + ], + # Containers + "/containers": [ + CallTypes.acreate_container, + CallTypes.create_container, + CallTypes.alist_containers, + CallTypes.list_containers, + ], + "/v1/containers": [ + CallTypes.acreate_container, + CallTypes.create_container, + CallTypes.alist_containers, + CallTypes.list_containers, + ], + "/containers/{container_id}": [ + CallTypes.aretrieve_container, + CallTypes.retrieve_container, + CallTypes.adelete_container, + CallTypes.delete_container, + ], + "/v1/containers/{container_id}": [ + CallTypes.aretrieve_container, + CallTypes.retrieve_container, + CallTypes.adelete_container, + CallTypes.delete_container, + ], + # Responses API + "/responses": [CallTypes.aresponses, CallTypes.responses], + "/v1/responses": [CallTypes.aresponses, CallTypes.responses], + "/responses/{response_id}": [CallTypes.aresponses, CallTypes.responses], + "/v1/responses/{response_id}": [CallTypes.aresponses, CallTypes.responses], + "/responses/{response_id}/input_items": [CallTypes.alist_input_items], + "/v1/responses/{response_id}/input_items": [CallTypes.alist_input_items], + # Realtime API + "/realtime": [CallTypes.arealtime], + "/v1/realtime": [CallTypes.arealtime], + # Provider-specific routes + "/anthropic/v1/messages": [CallTypes.anthropic_messages], + # Google GenAI routes + "/generate_content": [CallTypes.agenerate_content, CallTypes.generate_content], + "/models/{model}:generateContent": [ + CallTypes.agenerate_content, + CallTypes.generate_content, + ], + "/generate_content_stream": [ + CallTypes.agenerate_content_stream, + CallTypes.generate_content_stream, + ], + "/models/{model}:streamGenerateContent": [ + CallTypes.agenerate_content_stream, + CallTypes.generate_content_stream, + ], + # MCP (Model Context Protocol) + "/mcp/call_tool": [CallTypes.call_mcp_tool], + # Passthrough endpoints + "/llm_passthrough": [ + CallTypes.llm_passthrough_route, + CallTypes.allm_passthrough_route, + ], + "/v1/llm_passthrough": [ + CallTypes.llm_passthrough_route, + CallTypes.allm_passthrough_route, + ], +} + class PassthroughCallTypes(Enum): passthrough_image_generation = "passthrough-image-generation" @@ -1060,7 +1382,10 @@ class Usage(CompletionUsage): # Auto-calculate text_tokens only if provider didn't set it explicitly # Formula: text_tokens = completion_tokens - reasoning_tokens - image_tokens - audio_tokens - if _completion_tokens_details.text_tokens is None and completion_tokens is not None: + if ( + _completion_tokens_details.text_tokens is None + and completion_tokens is not None + ): calculated_text_tokens = completion_tokens - reasoning_tokens # Subtract other modality tokens if present diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py index 186056dac90..7ce99abdd15 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_apply_guardrail_endpoint.py @@ -1,6 +1,7 @@ """ Test the /guardrails/apply_guardrail endpoint """ + import os import sys from unittest.mock import AsyncMock, Mock, patch @@ -22,37 +23,45 @@ async def test_apply_guardrail_endpoint_returns_correct_response(): from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY") as mock_registry: + with patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry: # Create a mock guardrail mock_guardrail = Mock(spec=CustomGuardrail) - mock_guardrail.apply_guardrail = AsyncMock(return_value="Redacted text: [REDACTED] and [REDACTED]") - + # Apply guardrail now returns a tuple (List[str], Optional[List[str]]) + mock_guardrail.apply_guardrail = AsyncMock( + return_value=(["Redacted text: [REDACTED] and [REDACTED]"], None) + ) + # Configure the registry to return our mock guardrail mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail - + # Create the request request = ApplyGuardrailRequest( guardrail_name="test-guardrail", text="Test text with PII", language="en", - entities=["EMAIL_ADDRESS", "PERSON"] + entities=["EMAIL_ADDRESS", "PERSON"], ) - + # Create a mock user API key user_api_key_dict = UserAPIKeyAuth(api_key="test-key") - + # Call the endpoint - response = await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) - + response = await apply_guardrail( + request=request, user_api_key_dict=user_api_key_dict + ) + # Verify the response is of the correct type assert isinstance(response, ApplyGuardrailResponse) assert response.response_text == "Redacted text: [REDACTED] and [REDACTED]" - - # Verify the guardrail was called with correct parameters + + # Verify the guardrail was called with correct parameters (new signature) mock_guardrail.apply_guardrail.assert_called_once_with( - text="Test text with PII", - language="en", - entities=["EMAIL_ADDRESS", "PERSON"] + texts=["Test text with PII"], + request_data={}, + input_type="request", + images=None, ) @@ -63,23 +72,23 @@ async def test_apply_guardrail_endpoint_guardrail_not_found(): from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry to return None - with patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY") as mock_registry: + with patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry: mock_registry.get_initialized_guardrail_callback.return_value = None - + # Create the request request = ApplyGuardrailRequest( - guardrail_name="non-existent-guardrail", - text="Test text", - language="en" + guardrail_name="non-existent-guardrail", text="Test text", language="en" ) - + # Create a mock user API key user_api_key_dict = UserAPIKeyAuth(api_key="test-key") - + # Verify exception is raised with pytest.raises(ProxyException) as exc_info: await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) - + assert "non-existent-guardrail" in exc_info.value.message assert "not found" in exc_info.value.message @@ -90,34 +99,41 @@ async def test_apply_guardrail_endpoint_with_presidio_guardrail(): from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY") as mock_registry: + with patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry: # Create a mock guardrail that simulates Presidio behavior mock_guardrail = Mock(spec=CustomGuardrail) - # Simulate masking PII entities + # Simulate masking PII entities - returns tuple (List[str], Optional[List[str]]) mock_guardrail.apply_guardrail = AsyncMock( - return_value="My name is [PERSON] and my email is [EMAIL_ADDRESS]" + return_value=(["My name is [PERSON] and my email is [EMAIL_ADDRESS]"], None) ) - + # Configure the registry to return our mock guardrail mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail - + # Create the request request = ApplyGuardrailRequest( guardrail_name="pii-detection-guard", text="My name is John Doe and my email is john@example.com", language="en", - entities=["EMAIL_ADDRESS", "PERSON"] + entities=["EMAIL_ADDRESS", "PERSON"], ) - + # Create a mock user API key user_api_key_dict = UserAPIKeyAuth(api_key="test-key") - + # Call the endpoint - response = await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) - + response = await apply_guardrail( + request=request, user_api_key_dict=user_api_key_dict + ) + # Verify the response is of the correct type assert isinstance(response, ApplyGuardrailResponse) - assert response.response_text == "My name is [PERSON] and my email is [EMAIL_ADDRESS]" + assert ( + response.response_text + == "My name is [PERSON] and my email is [EMAIL_ADDRESS]" + ) assert "john@example.com" not in response.response_text assert "John Doe" not in response.response_text @@ -128,33 +144,37 @@ async def test_apply_guardrail_endpoint_without_optional_params(): from litellm.proxy.guardrails.guardrail_endpoints import apply_guardrail # Mock the guardrail registry - with patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY") as mock_registry: + with patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry: # Create a mock guardrail mock_guardrail = Mock(spec=CustomGuardrail) - mock_guardrail.apply_guardrail = AsyncMock(return_value="Processed text") - + # Returns tuple (List[str], Optional[List[str]]) + mock_guardrail.apply_guardrail = AsyncMock( + return_value=(["Processed text"], None) + ) + # Configure the registry to return our mock guardrail mock_registry.get_initialized_guardrail_callback.return_value = mock_guardrail - + # Create the request without optional parameters request = ApplyGuardrailRequest( - guardrail_name="test-guardrail", - text="Test text" + guardrail_name="test-guardrail", text="Test text" ) - + # Create a mock user API key user_api_key_dict = UserAPIKeyAuth(api_key="test-key") - + # Call the endpoint - response = await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) - + response = await apply_guardrail( + request=request, user_api_key_dict=user_api_key_dict + ) + # Verify the response is of the correct type assert isinstance(response, ApplyGuardrailResponse) assert response.response_text == "Processed text" - - # Verify the guardrail was called with None for optional parameters + + # Verify the guardrail was called with new signature mock_guardrail.apply_guardrail.assert_called_once_with( - text="Test text", - language=None, - entities=None + texts=["Test text"], request_data={}, input_type="request", images=None ) diff --git a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py index e65d01e41e4..203bd05c57c 100644 --- a/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py +++ b/tests/enterprise/litellm_enterprise/proxy/guardrails/test_bedrock_apply_guardrail.py @@ -1,6 +1,7 @@ """ Test the Bedrock guardrail apply_guardrail functionality """ + import os import sys from unittest.mock import AsyncMock, Mock, patch @@ -23,32 +24,29 @@ async def test_bedrock_apply_guardrail_success(): guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", guardrailIdentifier="test-guard-id", - guardrailVersion="DRAFT" + guardrailVersion="DRAFT", ) - + # Mock the make_bedrock_api_request method - with patch.object(guardrail, 'make_bedrock_api_request', new_callable=AsyncMock) as mock_api_request: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api_request: # Mock a successful response from Bedrock mock_response = { "action": "ALLOWED", - "content": [ - { - "text": { - "text": "This is a test message with some content" - } - } - ] + "content": [{"text": {"text": "This is a test message with some content"}}], } mock_api_request.return_value = mock_response - - # Test the apply_guardrail method - result = await guardrail.apply_guardrail( - text="This is a test message with some content", - language="en" + + # Test the apply_guardrail method with new signature + result, _ = await guardrail.apply_guardrail( + texts=["This is a test message with some content"], + request_data={}, + input_type="request", ) - + # Verify the result - assert result == "This is a test message with some content" + assert result == ["This is a test message with some content"] mock_api_request.assert_called_once() @@ -59,25 +57,23 @@ async def test_bedrock_apply_guardrail_blocked(): guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", guardrailIdentifier="test-guard-id", - guardrailVersion="DRAFT" + guardrailVersion="DRAFT", ) - + # Mock the make_bedrock_api_request method - with patch.object(guardrail, 'make_bedrock_api_request', new_callable=AsyncMock) as mock_api_request: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api_request: # Mock a blocked response from Bedrock - mock_response = { - "action": "BLOCKED", - "reason": "Content violates policy" - } + mock_response = {"action": "BLOCKED", "reason": "Content violates policy"} mock_api_request.return_value = mock_response - + # Test the apply_guardrail method should raise an exception with pytest.raises(Exception) as exc_info: await guardrail.apply_guardrail( - text="This is blocked content", - language="en" + texts=["This is blocked content"], request_data={}, input_type="request" ) - + assert "Content blocked by Bedrock guardrail" in str(exc_info.value) assert "Content violates policy" in str(exc_info.value) @@ -89,30 +85,29 @@ async def test_bedrock_apply_guardrail_with_masking(): guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", guardrailIdentifier="test-guard-id", - guardrailVersion="DRAFT" + guardrailVersion="DRAFT", ) - + # Mock the make_bedrock_api_request method - with patch.object(guardrail, 'make_bedrock_api_request', new_callable=AsyncMock) as mock_api_request: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api_request: # Mock a response with masked content mock_response = { "action": "ALLOWED", - "outputs": [ - { - "text": "This is a test message with [REDACTED] content" - } - ] + "outputs": [{"text": "This is a test message with [REDACTED] content"}], } mock_api_request.return_value = mock_response - - # Test the apply_guardrail method - result = await guardrail.apply_guardrail( - text="This is a test message with sensitive content", - language="en" + + # Test the apply_guardrail method with new signature + result, _ = await guardrail.apply_guardrail( + texts=["This is a test message with sensitive content"], + request_data={}, + input_type="request", ) - + # Verify the result contains the masked content - assert result == "This is a test message with [REDACTED] content" + assert result == ["This is a test message with [REDACTED] content"] mock_api_request.assert_called_once() @@ -123,21 +118,22 @@ async def test_bedrock_apply_guardrail_api_failure(): guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", guardrailIdentifier="test-guard-id", - guardrailVersion="DRAFT" + guardrailVersion="DRAFT", ) - + # Mock the make_bedrock_api_request method to raise an exception - with patch.object(guardrail, 'make_bedrock_api_request', new_callable=AsyncMock) as mock_api_request: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api_request: mock_api_request.side_effect = Exception("API connection failed") - + # Test the apply_guardrail method should raise an exception with pytest.raises(Exception) as exc_info: await guardrail.apply_guardrail( - text="This is a test message", - language="en" + texts=["This is a test message"], request_data={}, input_type="request" ) - - assert "Bedrock guardrail failed" in str(exc_info.value) + + # The error message should contain the original exception assert "API connection failed" in str(exc_info.value) @@ -150,44 +146,50 @@ async def test_bedrock_apply_guardrail_endpoint_integration(): guardrail = BedrockGuardrail( guardrail_name="test-bedrock-guard", guardrailIdentifier="test-guard-id", - guardrailVersion="DRAFT" + guardrailVersion="DRAFT", ) - + # Mock the guardrail registry - with patch("litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY") as mock_registry: + with patch( + "litellm.proxy.guardrails.guardrail_endpoints.GUARDRAIL_REGISTRY" + ) as mock_registry: # Mock the make_bedrock_api_request method - with patch.object(guardrail, 'make_bedrock_api_request', new_callable=AsyncMock) as mock_api_request: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api_request: # Mock a successful response from Bedrock mock_response = { "action": "ALLOWED", - "outputs": [ - { - "text": "This is a test message with processed content" - } - ] + "outputs": [{"text": "This is a test message with processed content"}], } mock_api_request.return_value = mock_response - + # Configure the registry to return our guardrail mock_registry.get_initialized_guardrail_callback.return_value = guardrail - + # Create the request request = ApplyGuardrailRequest( guardrail_name="test-bedrock-guard", text="This is a test message with some content", - language="en" + language="en", ) - + # Create a mock user API key user_api_key_dict = UserAPIKeyAuth(api_key="test-key") - + # Call the endpoint - response = await apply_guardrail(request=request, user_api_key_dict=user_api_key_dict) - + response = await apply_guardrail( + request=request, user_api_key_dict=user_api_key_dict + ) + # Verify the response assert isinstance(response, ApplyGuardrailResponse) - assert response.response_text == "This is a test message with processed content" - mock_api_request.assert_called_once() + assert ( + response.response_text + == "This is a test message with processed content" + ) + # Note: The endpoint now calls apply_guardrail which internally calls make_bedrock_api_request + # The call count check has been removed as it may be called multiple times through the chain @pytest.mark.asyncio @@ -208,18 +210,21 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable request_data = {"messages": request_messages} - with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api: mock_api.return_value = {"action": "ALLOWED"} - result = await guardrail.apply_guardrail( - text="latest question", + result, _ = await guardrail.apply_guardrail( + texts=["latest question"], request_data=request_data, + input_type="request", ) assert mock_api.called _, kwargs = mock_api.call_args assert kwargs["messages"] == [request_messages[-1]] - assert result == "latest question" + assert result == ["latest question"] @pytest.mark.asyncio @@ -238,19 +243,23 @@ async def test_bedrock_apply_guardrail_filters_request_messages_when_flag_enable request_data = {"messages": request_messages} - with patch.object(guardrail, "make_bedrock_api_request", new_callable=AsyncMock) as mock_api: + with patch.object( + guardrail, "make_bedrock_api_request", new_callable=AsyncMock + ) as mock_api: mock_api.return_value = {"action": "BLOCKED", "reason": "policy"} with pytest.raises(Exception, match="policy") as exc_info: await guardrail.apply_guardrail( - text="blocked", + texts=["blocked"], request_data=request_data, + input_type="request", ) assert mock_api.called _, kwargs = mock_api.call_args assert kwargs["messages"] == [request_messages[-1]] - assert "Bedrock guardrail failed" in str(exc_info.value) + assert "Content blocked by Bedrock guardrail" in str(exc_info.value) + def test_bedrock_guardrail_filters_latest_user_message_when_enabled(): guardrail = BedrockGuardrail( diff --git a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py index 8fdc09fc752..9c2bbeb7a68 100644 --- a/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py +++ b/tests/test_litellm/llms/cohere/rerank/test_rerank_guardrail_handler.py @@ -5,6 +5,7 @@ Unit tests for Cohere Rerank Guardrail Translation Handler import asyncio import os import sys +from typing import List, Optional, Tuple import pytest @@ -20,8 +21,10 @@ from litellm.types.utils import CallTypes class MockGuardrail(CustomGuardrail): """Mock guardrail for testing""" - async def apply_guardrail(self, text: str, language=None, entities=None) -> str: - return f"{text} [GUARDRAILED]" + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: + return ([f"{text} [GUARDRAILED]" for text in texts], None) class TestHandlerDiscovery: @@ -183,17 +186,20 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: import re - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - masked = masked.replace("John Doe", "[NAME_REDACTED]") - return masked + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + masked = masked.replace("John Doe", "[NAME_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = CohereRerankHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -231,21 +237,24 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: import re - # Mask emails - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - # Mask phone numbers - masked = re.sub(r"\d{3}-\d{3}-\d{4}", "[PHONE_REDACTED]", masked) - # Mask names - masked = masked.replace("Alice Smith", "[NAME_REDACTED]") - return masked + masked_texts = [] + for text in texts: + # Mask emails + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + # Mask phone numbers + masked = re.sub(r"\d{3}-\d{3}-\d{4}", "[PHONE_REDACTED]", masked) + # Mask names + masked = masked.replace("Alice Smith", "[NAME_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = CohereRerankHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -340,13 +349,16 @@ class TestContentFilteringScenario: """Mock content filter guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: bad_words = ["inappropriate", "offensive"] - filtered = text - for word in bad_words: - filtered = filtered.replace(word, "[FILTERED]") - return filtered + filtered_texts = [] + for text in texts: + filtered = text + for word in bad_words: + filtered = filtered.replace(word, "[FILTERED]") + filtered_texts.append(filtered) + return (filtered_texts, None) handler = CohereRerankHandler() guardrail = ContentFilterGuardrail(guardrail_name="content_filter") diff --git a/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py b/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py index 97ca423f773..c861e48ad48 100644 --- a/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/completion/test_text_completion_guardrail_handler.py @@ -4,6 +4,7 @@ Unit tests for OpenAI Text Completion Guardrail Translation Handler import os import sys +from typing import List, Optional, Tuple from unittest.mock import MagicMock import pytest @@ -21,8 +22,10 @@ from litellm.types.utils import CallTypes, TextChoices, TextCompletionResponse class MockGuardrail(CustomGuardrail): """Mock guardrail for testing""" - async def apply_guardrail(self, text: str, language=None, entities=None) -> str: - return f"{text} [GUARDRAILED]" + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: + return ([f"{text} [GUARDRAILED]" for text in texts], None) class TestHandlerDiscovery: @@ -243,19 +246,22 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: replace email-like patterns import re - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - # Replace names (simple mock) - masked = masked.replace("John Doe", "[NAME_REDACTED]") - return masked + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + # Replace names (simple mock) + masked = masked.replace("John Doe", "[NAME_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAITextCompletionHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -303,15 +309,19 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: import re - return re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAITextCompletionHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") diff --git a/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py b/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py index 529d74f63d9..5e183e32208 100644 --- a/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/image_generation/test_image_generation_guardrail_handler.py @@ -4,6 +4,7 @@ Unit tests for OpenAI Image Generation Guardrail Translation Handler import os import sys +from typing import List, Optional, Tuple import pytest @@ -20,8 +21,10 @@ from litellm.types.utils import CallTypes, ImageObject, ImageResponse class MockGuardrail(CustomGuardrail): """Mock guardrail for testing""" - async def apply_guardrail(self, text: str, language=None, entities=None) -> str: - return f"{text} [GUARDRAILED]" + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: + return ([f"{text} [GUARDRAILED]" for text in texts], None) class TestHandlerDiscovery: @@ -141,19 +144,22 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: replace email-like patterns import re - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - # Replace names (simple mock) - masked = masked.replace("John Doe", "[NAME_REDACTED]") - return masked + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + # Replace names (simple mock) + masked = masked.replace("John Doe", "[NAME_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAIImageGenerationHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") diff --git a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py index b447c281aa3..c04f825e362 100644 --- a/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/responses/test_openai_responses_guardrail_handler.py @@ -7,7 +7,7 @@ with guardrail transformations. import os import sys -from typing import Any +from typing import Any, List, Optional, Tuple from unittest.mock import AsyncMock, MagicMock import pytest @@ -29,9 +29,11 @@ from litellm.types.utils import CallTypes class MockGuardrail(CustomGuardrail): """Mock guardrail for testing that transforms text""" - async def apply_guardrail(self, text: str) -> str: + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: """Append [GUARDRAILED] to text""" - return f"{text} [GUARDRAILED]" + return ([f"{text} [GUARDRAILED]" for text in texts], None) class TestOpenAIResponsesHandlerDiscovery: @@ -450,7 +452,10 @@ class TestOpenAIResponsesHandlerEdgeCases: "role": "user", "content": [ {"type": "text", "text": "List content"}, - {"type": "image_url", "image_url": {"url": "http://example.com"}}, + { + "type": "image_url", + "image_url": {"url": "http://example.com"}, + }, ], "type": "message", }, @@ -492,4 +497,3 @@ class TestOpenAIResponsesHandlerEdgeCases: # Should skip processing and return unchanged assert result == response - diff --git a/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py b/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py index b4064a22c17..dfd96beb2f4 100644 --- a/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/speech/test_text_to_speech_guardrail_handler.py @@ -4,6 +4,7 @@ Unit tests for OpenAI Text-to-Speech Guardrail Translation Handler import os import sys +from typing import List, Optional, Tuple import pytest @@ -20,8 +21,10 @@ from litellm.types.utils import CallTypes class MockGuardrail(CustomGuardrail): """Mock guardrail for testing""" - async def apply_guardrail(self, text: str, language=None, entities=None) -> str: - return f"{text} [GUARDRAILED]" + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: + return ([f"{text} [GUARDRAILED]" for text in texts], None) class MockBinaryResponse: @@ -169,20 +172,23 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: replace email-like patterns import re - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - # Replace names (simple mock) - masked = masked.replace("John Doe", "[NAME_REDACTED]") - masked = masked.replace("555-1234", "[PHONE_REDACTED]") - return masked + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + # Replace names (simple mock) + masked = masked.replace("John Doe", "[NAME_REDACTED]") + masked = masked.replace("555-1234", "[PHONE_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAITextToSpeechHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -211,17 +217,24 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: import re - # Mask account numbers - masked = re.sub(r"account number \d{8,12}", "account number [REDACTED]", text) - # Mask SSNs - masked = re.sub(r"\d{3}-\d{2}-\d{4}", "[SSN_REDACTED]", masked) - # Mask credit cards - masked = re.sub(r"\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}", "[CC_REDACTED]", masked) - return masked + masked_texts = [] + for text in texts: + # Mask account numbers + masked = re.sub( + r"account number \d{8,12}", "account number [REDACTED]", text + ) + # Mask SSNs + masked = re.sub(r"\d{3}-\d{2}-\d{4}", "[SSN_REDACTED]", masked) + # Mask credit cards + masked = re.sub( + r"\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}", "[CC_REDACTED]", masked + ) + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAITextToSpeechHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -256,14 +269,17 @@ class TestContentModerationScenario: """Mock content filter guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: filter inappropriate words bad_words = ["badword", "inappropriate", "offensive"] - filtered = text - for word in bad_words: - filtered = filtered.replace(word, "[FILTERED]") - return filtered + filtered_texts = [] + for text in texts: + filtered = text + for word in bad_words: + filtered = filtered.replace(word, "[FILTERED]") + filtered_texts.append(filtered) + return (filtered_texts, None) handler = OpenAITextToSpeechHandler() guardrail = ContentFilterGuardrail(guardrail_name="content_filter") @@ -322,4 +338,3 @@ class TestMultilingualTTS: assert f"Testing with {voice} voice [GUARDRAILED]" == result["input"] assert result["voice"] == voice - diff --git a/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py b/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py index faa425eb714..4d2cb142b35 100644 --- a/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py +++ b/tests/test_litellm/llms/openai/transcriptions/test_audio_transcription_guardrail_handler.py @@ -4,6 +4,7 @@ Unit tests for OpenAI Audio Transcription Guardrail Translation Handler import os import sys +from typing import List, Optional, Tuple import pytest @@ -21,8 +22,10 @@ from litellm.utils import TranscriptionResponse class MockGuardrail(CustomGuardrail): """Mock guardrail for testing""" - async def apply_guardrail(self, text: str, language=None, entities=None) -> str: - return f"{text} [GUARDRAILED]" + async def apply_guardrail( + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: + return ([f"{text} [GUARDRAILED]" for text in texts], None) class TestHandlerDiscovery: @@ -140,20 +143,23 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: replace email-like patterns import re - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - text, - ) - # Replace names (simple mock) - masked = masked.replace("John Doe", "[NAME_REDACTED]") - masked = masked.replace("555-1234", "[PHONE_REDACTED]") - return masked + masked_texts = [] + for text in texts: + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + text, + ) + # Replace names (simple mock) + masked = masked.replace("John Doe", "[NAME_REDACTED]") + masked = masked.replace("555-1234", "[PHONE_REDACTED]") + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAIAudioTranscriptionHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -181,23 +187,26 @@ class TestPIIMaskingScenario: """Mock PII masking guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: import re - # Mask credit card numbers - masked = re.sub( - r"\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}", "[CC_REDACTED]", text - ) - # Mask SSNs - masked = re.sub(r"\d{3}-\d{2}-\d{4}", "[SSN_REDACTED]", masked) - # Mask emails - masked = re.sub( - r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", - "[EMAIL_REDACTED]", - masked, - ) - return masked + masked_texts = [] + for text in texts: + # Mask credit card numbers + masked = re.sub( + r"\d{4}[- ]?\d{4}[- ]?\d{4}[- ]?\d{4}", "[CC_REDACTED]", text + ) + # Mask SSNs + masked = re.sub(r"\d{3}-\d{2}-\d{4}", "[SSN_REDACTED]", masked) + # Mask emails + masked = re.sub( + r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b", + "[EMAIL_REDACTED]", + masked, + ) + masked_texts.append(masked) + return (masked_texts, None) handler = OpenAIAudioTranscriptionHandler() guardrail = PIIMaskingGuardrail(guardrail_name="mask_pii") @@ -231,14 +240,17 @@ class TestContentModerationScenario: """Mock profanity filter guardrail""" async def apply_guardrail( - self, text: str, language=None, entities=None - ) -> str: + self, texts: List[str], request_data: dict, input_type: str, **kwargs + ) -> Tuple[List[str], Optional[List[str]]]: # Simple mock: replace common profanity bad_words = ["badword1", "badword2", "inappropriate"] - filtered = text - for word in bad_words: - filtered = filtered.replace(word, "[FILTERED]") - return filtered + filtered_texts = [] + for text in texts: + filtered = text + for word in bad_words: + filtered = filtered.replace(word, "[FILTERED]") + filtered_texts.append(filtered) + return (filtered_texts, None) handler = OpenAIAudioTranscriptionHandler() guardrail = ProfanityFilterGuardrail(guardrail_name="content_filter") diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py index e756dd6bd5d..265605c1637 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/content_filter/test_content_filter.py @@ -40,12 +40,12 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-content-filter", patterns=patterns, ) - + assert guardrail.guardrail_name == "test-content-filter" assert len(guardrail.compiled_patterns) == 1 @@ -57,19 +57,19 @@ class TestContentFilterGuardrail: BlockedWord( keyword="secret_project", action=ContentFilterAction.BLOCK, - description="Top secret project" + description="Top secret project", ), BlockedWord( keyword="internal_api", action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-content-filter", blocked_words=blocked_words, ) - + assert len(guardrail.blocked_words) == 2 assert "secret_project" in guardrail.blocked_words assert guardrail.blocked_words["secret_project"][0] == ContentFilterAction.BLOCK @@ -85,18 +85,18 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-ssn", patterns=patterns, ) - + # Test with SSN result = guardrail._check_patterns("My SSN is 123-45-6789") assert result is not None assert result[1] == "us_ssn" assert result[2] == ContentFilterAction.BLOCK - + # Test without SSN result = guardrail._check_patterns("This is a normal message") assert result is None @@ -112,12 +112,12 @@ class TestContentFilterGuardrail: action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-email", patterns=patterns, ) - + result = guardrail._check_patterns("Contact me at test@example.com") assert result is not None assert result[1] == "email" @@ -135,12 +135,12 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-custom", patterns=patterns, ) - + result = guardrail._check_patterns("My ID is ABC-1234") assert result is not None assert result[1] == "custom_id" @@ -155,18 +155,18 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-words", blocked_words=blocked_words, ) - + # Test with blocked word result = guardrail._check_blocked_words("This is CONFIDENTIAL information") assert result is not None assert result[0] == "confidential" assert result[1] == ContentFilterAction.BLOCK - + # Test without blocked word result = guardrail._check_blocked_words("This is normal information") assert result is None @@ -183,15 +183,17 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-block", patterns=patterns, ) - + with pytest.raises(HTTPException) as exc_info: - await guardrail.apply_guardrail(text="My SSN is 123-45-6789") - + await guardrail.apply_guardrail( + texts=["My SSN is 123-45-6789"], request_data={}, input_type="request" + ) + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) @@ -207,17 +209,22 @@ class TestContentFilterGuardrail: action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-mask", patterns=patterns, ) - - result = await guardrail.apply_guardrail(text="Contact me at test@example.com") - + + result, _ = await guardrail.apply_guardrail( + texts=["Contact me at test@example.com"], + request_data={}, + input_type="request", + ) + assert result is not None - assert "[EMAIL_REDACTED]" in result - assert "test@example.com" not in result + assert len(result) == 1 + assert "[EMAIL_REDACTED]" in result[0] + assert "test@example.com" not in result[0] @pytest.mark.asyncio async def test_apply_guardrail_blocked_word_mask(self): @@ -230,17 +237,22 @@ class TestContentFilterGuardrail: action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-word-mask", blocked_words=blocked_words, ) - - result = await guardrail.apply_guardrail(text="This is PROPRIETARY information") - + + result, _ = await guardrail.apply_guardrail( + texts=["This is PROPRIETARY information"], + request_data={}, + input_type="request", + ) + assert result is not None - assert "[KEYWORD_REDACTED]" in result - assert "PROPRIETARY" not in result + assert len(result) == 1 + assert "[KEYWORD_REDACTED]" in result[0] + assert "PROPRIETARY" not in result[0] @pytest.mark.asyncio async def test_apply_guardrail_multiple_patterns(self): @@ -259,19 +271,22 @@ class TestContentFilterGuardrail: action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-multiple", patterns=patterns, ) - - result = await guardrail.apply_guardrail( - text="Contact user@test.com or SSN: 123-45-6789" + + result, _ = await guardrail.apply_guardrail( + texts=["Contact user@test.com or SSN: 123-45-6789"], + request_data={}, + input_type="request", ) - + assert result is not None + assert len(result) == 1 # At least one pattern should be redacted (first match wins) - assert "[EMAIL_REDACTED]" in result or "[US_SSN_REDACTED]" in result + assert "[EMAIL_REDACTED]" in result[0] or "[US_SSN_REDACTED]" in result[0] def test_mask_content(self): """ @@ -280,7 +295,7 @@ class TestContentFilterGuardrail: guardrail = ContentFilterGuardrail( guardrail_name="test-mask", ) - + masked = guardrail._mask_content("sensitive text", "us_ssn") assert masked == "[US_SSN_REDACTED]" @@ -291,28 +306,34 @@ class TestContentFilterGuardrail: import tempfile # Create a temporary blocked words file - with tempfile.NamedTemporaryFile(mode='w', suffix='.yaml', delete=False) as f: - f.write("""blocked_words: + with tempfile.NamedTemporaryFile(mode="w", suffix=".yaml", delete=False) as f: + f.write( + """blocked_words: - keyword: "test_keyword" action: "BLOCK" description: "Test keyword" - keyword: "another_word" action: "MASK" -""") +""" + ) temp_file = f.name - + try: guardrail = ContentFilterGuardrail( guardrail_name="test-file-load", blocked_words_file=temp_file, ) - + assert len(guardrail.blocked_words) == 2 assert "test_keyword" in guardrail.blocked_words - assert guardrail.blocked_words["test_keyword"][0] == ContentFilterAction.BLOCK + assert ( + guardrail.blocked_words["test_keyword"][0] == ContentFilterAction.BLOCK + ) assert guardrail.blocked_words["test_keyword"][1] == "Test keyword" assert "another_word" in guardrail.blocked_words - assert guardrail.blocked_words["another_word"][0] == ContentFilterAction.MASK + assert ( + guardrail.blocked_words["another_word"][0] == ContentFilterAction.MASK + ) finally: os.unlink(temp_file) @@ -327,17 +348,17 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-cc", patterns=patterns, ) - + # Test Visa card result = guardrail._check_patterns("My card is 4532-1234-5678-9010") assert result is not None assert result[1] == "visa" - + def test_api_key_patterns(self): """ Test API key pattern detection @@ -349,12 +370,12 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-api-key", patterns=patterns, ) - + # Test AWS Access Key result = guardrail._check_patterns("My key is AKIAIOSFODNN7EXAMPLE") assert result is not None @@ -368,7 +389,7 @@ class TestContentFilterGuardrail: from unittest.mock import AsyncMock from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - + patterns = [ ContentFilterPattern( pattern_type="prebuilt", @@ -376,34 +397,40 @@ class TestContentFilterGuardrail: action=ContentFilterAction.MASK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-streaming-mask", patterns=patterns, event_hook=GuardrailEventHooks.during_call, ) - + # Create mock streaming chunks async def mock_stream(): # Chunk 1: contains email chunk1 = ModelResponseStream( id="chunk1", - choices=[StreamingChoices(delta=Delta(content="Contact me at test@example.com"), index=0)], + choices=[ + StreamingChoices( + delta=Delta(content="Contact me at test@example.com"), index=0 + ) + ], model="gpt-4", ) yield chunk1 - + # Chunk 2: normal content chunk2 = ModelResponseStream( id="chunk2", - choices=[StreamingChoices(delta=Delta(content=" for more info"), index=0)], + choices=[ + StreamingChoices(delta=Delta(content=" for more info"), index=0) + ], model="gpt-4", ) yield chunk2 - + user_api_key_dict = MagicMock() request_data = {} - + # Process streaming response result_chunks = [] async for chunk in guardrail.async_post_call_streaming_iterator_hook( @@ -412,7 +439,7 @@ class TestContentFilterGuardrail: request_data=request_data, ): result_chunks.append(chunk) - + assert len(result_chunks) == 2 # First chunk should have email masked assert "[EMAIL_REDACTED]" in result_chunks[0].choices[0].delta.content @@ -428,7 +455,7 @@ class TestContentFilterGuardrail: from unittest.mock import AsyncMock from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices - + patterns = [ ContentFilterPattern( pattern_type="prebuilt", @@ -436,25 +463,27 @@ class TestContentFilterGuardrail: action=ContentFilterAction.BLOCK, ), ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-streaming-block", patterns=patterns, event_hook=GuardrailEventHooks.during_call, ) - + # Create mock streaming chunks with SSN async def mock_stream(): chunk = ModelResponseStream( id="chunk1", - choices=[StreamingChoices(delta=Delta(content="SSN: 123-45-6789"), index=0)], + choices=[ + StreamingChoices(delta=Delta(content="SSN: 123-45-6789"), index=0) + ], model="gpt-4", ) yield chunk - + user_api_key_dict = MagicMock() request_data = {} - + # Should raise HTTPException when SSN is detected with pytest.raises(HTTPException) as exc_info: async for chunk in guardrail.async_post_call_streaming_iterator_hook( @@ -463,7 +492,7 @@ class TestContentFilterGuardrail: request_data=request_data, ): pass - + assert exc_info.value.status_code == 400 assert "us_ssn" in str(exc_info.value.detail) @@ -487,9 +516,9 @@ class TestContentFilterGuardrail: "action": "MASK", "name": "email", "pattern": None, - } + }, ] - + blocked_words = [ { "keyword": "langchain", @@ -500,19 +529,19 @@ class TestContentFilterGuardrail: "keyword": "openai", "action": "MASK", "description": "Competitor name", - } + }, ] - + guardrail = ContentFilterGuardrail( guardrail_name="test-db-format", patterns=patterns, blocked_words=blocked_words, ) - + assert guardrail.guardrail_name == "test-db-format" assert len(guardrail.compiled_patterns) == 2 assert len(guardrail.blocked_words) == 2 - + # Verify blocked_words are stored as dict assert "langchain" in guardrail.blocked_words assert guardrail.blocked_words["langchain"] == ("BLOCK", None) diff --git a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py index 9543b61ef69..3e19437fe83 100644 --- a/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/test_litellm/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -537,24 +537,25 @@ async def test_logging_hook_multiple_content_items(presidio_guardrail): async def test_presidio_sets_guardrail_information_in_request_data(): """ Test that Presidio populates guardrail information into request_data metadata. - + This validates that add_standard_logging_guardrail_information_to_request_data correctly sets the guardrail information that will be used for logging. """ presidio = _OPTIONAL_PresidioPIIMasking( guardrail_name="test_presidio", output_parse_pii=True, + mock_testing=True, ) - + request_data = { "messages": [{"role": "user", "content": "Test"}], "model": "gpt-4o", "metadata": {}, } - + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): assert request_data is not None - + presidio.add_standard_logging_guardrail_information_to_request_data( guardrail_provider="presidio", guardrail_json_response=[], @@ -565,27 +566,30 @@ async def test_presidio_sets_guardrail_information_in_request_data(): duration=1.0, masked_entity_count={"EMAIL_ADDRESS": 1, "PERSON": 1}, ) - + return text - - with patch.object(presidio, 'check_pii', mock_check_pii): + + with patch.object(presidio, "check_pii", mock_check_pii): await presidio.apply_guardrail( - text="Test message", + texts=["Test message"], request_data=request_data, + input_type="request", ) - + assert "metadata" in request_data assert "standard_logging_guardrail_information" in request_data["metadata"] - - guardrail_info_list = request_data["metadata"]["standard_logging_guardrail_information"] + + guardrail_info_list = request_data["metadata"][ + "standard_logging_guardrail_information" + ] assert isinstance(guardrail_info_list, list) assert len(guardrail_info_list) > 0 - + guardrail_info = guardrail_info_list[0] assert "masked_entity_count" in guardrail_info assert guardrail_info["masked_entity_count"]["EMAIL_ADDRESS"] == 1 assert guardrail_info["masked_entity_count"]["PERSON"] == 1 - + print("✓ Presidio sets guardrail_information in request_data") @@ -593,7 +597,7 @@ async def test_presidio_sets_guardrail_information_in_request_data(): async def test_request_data_flows_to_apply_guardrail(): """ Test that request_data is correctly passed to apply_guardrail method. - + This validates the fix where guardrail translation handler passes data as request_data to apply_guardrail so guardrails can store metadata for logging. """ @@ -601,31 +605,32 @@ async def test_request_data_flows_to_apply_guardrail(): guardrail_name="test_presidio", output_parse_pii=True, ) - + request_data = { "messages": [{"role": "user", "content": "Test message"}], "model": "gpt-4o", "metadata": {}, } - + async def mock_check_pii(text, output_parse_pii, presidio_config, request_data): assert request_data is not None, "request_data should be passed to check_pii" assert "metadata" in request_data, "request_data should have metadata" - + request_data.setdefault("metadata", {}) request_data["metadata"]["test_flag"] = "passed_correctly" - + return text - - with patch.object(presidio, 'check_pii', mock_check_pii): + + with patch.object(presidio, "check_pii", mock_check_pii): result = await presidio.apply_guardrail( - text="Test message", + texts=["Test message"], request_data=request_data, + input_type="request", ) - + assert "metadata" in request_data assert request_data["metadata"].get("test_flag") == "passed_correctly" - + print("✓ request_data correctly passed to apply_guardrail") From 8eaabb4ad765cb1c8c7c8d5df6e2b0dcaf4cd3c5 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 15:29:47 +0530 Subject: [PATCH 20/24] Add vector store support for ragflow --- .../docs/completion/knowledgebase.md | 1 + .../docs/providers/ragflow_vector_store.md | 349 +++++++++++++++++ docs/my-website/docs/vector_stores/create.md | 1 + .../llms/ragflow/vector_stores/__init__.py | 2 + .../ragflow/vector_stores/transformation.py | 249 ++++++++++++ litellm/types/vector_stores.py | 4 +- litellm/utils.py | 6 + ...test_vector_store_create_provider_logic.py | 44 ++- .../test_ragflow_vector_store.py | 359 ++++++++++++++++++ 9 files changed, 1010 insertions(+), 5 deletions(-) create mode 100644 docs/my-website/docs/providers/ragflow_vector_store.md create mode 100644 litellm/llms/ragflow/vector_stores/__init__.py create mode 100644 litellm/llms/ragflow/vector_stores/transformation.py create mode 100644 tests/vector_store_tests/test_ragflow_vector_store.py diff --git a/docs/my-website/docs/completion/knowledgebase.md b/docs/my-website/docs/completion/knowledgebase.md index 759f7912d87..fd6ef7a9982 100644 --- a/docs/my-website/docs/completion/knowledgebase.md +++ b/docs/my-website/docs/completion/knowledgebase.md @@ -21,6 +21,7 @@ LiteLLM integrates with vector stores, allowing your models to access your organ - [Azure Vector Stores](https://learn.microsoft.com/en-us/azure/ai-services/openai/how-to/file-search?tabs=python#vector-stores) (Cannot be directly queried. Only available for calling in Assistants messages. We will be adding Azure AI Search Vector Store API support soon.) - [Vertex AI RAG API](https://cloud.google.com/vertex-ai/generative-ai/docs/rag-overview) - [Gemini File Search](https://ai.google.dev/gemini-api/docs/file-search) +- [RAGFlow Datasets](/docs/providers/ragflow_vector_store.md) (Dataset management only, search not supported) ## Quick Start diff --git a/docs/my-website/docs/providers/ragflow_vector_store.md b/docs/my-website/docs/providers/ragflow_vector_store.md new file mode 100644 index 00000000000..bc014cacbe6 --- /dev/null +++ b/docs/my-website/docs/providers/ragflow_vector_store.md @@ -0,0 +1,349 @@ +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; +import Image from '@theme/IdealImage'; + +# RAGFlow Vector Stores + +Litellm support creation and management of datasets for document processing and knowledge base management in Ragflow. + +| Property | Details | +|----------|---------| +| Description | RAGFlow datasets enable document processing, chunking, and knowledge base management for RAG applications. | +| Provider Route on LiteLLM | `ragflow` in the litellm vector_store_registry | +| Provider Doc | [RAGFlow API Documentation ↗](https://ragflow.io/docs) | +| Supported Operations | Dataset Management (Create, List, Update, Delete) | +| Search/Retrieval | ❌ Not supported (management only) | + +## Quick Start + +### LiteLLM Python SDK + +```python showLineNumbers title="Example using LiteLLM Python SDK" +import os +import litellm + +# Set RAGFlow credentials +os.environ["RAGFLOW_API_KEY"] = "your-ragflow-api-key" +os.environ["RAGFLOW_API_BASE"] = "http://localhost:9380" # Optional, defaults to localhost:9380 + +# Create a RAGFlow dataset +response = litellm.vector_stores.create( + name="my-dataset", + custom_llm_provider="ragflow", + metadata={ + "description": "My knowledge base dataset", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "chunk_method": "naive" + } +) + +print(f"Created dataset ID: {response.id}") +print(f"Dataset name: {response.name}") +``` + +### LiteLLM Proxy + +#### 1. Configure your vector_store_registry + + + + +```yaml +model_list: + - model_name: gpt-4o-mini + litellm_params: + model: gpt-4o-mini + api_key: os.environ/OPENAI_API_KEY + +vector_store_registry: + - vector_store_name: "ragflow-knowledge-base" + litellm_params: + vector_store_id: "your-dataset-id" + custom_llm_provider: "ragflow" + api_key: os.environ/RAGFLOW_API_KEY + api_base: os.environ/RAGFLOW_API_BASE # Optional + vector_store_description: "RAGFlow dataset for knowledge base" + vector_store_metadata: + source: "Company documentation" +``` + + + + + +On the LiteLLM UI, Navigate to Experimental > Vector Stores > Create Vector Store. On this page you can create a vector store with a name, vector store id and credentials. + + + + + + +#### 2. Create a dataset via Proxy + + + + +```bash +curl http://localhost:4000/v1/vector_stores \ + -H "Content-Type: application/json" \ + -H "Authorization: Bearer $LITELLM_API_KEY" \ + -d '{ + "name": "my-ragflow-dataset", + "custom_llm_provider": "ragflow", + "metadata": { + "description": "Test dataset", + "chunk_method": "naive" + } + }' +``` + + + + + +```python +from openai import OpenAI + +# Initialize client with your LiteLLM proxy URL +client = OpenAI( + base_url="http://localhost:4000", + api_key="your-litellm-api-key" +) + +# Create a RAGFlow dataset +response = client.vector_stores.create( + name="my-ragflow-dataset", + custom_llm_provider="ragflow", + metadata={ + "description": "Test dataset", + "chunk_method": "naive" + } +) + +print(f"Created dataset: {response.id}") +``` + + + + +## Configuration + +### Environment Variables + +RAGFlow vector stores support configuration via environment variables: + +- `RAGFLOW_API_KEY` - Your RAGFlow API key (required) +- `RAGFLOW_API_BASE` - RAGFlow API base URL (optional, defaults to `http://localhost:9380`) + +### Parameters + +You can also pass these via `litellm_params`: + +- `api_key` - RAGFlow API key (overrides `RAGFLOW_API_KEY` env var) +- `api_base` - RAGFlow API base URL (overrides `RAGFLOW_API_BASE` env var) + +## Dataset Creation Options + +### Basic Dataset Creation + +```python +response = litellm.vector_stores.create( + name="basic-dataset", + custom_llm_provider="ragflow" +) +``` + +### Dataset with Chunk Method + +RAGFlow supports various chunk methods for different document types: + + + + +```python +response = litellm.vector_stores.create( + name="general-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "naive", + "parser_config": { + "chunk_token_num": 512, + "delimiter": "\n", + "html4excel": False, + "layout_recognize": "DeepDOC" + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="book-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "book", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="qa-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "qa", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + + +```python +response = litellm.vector_stores.create( + name="paper-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "paper", + "parser_config": { + "raptor": { + "use_raptor": False + } + } + } +) +``` + + + + +### Dataset with Ingestion Pipeline + +Instead of using a chunk method, you can use an ingestion pipeline: + +```python +response = litellm.vector_stores.create( + name="pipeline-dataset", + custom_llm_provider="ragflow", + metadata={ + "parse_type": 2, # Number of parsers in your pipeline + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005" # 32-character hex ID + } +) +``` + +**Note**: `chunk_method` and `pipeline_id` are mutually exclusive. Use one or the other. + +### Advanced Parser Configuration + +```python +response = litellm.vector_stores.create( + name="advanced-dataset", + custom_llm_provider="ragflow", + metadata={ + "chunk_method": "naive", + "description": "Advanced dataset with custom parser config", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "permission": "me", # or "team" + "parser_config": { + "chunk_token_num": 1024, + "delimiter": "\n!?;。；！？", + "html4excel": True, + "layout_recognize": "DeepDOC", + "auto_keywords": 5, + "auto_questions": 3, + "task_page_size": 12, + "raptor": { + "use_raptor": True + }, + "graphrag": { + "use_graphrag": False + } + } + } +) +``` + +## Supported Chunk Methods + +RAGFlow supports the following chunk methods: + +- `naive` - General purpose (default) +- `book` - For book documents +- `email` - For email documents +- `laws` - For legal documents +- `manual` - Manual chunking +- `one` - Single chunk +- `paper` - For academic papers +- `picture` - For image documents +- `presentation` - For presentation documents +- `qa` - Q&A format +- `table` - For table documents +- `tag` - Tag-based chunking + +## RAGFlow-Specific Parameters + +All RAGFlow-specific parameters should be passed via the `metadata` field: + +| Parameter | Type | Description | +|-----------|------|-------------| +| `avatar` | string | Base64 encoding of the avatar (max 65535 chars) | +| `description` | string | Brief description of the dataset (max 65535 chars) | +| `embedding_model` | string | Embedding model name (e.g., "BAAI/bge-large-zh-v1.5@BAAI") | +| `permission` | string | Access permission: "me" (default) or "team" | +| `chunk_method` | string | Chunking method (see supported methods above) | +| `parser_config` | object | Parser configuration (varies by chunk_method) | +| `parse_type` | int | Number of parsers in pipeline (required with pipeline_id) | +| `pipeline_id` | string | 32-character hex pipeline ID (required with parse_type) | + +## Error Handling + +RAGFlow returns error responses in the following format: + +```json +{ + "code": 101, + "message": "Dataset name 'my-dataset' already exists" +} +``` + +LiteLLM automatically maps these to appropriate exceptions: + +- `code != 0` → Raises exception with the error message +- Missing required fields → Raises `ValueError` +- Mutually exclusive parameters → Raises `ValueError` + +## Limitations + +- **Search/Retrieval**: RAGFlow vector stores support dataset management only. Search operations are not supported and will raise `NotImplementedError`. +- **List/Update/Delete**: These operations are not yet implemented through the standard vector store API. Use RAGFlow's native API endpoints directly. + +## Further Reading + +Vector Stores: +- [Vector Store Creation](../vector_stores/create.md) +- [Using Vector Stores with Completions](../completion/knowledgebase.md) +- [Vector Store Registry](../completion/knowledgebase.md#vectorstoreregistry) + diff --git a/docs/my-website/docs/vector_stores/create.md b/docs/my-website/docs/vector_stores/create.md index 19b4f39cd9e..7025c490a32 100644 --- a/docs/my-website/docs/vector_stores/create.md +++ b/docs/my-website/docs/vector_stores/create.md @@ -14,6 +14,7 @@ Create a vector store which can be used to store and search document chunks for | End-user Tracking | ✅ | | | Support LLM Providers (OpenAI `/vector_stores` API) | **OpenAI** | Full vector stores API support across providers | | Support LLM Providers (Passthrough API) | [**Azure AI**](/docs/providers/azure_ai/azure_ai_vector_stores_passthrough) | Full vector stores API support across providers | +| Support LLM Providers (Dataset Management) | [**RAGFlow**](/docs/providers/ragflow_vector_store.md) | Dataset creation and management (search not supported) | ## Usage diff --git a/litellm/llms/ragflow/vector_stores/__init__.py b/litellm/llms/ragflow/vector_stores/__init__.py new file mode 100644 index 00000000000..3be29310b39 --- /dev/null +++ b/litellm/llms/ragflow/vector_stores/__init__.py @@ -0,0 +1,2 @@ +# RAGFlow vector stores module + diff --git a/litellm/llms/ragflow/vector_stores/transformation.py b/litellm/llms/ragflow/vector_stores/transformation.py new file mode 100644 index 00000000000..b6401a4b8d7 --- /dev/null +++ b/litellm/llms/ragflow/vector_stores/transformation.py @@ -0,0 +1,249 @@ +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +import httpx + +from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig +from litellm.secret_managers.main import get_secret_str +from litellm.types.router import GenericLiteLLMParams +from litellm.types.vector_stores import ( + BaseVectorStoreAuthCredentials, + VectorStoreCreateOptionalRequestParams, + VectorStoreCreateResponse, + VectorStoreFileCounts, + VectorStoreIndexEndpoints, + VectorStoreSearchOptionalRequestParams, + VectorStoreSearchResponse, +) + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +else: + LiteLLMLoggingObj = Any + + +class RAGFlowVectorStoreConfig(BaseVectorStoreConfig): + """Vector store configuration for RAGFlow datasets.""" + + def get_auth_credentials( + self, litellm_params: dict + ) -> BaseVectorStoreAuthCredentials: + api_key = litellm_params.get("api_key") + if api_key is None: + # Try to get from environment variable + api_key = get_secret_str("RAGFLOW_API_KEY") + if api_key is None: + raise ValueError("api_key is required (set RAGFLOW_API_KEY env var or pass in litellm_params)") + return { + "headers": { + "Authorization": f"Bearer {api_key}", + }, + } + + def get_vector_store_endpoints_by_type(self) -> VectorStoreIndexEndpoints: + """RAGFlow vector stores are management-only, no search support.""" + return { + "read": [], + "write": [], + } + + def validate_environment( + self, headers: dict, litellm_params: Optional[GenericLiteLLMParams] + ) -> dict: + """Validate environment and set headers for RAGFlow API.""" + litellm_params = litellm_params or GenericLiteLLMParams() + api_key = ( + litellm_params.api_key + or get_secret_str("RAGFLOW_API_KEY") + ) + + if api_key is None: + raise ValueError("RAGFLOW_API_KEY is required (set env var or pass in litellm_params)") + + headers.update( + { + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + } + ) + return headers + + def get_complete_url( + self, + api_base: Optional[str], + litellm_params: dict, + ) -> str: + """ + Get the complete URL for RAGFlow datasets API. + + Supports: + - RAGFLOW_API_BASE env var + - api_base in litellm_params + - Default: http://localhost:9380 + """ + api_base = ( + api_base + or litellm_params.get("api_base") + or get_secret_str("RAGFLOW_API_BASE") + or "http://localhost:9380" + ) + + # Remove trailing slashes + api_base = api_base.rstrip("/") + + # RAGFlow datasets API endpoint + return f"{api_base}/api/v1/datasets" + + def transform_search_vector_store_request( + self, + vector_store_id: str, + query: Union[str, List[str]], + vector_store_search_optional_params: VectorStoreSearchOptionalRequestParams, + api_base: str, + litellm_logging_obj: LiteLLMLoggingObj, + litellm_params: dict, + ) -> Tuple[str, Dict]: + """RAGFlow vector stores are management-only, search is not supported.""" + raise NotImplementedError( + "RAGFlow vector stores support dataset management only, not search/retrieval" + ) + + def transform_search_vector_store_response( + self, response: httpx.Response, litellm_logging_obj: LiteLLMLoggingObj + ) -> VectorStoreSearchResponse: + """RAGFlow vector stores are management-only, search is not supported.""" + raise NotImplementedError( + "RAGFlow vector stores support dataset management only, not search/retrieval" + ) + + def transform_create_vector_store_request( + self, + vector_store_create_optional_params: VectorStoreCreateOptionalRequestParams, + api_base: str, + ) -> Tuple[str, Dict]: + """ + Transform create request to RAGFlow POST /api/v1/datasets format. + + Maps LiteLLM params to RAGFlow dataset creation parameters. + RAGFlow-specific fields can be passed via metadata. + """ + url = api_base # Already includes /api/v1/datasets from get_complete_url + + # Extract name (required by RAGFlow) + name = vector_store_create_optional_params.get("name") + if not name: + raise ValueError("name is required for RAGFlow dataset creation") + + # Build request body + request_body: Dict[str, Any] = { + "name": name, + } + + # Extract RAGFlow-specific fields from metadata + metadata = vector_store_create_optional_params.get("metadata") + if metadata: + # RAGFlow-specific fields that can be in metadata + ragflow_fields = [ + "avatar", + "description", + "embedding_model", + "permission", + "chunk_method", + "parser_config", + "parse_type", + "pipeline_id", + ] + + for field in ragflow_fields: + if field in metadata: + request_body[field] = metadata[field] + + # Validate: chunk_method and pipeline_id are mutually exclusive + if "chunk_method" in request_body and "pipeline_id" in request_body: + raise ValueError( + "chunk_method and pipeline_id are mutually exclusive. " + "Specify either chunk_method or pipeline_id, not both." + ) + + # If neither chunk_method nor pipeline_id is specified, default to naive + if "chunk_method" not in request_body and "pipeline_id" not in request_body: + request_body["chunk_method"] = "naive" + + return url, request_body + + def transform_create_vector_store_response( + self, response: httpx.Response + ) -> VectorStoreCreateResponse: + """ + Transform RAGFlow response to VectorStoreCreateResponse format. + + RAGFlow response format: + { + "code": 0, + "data": { + "id": "...", + "name": "...", + "create_time": 1745836841611, # milliseconds + ... + } + } + """ + try: + response_json = response.json() + + # Check for RAGFlow error response + if response_json.get("code") != 0: + error_message = response_json.get("message", "Unknown error") + raise self.get_error_class( + error_message=error_message, + status_code=response.status_code, + headers=response.headers, + ) + + data = response_json.get("data", {}) + + # Extract dataset ID + dataset_id = data.get("id") + if not dataset_id: + raise ValueError("RAGFlow response missing dataset id") + + # Extract name + name = data.get("name") + + # Convert create_time from milliseconds to seconds (Unix timestamp) + create_time_ms = data.get("create_time", 0) + created_at = int(create_time_ms / 1000) if create_time_ms else None + + # Build VectorStoreCreateResponse + return VectorStoreCreateResponse( + id=dataset_id, + object="vector_store", + created_at=created_at or 0, + name=name, + bytes=0, # RAGFlow doesn't provide bytes in response + file_counts=VectorStoreFileCounts( + in_progress=0, + completed=0, + failed=0, + cancelled=0, + total=0, + ), + status="completed", + expires_after=None, + expires_at=None, + last_active_at=None, + metadata=None, + ) + except Exception as e: + # If it's already a ValueError we raised, re-raise it + if isinstance(e, ValueError) and "RAGFlow response" in str(e): + raise + # If it's already our error class (has status_code), re-raise + if hasattr(e, "status_code"): + raise + # Otherwise, wrap in our error class + raise self.get_error_class( + error_message=str(e), + status_code=response.status_code, + headers=response.headers, + ) + diff --git a/litellm/types/vector_stores.py b/litellm/types/vector_stores.py index 6ae0b4bd2fd..a4ceb2c9ac7 100644 --- a/litellm/types/vector_stores.py +++ b/litellm/types/vector_stores.py @@ -3,17 +3,15 @@ from datetime import datetime from enum import Enum from typing import Any, Dict, List, Literal, Optional, Tuple, Union -from annotated_types import Ge from pydantic import BaseModel from typing_extensions import TypedDict -from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams - class SupportedVectorStoreIntegrations(str, Enum): """Supported vector store integrations.""" BEDROCK = "bedrock" + RAGFLOW = "ragflow" class LiteLLM_VectorStoreConfig(TypedDict, total=False): diff --git a/litellm/utils.py b/litellm/utils.py index 44d4b5c25fd..b77c0e62e7d 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -7633,6 +7633,12 @@ class ProviderConfigManager: ) return GeminiVectorStoreConfig() + elif litellm.LlmProviders.RAGFLOW == provider: + from litellm.llms.ragflow.vector_stores.transformation import ( + RAGFlowVectorStoreConfig, + ) + + return RAGFlowVectorStoreConfig() return None @staticmethod diff --git a/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py b/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py index 501a5ff0389..cca20847f12 100644 --- a/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py +++ b/tests/test_litellm/vector_stores/test_vector_store_create_provider_logic.py @@ -9,9 +9,12 @@ sys.path.insert( ) # Adds the parent directory to the system path import litellm -from litellm.utils import ProviderConfigManager from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig -from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import VertexVectorStoreConfig +from litellm.llms.ragflow.vector_stores.transformation import RAGFlowVectorStoreConfig +from litellm.llms.vertex_ai.vector_stores.rag_api.transformation import ( + VertexVectorStoreConfig, +) +from litellm.utils import ProviderConfigManager def test_vector_store_create_with_simple_provider_name(): @@ -100,3 +103,40 @@ def test_vector_store_create_with_provider_api_type(): print("✅ Test passed: Provider with api_type 'vertex_ai/rag_api' handled correctly") + +def test_vector_store_create_with_ragflow_provider(): + """ + Test that vector store create correctly handles RAGFlow provider. + + This should: + - Return correct RAGFlowVectorStoreConfig + - Support dataset management operations + """ + custom_llm_provider = "ragflow" + + # Simulate the logic from vector_stores/main.py create function + if "/" in custom_llm_provider: + pytest.fail("Should not enter this branch for RAGFlow provider") + else: + api_type = None + custom_llm_provider = custom_llm_provider # Keep as-is + + # Verify api_type is None + assert api_type is None, "api_type should be None for RAGFlow provider" + + # Verify custom_llm_provider is unchanged + assert custom_llm_provider == "ragflow", "custom_llm_provider should remain 'ragflow'" + + # Verify ProviderConfigManager returns correct config + vector_store_provider_config = ProviderConfigManager.get_provider_vector_stores_config( + provider=litellm.LlmProviders(custom_llm_provider), + api_type=api_type, + ) + + assert vector_store_provider_config is not None, "Should return a config for RAGFlow" + assert isinstance( + vector_store_provider_config, RAGFlowVectorStoreConfig + ), "Should return RAGFlowVectorStoreConfig for RAGFlow provider" + + print("✅ Test passed: RAGFlow provider handled correctly") + diff --git a/tests/vector_store_tests/test_ragflow_vector_store.py b/tests/vector_store_tests/test_ragflow_vector_store.py new file mode 100644 index 00000000000..0839bd7153b --- /dev/null +++ b/tests/vector_store_tests/test_ragflow_vector_store.py @@ -0,0 +1,359 @@ +""" +Test RAGFlow Vector Store helper functions and transformation. +""" +import os +import sys +import json +import pytest +from unittest.mock import Mock, patch, MagicMock +import httpx + +sys.path.insert(0, os.path.abspath("../..")) +import litellm + +from tests.vector_store_tests.base_vector_store_test import BaseVectorStoreTest +from litellm.llms.ragflow.vector_stores.transformation import RAGFlowVectorStoreConfig +from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj +from litellm.types.vector_stores import VectorStoreCreateOptionalRequestParams + + +class TestRAGFlowVectorStore(BaseVectorStoreTest): + """ + Test the RAGFlow vector store transformation functionality. + """ + + def get_base_create_vector_store_args(self) -> dict: + """Must return the base create vector store args""" + return { + "custom_llm_provider": "ragflow", + "api_key": os.getenv("RAGFLOW_API_KEY", "test-api-key"), + "api_base": os.getenv("RAGFLOW_API_BASE", "http://localhost:9380") + } + + def get_base_request_args(self): + # RAGFlow doesn't support search, so we'll skip search tests + return { + "vector_store_id": "test-dataset-id", + "custom_llm_provider": "ragflow", + "query": "test query" + } + + def test_get_auth_credentials(self): + """Test that auth credentials are correctly extracted.""" + config = RAGFlowVectorStoreConfig() + + # Test with api_key in params + litellm_params = {"api_key": "test-api-key-123"} + credentials = config.get_auth_credentials(litellm_params) + assert "headers" in credentials + assert credentials["headers"]["Authorization"] == "Bearer test-api-key-123" + + # Test with missing api_key (should raise ValueError) + with pytest.raises(ValueError, match="api_key is required"): + config.get_auth_credentials({}) + + def test_get_complete_url(self): + """Test that complete URL is correctly constructed.""" + config = RAGFlowVectorStoreConfig() + + # Test with api_base in params + litellm_params = {"api_base": "http://custom-host:9999"} + url = config.get_complete_url(api_base=None, litellm_params=litellm_params) + assert url == "http://custom-host:9999/api/v1/datasets" + + # Test with api_base parameter + url = config.get_complete_url(api_base="http://test-host:8888", litellm_params={}) + assert url == "http://test-host:8888/api/v1/datasets" + + # Test with default (no api_base provided) + with patch.dict(os.environ, {}, clear=True): + url = config.get_complete_url(api_base=None, litellm_params={}) + assert url == "http://localhost:9380/api/v1/datasets" + + # Test with trailing slash removal + url = config.get_complete_url(api_base="http://test-host:8888/", litellm_params={}) + assert url == "http://test-host:8888/api/v1/datasets" + + def test_validate_environment(self): + """Test environment validation and header setting.""" + config = RAGFlowVectorStoreConfig() + from litellm.types.router import GenericLiteLLMParams + + # Test with api_key in litellm_params + litellm_params = GenericLiteLLMParams(api_key="test-key") + headers = config.validate_environment({}, litellm_params) + assert headers["Authorization"] == "Bearer test-key" + assert headers["Content-Type"] == "application/json" + + # Test with missing api_key + with pytest.raises(ValueError, match="RAGFLOW_API_KEY"): + config.validate_environment({}, GenericLiteLLMParams()) + + def test_get_vector_store_endpoints_by_type(self): + """Test that endpoints are correctly configured (empty for management only).""" + config = RAGFlowVectorStoreConfig() + endpoints = config.get_vector_store_endpoints_by_type() + assert endpoints["read"] == [] + assert endpoints["write"] == [] + + def test_transform_create_vector_store_request_basic(self): + """Test basic dataset creation request transformation.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-dataset" + } + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert url == "http://localhost:9380/api/v1/datasets" + assert body["name"] == "test-dataset" + assert body["chunk_method"] == "naive" # Default chunk method + + def test_transform_create_vector_store_request_with_metadata(self): + """Test dataset creation with RAGFlow-specific metadata.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-dataset-advanced", + "metadata": { + "description": "Test dataset", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI", + "permission": "me", + "chunk_method": "naive", + "parser_config": { + "chunk_token_num": 512, + "delimiter": "\n" + } + } + } + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert body["name"] == "test-dataset-advanced" + assert body["description"] == "Test dataset" + assert body["embedding_model"] == "BAAI/bge-large-zh-v1.5@BAAI" + assert body["permission"] == "me" + assert body["chunk_method"] == "naive" + assert "parser_config" in body + assert body["parser_config"]["chunk_token_num"] == 512 + + def test_transform_create_vector_store_request_missing_name(self): + """Test that missing name raises ValueError.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = {} + + with pytest.raises(ValueError, match="name is required"): + config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + def test_transform_create_vector_store_request_mutually_exclusive(self): + """Test that chunk_method and pipeline_id are mutually exclusive.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-dataset", + "metadata": { + "chunk_method": "naive", + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005" + } + } + + with pytest.raises(ValueError, match="mutually exclusive"): + config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + def test_transform_create_vector_store_request_with_pipeline(self): + """Test dataset creation with ingestion pipeline.""" + config = RAGFlowVectorStoreConfig() + + params: VectorStoreCreateOptionalRequestParams = { + "name": "test-pipeline-dataset", + "metadata": { + "parse_type": 2, + "pipeline_id": "d0bebe30ae2211f0970942010a8e0005" + } + } + + url, body = config.transform_create_vector_store_request( + params, "http://localhost:9380/api/v1/datasets" + ) + + assert body["name"] == "test-pipeline-dataset" + assert body["parse_type"] == 2 + assert body["pipeline_id"] == "d0bebe30ae2211f0970942010a8e0005" + assert "chunk_method" not in body + + def test_transform_create_vector_store_response_success(self): + """Test successful response transformation.""" + config = RAGFlowVectorStoreConfig() + + # Mock RAGFlow response + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 0, + "data": { + "id": "3b4de7d4241d11f0a6a79f24fc270c7f", + "name": "test-dataset", + "create_time": 1745836841611, + "chunk_method": "naive", + "embedding_model": "BAAI/bge-large-zh-v1.5@BAAI" + } + } + + response = config.transform_create_vector_store_response(mock_response) + + assert response["id"] == "3b4de7d4241d11f0a6a79f24fc270c7f" + assert response["name"] == "test-dataset" + assert response["object"] == "vector_store" + assert response["status"] == "completed" + assert response["created_at"] == 1745836841 # Converted from milliseconds + assert response["bytes"] == 0 + assert "file_counts" in response + + def test_transform_create_vector_store_response_error(self): + """Test error response transformation.""" + config = RAGFlowVectorStoreConfig() + + # Mock RAGFlow error response + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 400 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 101, + "message": "Dataset name 'test-dataset' already exists" + } + + with pytest.raises(Exception): # Should raise BaseLLMException + config.transform_create_vector_store_response(mock_response) + + def test_transform_create_vector_store_response_missing_id(self): + """Test response with missing dataset ID.""" + config = RAGFlowVectorStoreConfig() + + mock_response = Mock(spec=httpx.Response) + mock_response.status_code = 200 + mock_response.headers = {} + mock_response.json.return_value = { + "code": 0, + "data": { + "name": "test-dataset" + # Missing "id" + } + } + + with pytest.raises(ValueError, match="missing dataset id"): + config.transform_create_vector_store_response(mock_response) + + def test_transform_search_vector_store_request_not_implemented(self): + """Test that search operations raise NotImplementedError.""" + config = RAGFlowVectorStoreConfig() + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + + with pytest.raises(NotImplementedError, match="management only"): + config.transform_search_vector_store_request( + vector_store_id="test-id", + query="test query", + vector_store_search_optional_params={}, + api_base="http://localhost:9380", + litellm_logging_obj=logging_obj, + litellm_params={} + ) + + def test_transform_search_vector_store_response_not_implemented(self): + """Test that search response transformation raises NotImplementedError.""" + config = RAGFlowVectorStoreConfig() + logging_obj = MagicMock(spec=LiteLLMLoggingObj) + mock_response = Mock(spec=httpx.Response) + + with pytest.raises(NotImplementedError, match="management only"): + config.transform_search_vector_store_response(mock_response, logging_obj) + + def _validate_vector_store_create_response(self, response): + """Override to handle RAGFlow-specific response format.""" + # RAGFlow IDs are hex strings (not OpenAI-style vs_* format) + # So we override the base validation to not check for vs_ prefix + assert isinstance(response, dict), f"Response should be a dict, got {type(response)}" + assert "id" in response, "Missing required field 'id' in create response" + assert "object" in response, "Missing required field 'object' in create response" + assert "created_at" in response, "Missing required field 'created_at' in create response" + + assert response["object"] == "vector_store", \ + f"Expected object to be 'vector_store', got '{response['object']}'" + + assert isinstance(response["id"], str), \ + f"id should be a string, got {type(response['id'])}" + assert len(response["id"]) > 0, "id should not be empty" + # RAGFlow IDs are hex strings, not OpenAI-style vs_* format + + assert isinstance(response["created_at"], int), \ + f"created_at should be an integer, got {type(response['created_at'])}" + assert response["created_at"] > 0, "created_at should be a positive timestamp" + + print(f"✅ RAGFlow create response validation passed: Dataset '{response['id']}' created successfully") + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_create_vector_store(self, sync_mode): + """Override to handle RAGFlow-specific connection errors.""" + litellm._turn_on_debug() + litellm.set_verbose = True + base_request_args = self.get_base_create_vector_store_args() + + # Skip if no API key is set + if not os.getenv("RAGFLOW_API_KEY") and not base_request_args.get("api_key"): + pytest.skip("RAGFLOW_API_KEY not set, skipping integration test") + + # Extract custom_llm_provider from base args if present + create_args = base_request_args + try: + if sync_mode: + response = litellm.vector_stores.create( + name=f"test-ragflow-{int(__import__('time').time())}", + **create_args + ) + else: + response = await litellm.vector_stores.acreate( + name=f"test-ragflow-{int(__import__('time').time())}", + **create_args + ) + except litellm.InternalServerError: + pytest.skip("Skipping test due to litellm.InternalServerError") + except Exception as e: + error_str = str(e).lower() + error_type = type(e).__name__ + + # Check if it's a connection error + if (isinstance(e, (ConnectionError, OSError)) or + "connection" in error_str or + "connect" in error_str or + "APIConnectionError" in error_type): + pytest.skip(f"Skipping test due to connection error (RAGFlow instance may not be running): {e}") + + # If this is an authentication or permission error, skip the test + if "authentication" in error_str or "permission" in error_str or "unauthorized" in error_str: + pytest.skip(f"Skipping test due to authentication/permission error: {e}") + + # Re-raise if it's not a handled error + raise + + print("litellm create response=", json.dumps(response, indent=4, default=str)) + + # Validate response structure + self._validate_vector_store_create_response(response) + + @pytest.mark.parametrize("sync_mode", [True, False]) + @pytest.mark.asyncio + async def test_basic_search_vector_store(self, sync_mode): + """Override search test - RAGFlow doesn't support search.""" + pytest.skip("RAGFlow vector stores support dataset management only, not search") + From dad0b2c1117d3c2624d27dd229f7faa8babfbb41 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 15:32:42 +0530 Subject: [PATCH 21/24] Fix unused imports --- litellm/llms/ragflow/chat/transformation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/llms/ragflow/chat/transformation.py b/litellm/llms/ragflow/chat/transformation.py index d33a1593be8..58fbfa83c98 100644 --- a/litellm/llms/ragflow/chat/transformation.py +++ b/litellm/llms/ragflow/chat/transformation.py @@ -10,7 +10,7 @@ Model name format: - Agent: ragflow/agent/{agent_id}/{model_name} """ -from typing import Any, List, Optional, Tuple +from typing import List, Optional, Tuple import litellm from litellm.llms.openai.openai import OpenAIConfig From c9c7823f43b072a62ca2cd80bb9fc33f309dd1b1 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Wed, 3 Dec 2025 17:31:20 +0530 Subject: [PATCH 22/24] Fix bedrock models in model map --- ...odel_prices_and_context_window_backup.json | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 0fc97ce7b0a..dbaa60e0f1d 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -9629,6 +9629,21 @@ "supports_prompt_caching": true, "supports_tool_choice": true }, + "deepseek/deepseek-v3.2": { + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, + "litellm_provider": "deepseek", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "deepseek.v3-v1:0": { "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", @@ -20637,6 +20652,21 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "openrouter/deepseek/deepseek-v3.2": { + "input_cost_per_token": 2.8e-07, + "input_cost_per_token_cache_hit": 2.8e-08, + "litellm_provider": "openrouter", + "max_input_tokens": 163840, + "max_output_tokens": 163840, + "max_tokens": 8192, + "mode": "chat", + "output_cost_per_token": 4e-07, + "supports_assistant_prefill": true, + "supports_function_calling": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_tool_choice": true + }, "openrouter/deepseek/deepseek-v3.2-exp": { "input_cost_per_token": 2e-07, "input_cost_per_token_cache_hit": 2e-08, @@ -23892,6 +23922,32 @@ "supports_vision": true, "tool_use_system_prompt_tokens": 159 }, + "global.anthropic.claude-opus-4-5-20251101-v1:0": { + "cache_creation_input_token_cost": 6.25e-06, + "cache_read_input_token_cost": 5e-07, + "input_cost_per_token": 5e-06, + "litellm_provider": "bedrock_converse", + "max_input_tokens": 200000, + "max_output_tokens": 64000, + "max_tokens": 64000, + "mode": "chat", + "output_cost_per_token": 2.5e-05, + "search_context_cost_per_query": { + "search_context_size_high": 0.01, + "search_context_size_low": 0.01, + "search_context_size_medium": 0.01 + }, + "supports_assistant_prefill": true, + "supports_computer_use": true, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159 + }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { "cache_creation_input_token_cost": 3.75e-06, "cache_read_input_token_cost": 3e-07, From be5dd234bfe46f28bdfced4333f4df7562236c64 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 3 Dec 2025 08:01:26 -0800 Subject: [PATCH 23/24] docs: fix list --- docs/my-website/docs/providers/bedrock.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/docs/my-website/docs/providers/bedrock.md b/docs/my-website/docs/providers/bedrock.md index a9ac85a7571..7fc7758b15c 100644 --- a/docs/my-website/docs/providers/bedrock.md +++ b/docs/my-website/docs/providers/bedrock.md @@ -43,6 +43,8 @@ export AWS_BEARER_TOKEN_BEDROCK="your-api-key" Option 2: use the api_key parameter to pass in API key for completion, embedding, image_generation API calls. + + ```python response = completion( model="bedrock/anthropic.claude-3-sonnet-20240229-v1:0", @@ -50,7 +52,17 @@ response = completion( api_key="your-api-key" ) ``` - + + +```yaml +model_list: + - model_name: bedrock-claude-3-sonnet + litellm_params: + model: bedrock/anthropic.claude-3-sonnet-20240229-v1:0 + api_key: os.environ/AWS_BEARER_TOKEN_BEDROCK +``` + + ## Usage From 5e791464afb6eb2b6606d20cf23bdb87dd02e10d Mon Sep 17 00:00:00 2001 From: Cesar Garcia <128240629+Chesars@users.noreply.github.com> Date: Wed, 3 Dec 2025 14:07:02 -0300 Subject: [PATCH 24/24] docs: add Microsoft Agent Lightning to projects (#17422) Add Agent Lightning, Microsoft's open-source framework for training AI agents with RL, APO, and SFT. Uses LiteLLM Proxy for LLM routing and trace collection. --- docs/my-website/docs/projects/Agent Lightning.md | 10 ++++++++++ docs/my-website/sidebars.js | 1 + 2 files changed, 11 insertions(+) create mode 100644 docs/my-website/docs/projects/Agent Lightning.md diff --git a/docs/my-website/docs/projects/Agent Lightning.md b/docs/my-website/docs/projects/Agent Lightning.md new file mode 100644 index 00000000000..28e5546e398 --- /dev/null +++ b/docs/my-website/docs/projects/Agent Lightning.md @@ -0,0 +1,10 @@ + +# Agent Lightning + +[Agent Lightning](https://github.com/microsoft/agent-lightning) is Microsoft's open-source framework for training and optimizing AI agents with Reinforcement Learning, Automatic Prompt Optimization, and Supervised Fine-tuning — with almost zero code changes. + +It works with any agent framework including LangChain, OpenAI Agents SDK, AutoGen, and CrewAI. Agent Lightning uses LiteLLM Proxy under the hood to route LLM requests and collect traces that power its training algorithms. + +- [GitHub](https://github.com/microsoft/agent-lightning) +- [Docs](https://microsoft.github.io/agent-lightning/) +- [arXiv Paper](https://arxiv.org/abs/2508.03680) diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index e2f682f0a50..4a0f592c422 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -826,6 +826,7 @@ const sidebars = { "projects/mini-swe-agent", "projects/openai-agents", "projects/Google ADK", + "projects/Agent Lightning", "projects/Harbor", "projects/Docq.AI", "projects/PDL",