diff --git a/.circleci/config.yml b/.circleci/config.yml index fbbb6deeba8..8709f730c23 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -4100,6 +4100,63 @@ jobs: path: playwright-report destination: playwright-report + prisma_schema_sync: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Install Neon CLI + command: | + npm i -g neonctl + - run: + name: Install curl and dockerize + command: | + sudo apt-get update + sudo apt-get install -y curl + sudo wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + sudo rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Sync schema on base e2e database + command: | + BASE_DATABASE_URL=$(neon connection-string \ + --project-id $NEON_PROJECT_ID \ + --api-key $NEON_API_KEY \ + --branch br-fancy-paper-ad1olsb3 \ + --database-name yuneng-trial-db \ + --role neondb_owner) + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=$BASE_DATABASE_URL \ + -e LITELLM_MASTER_KEY="sk-1234" \ + --name schema-sync \ + -v $(pwd)/litellm/proxy/example_config_yaml/simple_config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --use_prisma_db_push + - run: + name: Start outputting logs + command: docker logs -f schema-sync + background: true + - run: + name: Wait for proxy to be ready (schema sync complete) + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Stop schema sync container + command: docker stop schema-sync + test_nonroot_image: machine: image: ubuntu-2204:2023.10.1 @@ -4298,6 +4355,15 @@ workflows: only: - main - /litellm_.*/ + - prisma_schema_sync: + context: e2e_ui_tests + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ - e2e_ui_testing: name: e2e_ui_testing_chromium browser: chromium @@ -4305,6 +4371,7 @@ workflows: requires: - ui_build - build_docker_database_image + - prisma_schema_sync filters: branches: only: @@ -4317,6 +4384,7 @@ workflows: requires: - ui_build - build_docker_database_image + - prisma_schema_sync filters: branches: only: diff --git a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md index b6665a76773..ab2cf334459 100644 --- a/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md +++ b/cookbook/misc/RELEASE_NOTES_GENERATION_INSTRUCTIONS.md @@ -404,7 +404,7 @@ This release has a known issue... - **New Providers** - Provider name, supported endpoints, description - **New LLM API Endpoints** (optional) - Endpoint, method, description, documentation link - Only include major new provider integrations, not minor provider updates -- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` (see Section 13) +- **IMPORTANT**: When adding new providers, also update `provider_endpoints_support.json` in the repository root (see Section 13) ### 12. Section Header Counts @@ -442,7 +442,7 @@ This release has a known issue... ### 13. Update provider_endpoints_support.json -**When adding new providers or endpoints, you MUST also update `litellm/proxy/public_endpoints/provider_endpoints_support.json`.** +**When adding new providers or endpoints, you MUST also update `provider_endpoints_support.json` in the repository root.** This file tracks which endpoints are supported by each LiteLLM provider and is used to generate documentation. diff --git a/docs/my-website/docs/ocr.md b/docs/my-website/docs/ocr.md index 93cb74ee69f..cea6fce1254 100644 --- a/docs/my-website/docs/ocr.md +++ b/docs/my-website/docs/ocr.md @@ -61,6 +61,52 @@ async def test_async_ocr(): asyncio.run(test_async_ocr()) ``` +### Using Local Files + +LiteLLM can read local files directly — no manual base64 encoding needed: + +```python +from litellm import ocr + +# OCR with a local PDF file path +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": "/path/to/document.pdf" + } +) + +# OCR with a file object +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": open("document.pdf", "rb") + } +) + +# OCR with raw bytes +with open("document.pdf", "rb") as f: + pdf_bytes = f.read() + +response = ocr( + model="mistral/mistral-ocr-latest", + document={ + "type": "file", + "file": pdf_bytes, + "mime_type": "application/pdf" # recommended for raw bytes (auto-detected from extension for file paths) + } +) +``` + +The `file` field accepts: +- **File path** (`str` or `pathlib.Path`) — LiteLLM reads the file and detects the MIME type from the extension +- **File object** (binary file-like object) — e.g. `open("doc.pdf", "rb")` +- **Raw bytes** (`bytes`) — use `mime_type` to specify the content type + +LiteLLM automatically converts file inputs to base64 data URIs internally, so all providers work seamlessly. + ### Using Base64 Encoded Documents ```python @@ -121,7 +167,7 @@ litellm --config /path/to/config.yaml # RUNNING on http://0.0.0.0:4000 ``` -Test request +**Test request — JSON body** ```bash curl http://0.0.0.0:4000/v1/ocr \ @@ -136,6 +182,27 @@ curl http://0.0.0.0:4000/v1/ocr \ }' ``` +**Test request — multipart file upload** + +Upload a file directly using multipart form data. No need to base64-encode the file yourself. + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@/path/to/document.pdf" +``` + +You can also pass optional parameters as additional form fields: + +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@screenshot.png" \ + -F 'pages=[0,1,2]' \ + -F "include_image_base64=true" +``` ## **Request/Response Format** @@ -168,10 +235,12 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie | Parameter | Type | Required | Description | |-----------|------|----------|-------------| | `model` | string | Yes | The OCR model to use (e.g., `"mistral/mistral-ocr-latest"`) | -| `document` | object | Yes | Document to process. Must contain `type` and URL field | -| `document.type` | string | Yes | Either `"document_url"` for PDFs/docs or `"image_url"` for images | -| `document.document_url` | string | Conditional | URL to the document (required if `type` is `"document_url"`) | -| `document.image_url` | string | Conditional | URL to the image (required if `type` is `"image_url"`) | +| `document` | object | Yes | Document to process. Must contain `type` and the corresponding field | +| `document.type` | string | Yes | `"document_url"` for PDFs/docs, `"image_url"` for images, or `"file"` for local files | +| `document.document_url` | string | Conditional | URL or data URI to the document (required if `type` is `"document_url"`) | +| `document.image_url` | string | Conditional | URL or data URI to the image (required if `type` is `"image_url"`) | +| `document.file` | string/bytes/file | Conditional | File path, bytes, or file-like object (required if `type` is `"file"`) | +| `document.mime_type` | string | No | Explicit MIME type for file inputs (auto-detected from extension if not provided) | | `pages` | array | No | List of specific page indices to process (0-indexed) | | `include_image_base64` | boolean | No | Whether to include extracted images as base64 strings | | `image_limit` | integer | No | Maximum number of images to return | @@ -179,7 +248,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie #### Document Format Examples -**For PDFs and documents:** +**For PDFs and documents (URL):** ```json { "type": "document_url", @@ -187,7 +256,7 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` -**For images:** +**For images (URL):** ```json { "type": "image_url", @@ -203,6 +272,21 @@ See the [official Mistral OCR documentation](https://docs.mistral.ai/capabilitie } ``` +**For local files (SDK):** +```python +{"type": "file", "file": "/path/to/document.pdf"} +{"type": "file", "file": open("image.png", "rb")} +{"type": "file", "file": pdf_bytes, "mime_type": "application/pdf"} +``` + +**For file uploads (Proxy — multipart form):** +```bash +curl http://0.0.0.0:4000/v1/ocr \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" +``` + ### Response Format The response follows Mistral's OCR format with the following structure: diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index b694549cf40..decffb18833 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -196,6 +196,7 @@ router_settings: | disable_end_user_cost_tracking_prometheus_only | boolean | If true, turns off end user cost tracking on prometheus metrics only. | | key_generation_settings | object | Restricts who can generate keys. [Further docs](./virtual_keys.md#restricting-key-generation) | | disable_add_transform_inline_image_block | boolean | For Fireworks AI models - if true, turns off the auto-add of `#transform=inline` to the url of the image_url, if the model is not a vision model. | +| use_chat_completions_url_for_anthropic_messages | boolean | If true, routes OpenAI `/v1/messages` requests through chat/completions instead of the Responses API. Can also be set via env var `LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES=true`. | | disable_hf_tokenizer_download | boolean | If true, it defaults to using the openai tokenizer for all models (including huggingface models). | | enable_json_schema_validation | boolean | If true, enables json schema validation for all requests. | | disable_copilot_system_to_assistant | boolean | **DEPRECATED** - GitHub Copilot API supports system prompts. | diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl new file mode 100644 index 00000000000..e44b58f8e63 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49-py3-none-any.whl differ diff --git a/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz new file mode 100644 index 00000000000..2c8549ad069 Binary files /dev/null and b/litellm-proxy-extras/dist/litellm_proxy_extras-0.4.49.tar.gz differ diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql new file mode 100644 index 00000000000..594ab9ac1a2 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20260226202727_add_agent_id_to_delete_keys/migration.sql @@ -0,0 +1,3 @@ +-- AlterTable +ALTER TABLE "LiteLLM_DeletedVerificationToken" ADD COLUMN "agent_id" TEXT; + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 440c9c1d829..13461be3e7c 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? @@ -504,6 +505,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index bd57b248cdb..968536712dc 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." authors = ["BerriAI"] readme = "README.md" @@ -22,7 +22,7 @@ requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "0.4.48" +version = "0.4.49" version_files = [ "pyproject.toml:version", "../requirements.txt:litellm-proxy-extras==", diff --git a/litellm/__init__.py b/litellm/__init__.py index 6e42f2c1ea5..50fa0e76755 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -197,6 +197,9 @@ telemetry = True max_tokens: int = DEFAULT_MAX_TOKENS # OpenAI Defaults drop_params = bool(os.getenv("LITELLM_DROP_PARAMS", False)) modify_params = bool(os.getenv("LITELLM_MODIFY_PARAMS", False)) +use_chat_completions_url_for_anthropic_messages: bool = bool( + os.getenv("LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES", False) +) # When True, routes OpenAI /v1/messages requests to chat/completions instead of the Responses API retry = True ### AUTH ### api_key: Optional[str] = None diff --git a/litellm/caching/llm_caching_handler.py b/litellm/caching/llm_caching_handler.py index 5dc16a224c7..16eb824f4c9 100644 --- a/litellm/caching/llm_caching_handler.py +++ b/litellm/caching/llm_caching_handler.py @@ -8,25 +8,6 @@ from .in_memory_cache import InMemoryCache class LLMClientCache(InMemoryCache): - def _remove_key(self, key: str) -> None: - """Close async clients before evicting them to prevent connection pool leaks.""" - value = self.cache_dict.get(key) - super()._remove_key(key) - if value is not None: - close_fn = getattr(value, "aclose", None) or getattr( - value, "close", None - ) - if close_fn and asyncio.iscoroutinefunction(close_fn): - try: - asyncio.get_running_loop().create_task(close_fn()) - except RuntimeError: - pass - elif close_fn and callable(close_fn): - try: - close_fn() - except Exception: - pass - def update_cache_key_with_event_loop(self, key): """ Add the event loop to the cache key, to prevent event loop closed errors. diff --git a/litellm/exceptions.py b/litellm/exceptions.py index eb027334606..b36d4ef877c 100644 --- a/litellm/exceptions.py +++ b/litellm/exceptions.py @@ -955,7 +955,8 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore generated_content: str = "", is_pre_first_chunk: bool = False, ): - self.status_code = 503 # Service Unavailable + original_status = getattr(original_exception, "status_code", None) + self.status_code = int(original_status) if original_status is not None else 503 self.message = f"litellm.MidStreamFallbackError: {message}" self.model = model self.llm_provider = llm_provider @@ -978,7 +979,14 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore else: self.response = response - # Call the parent constructor + # Save the original attributes before they are overridden by ServiceUnavailableError + _saved_response = self.response + _saved_request = getattr(self.response, "request", None) or httpx.Request( + method="POST", url=f"https://{llm_provider}.com/v1/" + ) + _saved_message = self.message + + # Call the parent constructor (which hardcodes status_code=503 and modifies the response object) super().__init__( message=self.message, llm_provider=llm_provider, @@ -988,6 +996,13 @@ class MidStreamFallbackError(ServiceUnavailableError): # type: ignore max_retries=self.max_retries, num_retries=self.num_retries, ) + + # Restore the propagated status and original response/request objects + self.status_code = int(original_status) if original_status is not None else 503 + self.response = _saved_response + self.request = _saved_request + self.message = _saved_message + self.args = (_saved_message,) def __str__(self): _message = self.message diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index 08db77e8571..7a08432b9a1 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -2686,6 +2686,8 @@ class PrometheusLogger(CustomLogger): if team_info: team_object.budget_reset_at = team_info.budget_reset_at + if team_object.max_budget is None and team_info.max_budget is not None: + team_object.max_budget = team_info.max_budget return team_object @@ -2903,6 +2905,8 @@ class PrometheusLogger(CustomLogger): if user_info: user_object.budget_reset_at = user_info.budget_reset_at + if user_object.max_budget is None and user_info.max_budget is not None: + user_object.max_budget = user_info.max_budget return user_object diff --git a/litellm/litellm_core_utils/llm_cost_calc/utils.py b/litellm/litellm_core_utils/llm_cost_calc/utils.py index a9fd0f4ea8a..bf0b2709365 100644 --- a/litellm/litellm_core_utils/llm_cost_calc/utils.py +++ b/litellm/litellm_core_utils/llm_cost_calc/utils.py @@ -8,9 +8,11 @@ from litellm._logging import verbose_logger from litellm.types.utils import ( CacheCreationTokenDetails, CallTypes, + CompletionTokensDetailsWrapper, ImageResponse, ModelInfo, PassthroughCallTypes, + PromptTokensDetailsWrapper, ServiceTier, Usage, ) @@ -767,6 +769,64 @@ def generic_cost_per_token( # noqa: PLR0915 return prompt_cost, completion_cost +def calculate_image_response_cost_from_usage( + model: str, + image_response: ImageResponse, + custom_llm_provider: str, +) -> Optional[float]: + """ + Calculate image generation cost from usage metadata when available. + + Returns: + Optional[float]: total cost from token usage, or None when usage metadata + is missing/incomplete and caller should fall back to flat per-image pricing. + """ + usage = image_response.usage + if usage is None: + return None + + prompt_tokens = usage.input_tokens + completion_tokens = usage.output_tokens + total_tokens = usage.total_tokens + + if prompt_tokens is None or completion_tokens is None or total_tokens is None: + return None + + # ImageResponse may carry a default zeroed usage object even when provider + # usage metadata is absent. Treat this as missing usage and fall back. + if prompt_tokens == 0 and completion_tokens == 0 and total_tokens == 0: + return None + + input_tokens_details = getattr(usage, "input_tokens_details", None) + prompt_tokens_details: Optional[PromptTokensDetailsWrapper] = None + if input_tokens_details is not None: + prompt_tokens_details = PromptTokensDetailsWrapper( + text_tokens=getattr(input_tokens_details, "text_tokens", None), + image_tokens=getattr(input_tokens_details, "image_tokens", None), + cached_tokens=0, + ) + + normalized_usage = Usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + prompt_tokens_details=prompt_tokens_details, + completion_tokens_details=CompletionTokensDetailsWrapper( + text_tokens=0, + image_tokens=completion_tokens, + reasoning_tokens=0, + audio_tokens=0, + ), + ) + + prompt_cost, completion_cost = generic_cost_per_token( + model=model, + usage=normalized_usage, + custom_llm_provider=custom_llm_provider, + ) + return prompt_cost + completion_cost + + class CostCalculatorUtils: @staticmethod def _call_type_has_image_response(call_type: str) -> bool: diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index ba415af9a5a..796223ff8e1 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -1766,6 +1766,7 @@ def convert_function_to_anthropic_tool_invoke( def convert_to_anthropic_tool_invoke( tool_calls: List[ChatCompletionAssistantToolCall], web_search_results: Optional[List[Any]] = None, + tool_results: Optional[List[Any]] = None, ) -> List[Union[AnthropicMessagesToolUseParam, Dict[str, Any]]]: """ OpenAI tool invokes: @@ -1840,12 +1841,18 @@ def convert_to_anthropic_tool_invoke( } anthropic_tool_invoke.append(_anthropic_server_tool_use) - # Add corresponding web_search_tool_result if available + # Add corresponding tool result if available. + # Check both web_search_results (web_search_tool_result / web_fetch_tool_result) + # and tool_results (bash_code_execution_tool_result, etc.) + _all_tool_results: List[Any] = [] if web_search_results: - for result in web_search_results: - if result.get("tool_use_id") == tool_id: - anthropic_tool_invoke.append(result) - break + _all_tool_results.extend(web_search_results) + if tool_results: + _all_tool_results.extend(tool_results) + for result in _all_tool_results: + if result.get("tool_use_id") == tool_id: + anthropic_tool_invoke.append(result) + break else: # Regular tool_use sanitized_tool_id = _sanitize_anthropic_tool_use_id(tool_id) @@ -2472,9 +2479,10 @@ def anthropic_messages_pt( # noqa: PLR0915 # Pass through as-is since these are Anthropic-native content types elif m.get("type", "") == "server_tool_use": assistant_content.append(m) # type: ignore - # handle tool_search_tool_result blocks + # handle all *_tool_result blocks (tool_search_tool_result, + # web_search_tool_result, bash_code_execution_tool_result, etc.) # Pass through as-is since these are Anthropic-native content types - elif m.get("type", "") == "tool_search_tool_result": + elif m.get("type", "").endswith("_tool_result"): assistant_content.append(m) # type: ignore elif ( "content" in assistant_content_block @@ -2504,7 +2512,8 @@ def anthropic_messages_pt( # noqa: PLR0915 if ( assistant_tool_calls is not None ): # support assistant tool invoke conversion - # Get web_search_results from provider_specific_fields for server_tool_use reconstruction + # Get web_search_results and tool_results from provider_specific_fields + # for server_tool_use reconstruction. # Fixes: https://github.com/BerriAI/litellm/issues/17737 _provider_specific_fields_raw = assistant_content_block.get( "provider_specific_fields" @@ -2517,9 +2526,11 @@ def anthropic_messages_pt( # noqa: PLR0915 _web_search_results = _provider_specific_fields.get( "web_search_results" ) + _tool_results = _provider_specific_fields.get("tool_results") tool_invoke_results = convert_to_anthropic_tool_invoke( assistant_tool_calls, web_search_results=_web_search_results, + tool_results=_tool_results, ) # Prevent "tool_use ids must be unique" errors by filtering duplicates diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index 449a4892621..294f9c485c1 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -72,6 +72,9 @@ class RealTimeStreaming: self.request_data: Dict = request_data or {} # Violation counter for end_session_after_n_fails support self._violation_count: int = 0 + # When a text message is blocked, hold the guardrail reason so the next + # response.create can be rewritten to include the failure context. + self._pending_guardrail_message: Optional[str] = None def _should_store_message( self, @@ -261,18 +264,12 @@ class RealTimeStreaming: When this returns True, we inject a session.update to disable the LLM's auto-response so the guardrail can gate it first. - """ - from litellm.integrations.custom_guardrail import CustomGuardrail - from litellm.types.guardrails import GuardrailEventHooks - return any( - isinstance(cb, CustomGuardrail) - and cb.should_run_guardrail( - data=self.request_data, - event_type=GuardrailEventHooks.realtime_input_transcription, - ) - for cb in litellm.callbacks - ) + Must match the same hook criteria as run_realtime_guardrails() so that + any guardrail that would actually check the transcript also disables + auto-response before the transcript arrives. + """ + return self._has_realtime_guardrails() async def run_realtime_guardrails( self, @@ -335,18 +332,35 @@ class RealTimeStreaming: # Use realtime_violation_message if configured; fall back to guardrail error text. error_msg = getattr(callback, "realtime_violation_message", None) or safe_msg - # Return the error directly to the WebSocket consumer. + # Cancel any in-progress LLM response (e.g. VAD auto-response). + await self._send_to_backend(json.dumps({"type": "response.cancel"})) + # Send the policy violation hint (shows as small gray status text in UI). await self.websocket.send_text( - json.dumps( - { - "type": "error", - "error": { - "type": "guardrail_violation", - "message": error_msg, - "code": "content_policy_violation", - }, - } - ) + json.dumps({ + "type": "error", + "error": { + "type": "guardrail_violation", + "message": error_msg, + "code": "content_policy_violation", + }, + }) + ) + # Ask the LLM to voice the exact guardrail message so the + # user hears it as audio in voice sessions (not just text). + guardrail_prompt = ( + f"Say exactly the following message to the user, word for word, " + f"do not add anything else: {error_msg}" + ) + await self._send_to_backend(json.dumps({ + "type": "conversation.item.create", + "item": { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": guardrail_prompt}], + }, + })) + await self._send_to_backend( + json.dumps({"type": "response.create"}) ) self._violation_count += 1 @@ -559,7 +573,17 @@ class RealTimeStreaming: combined_text ) if blocked: - continue # don't forward to backend + # Store the guardrail reason so the next response.create + # (sent automatically by the client) is rewritten to + # include it as response instructions. + self._pending_guardrail_message = combined_text + continue # don't forward the original blocked message + + if msg_type == "response.create" and self._pending_guardrail_message: + # The guardrail already sent the synthetic AI bubble — drop this + # response.create so OpenAI doesn't generate an additional response. + self._pending_guardrail_message = None + continue except (json.JSONDecodeError, AttributeError): pass diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index baf274f2c62..2c2291ae889 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -161,6 +161,7 @@ class CustomStreamWrapper: ) # keep track of the returned chunks - used for calculating the input/output tokens for stream options self.is_function_call = self.check_is_function_call(logging_obj=logging_obj) self.created: Optional[int] = None + self._last_returned_hidden_params: Optional[dict] = None def _check_max_streaming_duration(self) -> None: """Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS.""" @@ -1835,6 +1836,7 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) response._hidden_params["usage"] = usage + self._last_returned_hidden_params = response._hidden_params # Add MCP metadata to final chunk if present response = self._add_mcp_metadata_to_final_chunk(response) # RETURN RESULT @@ -1876,6 +1878,24 @@ class CustomStreamWrapper: None, cache_hit, ) + # Update hidden_params with final usage from + # stream_chunk_builder. Some providers (e.g. OpenRouter) + # send usage in a chunk after finish_reason, which arrives + # after _hidden_params["usage"] was initially set. The + # _hidden_params dict is the same object the user received + # (shared by reference), so mutating it here also corrects + # the user's copy. + if ( + self.stream_options is None + and complete_streaming_response is not None + and self._last_returned_hidden_params is not None + ): + final_usage = getattr( + complete_streaming_response, "usage", None + ) + if final_usage is not None: + self._last_returned_hidden_params["usage"] = final_usage + if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True return response @@ -2000,6 +2020,7 @@ class CustomStreamWrapper: if self.sent_last_chunk is True and self.stream_options is None: usage = calculate_total_usage(chunks=self.chunks) processed_chunk._hidden_params["usage"] = usage + self._last_returned_hidden_params = processed_chunk._hidden_params # Call post-call streaming deployment hook for final chunk if self.sent_last_chunk is True: @@ -2064,6 +2085,19 @@ class CustomStreamWrapper: cache_hit=cache_hit, ) ) + # Update hidden_params with final usage from + # stream_chunk_builder (see sync __next__ for full comment). + if ( + self.stream_options is None + and complete_streaming_response is not None + and self._last_returned_hidden_params is not None + ): + final_usage = getattr( + complete_streaming_response, "usage", None + ) + if final_usage is not None: + self._last_returned_hidden_params["usage"] = final_usage + if self.sent_stream_usage is False and self.send_stream_usage is True: self.sent_stream_usage = True return response diff --git a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py index 7e5a4f22a7f..5b215c1fe54 100644 --- a/litellm/llms/anthropic/experimental_pass_through/messages/handler.py +++ b/litellm/llms/anthropic/experimental_pass_through/messages/handler.py @@ -25,8 +25,24 @@ from litellm.types.router import GenericLiteLLMParams from litellm.utils import ProviderConfigManager, client from ..adapters.handler import LiteLLMMessagesToCompletionTransformationHandler +from ..responses_adapters.handler import LiteLLMMessagesToResponsesAPIHandler from .utils import AnthropicMessagesRequestUtils, mock_response +# Providers that are routed directly to the OpenAI Responses API instead of +# going through chat/completions. +_RESPONSES_API_PROVIDERS = frozenset({"openai"}) + + +def _should_route_to_responses_api(custom_llm_provider: Optional[str]) -> bool: + """Return True when the provider should use the Responses API path. + + Set ``litellm.use_chat_completions_url_for_anthropic_messages = True`` to + opt out and route OpenAI/Azure requests through chat/completions instead. + """ + if litellm.use_chat_completions_url_for_anthropic_messages: + return False + return custom_llm_provider in _RESPONSES_API_PROVIDERS + ####### ENVIRONMENT VARIABLES ################### # Initialize any necessary instances or variables here base_llm_http_handler = BaseLLMHTTPHandler() @@ -282,29 +298,34 @@ def anthropic_messages_handler( ) ) if anthropic_messages_provider_config is None: - # Handle non-Anthropic models using the adapter - return ( - LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( - max_tokens=max_tokens, - messages=messages, - model=model, - metadata=metadata, - stop_sequences=stop_sequences, - stream=stream, - system=system, - temperature=temperature, - thinking=thinking, - tool_choice=tool_choice, - tools=tools, - top_k=top_k, - top_p=top_p, - _is_async=is_async, - api_key=api_key, - api_base=api_base, - client=client, - custom_llm_provider=custom_llm_provider, - **kwargs, + # Route to Responses API for OpenAI / Azure, chat/completions for everything else. + _shared_kwargs = dict( + max_tokens=max_tokens, + messages=messages, + model=model, + metadata=metadata, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + _is_async=is_async, + api_key=api_key, + api_base=api_base, + client=client, + custom_llm_provider=custom_llm_provider, + **kwargs, + ) + if _should_route_to_responses_api(custom_llm_provider): + return LiteLLMMessagesToResponsesAPIHandler.anthropic_messages_handler( + **_shared_kwargs ) + return LiteLLMMessagesToCompletionTransformationHandler.anthropic_messages_handler( + **_shared_kwargs ) if custom_llm_provider is None: diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..6ad3c7b0164 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py @@ -0,0 +1,3 @@ +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +__all__ = ["LiteLLMAnthropicToResponsesAPIAdapter"] diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py new file mode 100644 index 00000000000..c268d6c5be8 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/handler.py @@ -0,0 +1,229 @@ +""" +Handler for the Anthropic v1/messages -> OpenAI Responses API path. + +Used when the target model is an OpenAI or Azure model. +""" + +from typing import Any, AsyncIterator, Coroutine, Dict, List, Optional, Union + +import litellm +from litellm.types.llms.anthropic import AnthropicMessagesRequest +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, +) +from litellm.types.llms.openai import ResponsesAPIResponse + +from .streaming_iterator import AnthropicResponsesStreamWrapper +from .transformation import LiteLLMAnthropicToResponsesAPIAdapter + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +def _build_responses_kwargs( + *, + max_tokens: int, + messages: List[Dict], + model: str, + context_management: Optional[Dict] = None, + metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + extra_kwargs: Optional[Dict[str, Any]] = None, +) -> Dict[str, Any]: + """ + Build the kwargs dict to pass directly to litellm.responses() / litellm.aresponses(). + """ + # Build a typed AnthropicMessagesRequest for the adapter + request_data: Dict[str, Any] = {"model": model, "messages": messages, "max_tokens": max_tokens} + if context_management: + request_data["context_management"] = context_management + if output_config: + request_data["output_config"] = output_config + if metadata: + request_data["metadata"] = metadata + if system: + request_data["system"] = system + if temperature is not None: + request_data["temperature"] = temperature + if thinking: + request_data["thinking"] = thinking + if tool_choice: + request_data["tool_choice"] = tool_choice + if tools: + request_data["tools"] = tools + if top_p is not None: + request_data["top_p"] = top_p + if output_format: + request_data["output_format"] = output_format + + anthropic_request = AnthropicMessagesRequest(**request_data) + responses_kwargs = _ADAPTER.translate_request(anthropic_request) + + if stream: + responses_kwargs["stream"] = True + + # Forward litellm-specific kwargs (api_key, api_base, logging obj, etc.) + excluded = {"anthropic_messages"} + for key, value in (extra_kwargs or {}).items(): + if key == "litellm_logging_obj" and value is not None: + from litellm.litellm_core_utils.litellm_logging import ( + Logging as LiteLLMLoggingObject, + ) + from litellm.types.utils import CallTypes + + if isinstance(value, LiteLLMLoggingObject): + # Reclassify as acompletion so the success handler doesn't try to + # validate the Responses API event as an AnthropicResponse. + # (Mirrors the pattern used in LiteLLMMessagesToCompletionTransformationHandler.) + setattr(value, "call_type", CallTypes.acompletion.value) + responses_kwargs[key] = value + elif key not in excluded and key not in responses_kwargs and value is not None: + responses_kwargs[key] = value + + return responses_kwargs + + +class LiteLLMMessagesToResponsesAPIHandler: + """ + Handles Anthropic /v1/messages requests for OpenAI / Azure models by + calling litellm.responses() / litellm.aresponses() directly and translating + the response back to Anthropic format. + """ + + @staticmethod + async def async_anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + context_management: Optional[Dict] = None, + metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + **kwargs, + ) -> Union[AnthropicMessagesResponse, AsyncIterator]: + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + context_management=context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = await litellm.aresponses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) + + @staticmethod + def anthropic_messages_handler( + max_tokens: int, + messages: List[Dict], + model: str, + context_management: Optional[Dict] = None, + metadata: Optional[Dict] = None, + output_config: Optional[Dict] = None, + stop_sequences: Optional[List[str]] = None, + stream: Optional[bool] = False, + system: Optional[str] = None, + temperature: Optional[float] = None, + thinking: Optional[Dict] = None, + tool_choice: Optional[Dict] = None, + tools: Optional[List[Dict]] = None, + top_k: Optional[int] = None, + top_p: Optional[float] = None, + output_format: Optional[Dict] = None, + _is_async: bool = False, + **kwargs, + ) -> Union[ + AnthropicMessagesResponse, + AsyncIterator[Any], + Coroutine[Any, Any, Union[AnthropicMessagesResponse, AsyncIterator[Any]]], + ]: + if _is_async: + return LiteLLMMessagesToResponsesAPIHandler.async_anthropic_messages_handler( + max_tokens=max_tokens, + messages=messages, + model=model, + context_management=context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + **kwargs, + ) + + # Sync path + responses_kwargs = _build_responses_kwargs( + max_tokens=max_tokens, + messages=messages, + model=model, + context_management=context_management, + metadata=metadata, + output_config=output_config, + stop_sequences=stop_sequences, + stream=stream, + system=system, + temperature=temperature, + thinking=thinking, + tool_choice=tool_choice, + tools=tools, + top_k=top_k, + top_p=top_p, + output_format=output_format, + extra_kwargs=kwargs, + ) + + result = litellm.responses(**responses_kwargs) + + if stream: + wrapper = AnthropicResponsesStreamWrapper(responses_stream=result, model=model) + return wrapper.async_anthropic_sse_wrapper() + + if not isinstance(result, ResponsesAPIResponse): + raise ValueError(f"Expected ResponsesAPIResponse, got {type(result)}") + + return _ADAPTER.translate_response(result) diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py new file mode 100644 index 00000000000..0e6268e82f3 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/streaming_iterator.py @@ -0,0 +1,265 @@ +# What is this? +## Translates OpenAI call to Anthropic `/v1/messages` format +import json +import traceback +from collections import deque +from typing import Any, AsyncIterator, Dict + +from litellm import verbose_logger +from litellm._uuid import uuid + + +class AnthropicResponsesStreamWrapper: + """ + Wraps a Responses API streaming iterator and re-emits events in Anthropic SSE format. + + Responses API event flow (relevant subset): + response.created -> message_start + response.output_item.added -> content_block_start (if message/function_call) + response.output_text.delta -> content_block_delta (text_delta) + response.reasoning_summary_text.delta -> content_block_delta (thinking_delta) + response.function_call_arguments.delta -> content_block_delta (input_json_delta) + response.output_item.done -> content_block_stop + response.completed -> message_delta + message_stop + """ + + def __init__( + self, + responses_stream: Any, + model: str, + ) -> None: + self.responses_stream = responses_stream + self.model = model + self._message_id: str = f"msg_{uuid.uuid4()}" + self._current_block_index: int = -1 + # Map item_id -> content_block_index so we can stop the right block later + self._item_id_to_block_index: Dict[str, int] = {} + # Track open function_call items by item_id so we can emit tool_use start + self._pending_tool_ids: Dict[str, str] = {} # item_id -> call_id / name accumulator + self._sent_message_start = False + self._sent_message_stop = False + self._chunk_queue: deque = deque() + + def _make_message_start(self) -> Dict[str, Any]: + return { + "type": "message_start", + "message": { + "id": self._message_id, + "type": "message", + "role": "assistant", + "content": [], + "model": self.model, + "stop_reason": None, + "stop_sequence": None, + "usage": { + "input_tokens": 0, + "output_tokens": 0, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + } + + def _next_block_index(self) -> int: + self._current_block_index += 1 + return self._current_block_index + + def _process_event(self, event: Any) -> None: + """Convert one Responses API event into zero or more Anthropic chunks queued for emission.""" + event_type = getattr(event, "type", None) + if event_type is None and isinstance(event, dict): + event_type = event.get("type") + + if event_type is None: + return + + # ---- message_start ---- + if event_type == "response.created": + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return + + # ---- content_block_start for a new output message item ---- + if event_type == "response.output_item.added": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + if item is None: + return + item_type = getattr(item, "type", None) or (item.get("type") if isinstance(item, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) + + if item_type == "message": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "text", "text": ""}, + }) + elif item_type == "function_call": + call_id = getattr(item, "call_id", None) or (item.get("call_id") if isinstance(item, dict) else None) or "" + name = getattr(item, "name", None) or (item.get("name") if isinstance(item, dict) else None) or "" + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._pending_tool_ids[item_id] = call_id + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": { + "type": "tool_use", + "id": call_id, + "name": name, + "input": {}, + }, + }) + elif item_type == "reasoning": + block_idx = self._next_block_index() + if item_id: + self._item_id_to_block_index[item_id] = block_idx + self._chunk_queue.append({ + "type": "content_block_start", + "index": block_idx, + "content_block": {"type": "thinking", "thinking": ""}, + }) + return + + # ---- text delta ---- + if event_type == "response.output_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "text_delta", "text": delta}, + }) + return + + # ---- reasoning summary text delta ---- + if event_type == "response.reasoning_summary_text.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "thinking_delta", "thinking": delta}, + }) + return + + # ---- function call arguments delta ---- + if event_type == "response.function_call_arguments.delta": + item_id = getattr(event, "item_id", None) or (event.get("item_id") if isinstance(event, dict) else None) + delta = getattr(event, "delta", "") or (event.get("delta", "") if isinstance(event, dict) else "") + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_delta", + "index": block_idx, + "delta": {"type": "input_json_delta", "partial_json": delta}, + }) + return + + # ---- output item done -> content_block_stop ---- + if event_type == "response.output_item.done": + item = getattr(event, "item", None) or (event.get("item") if isinstance(event, dict) else None) + item_id = getattr(item, "id", None) or (item.get("id") if isinstance(item, dict) else None) if item else None + block_idx = self._item_id_to_block_index.get(item_id, self._current_block_index) if item_id else self._current_block_index + self._chunk_queue.append({ + "type": "content_block_stop", + "index": block_idx, + }) + return + + # ---- response completed -> message_delta + message_stop ---- + if event_type in ("response.completed", "response.failed", "response.incomplete"): + response_obj = getattr(event, "response", None) or (event.get("response") if isinstance(event, dict) else None) + stop_reason = "end_turn" + input_tokens = 0 + output_tokens = 0 + cache_creation_tokens = 0 + cache_read_tokens = 0 + + if response_obj is not None: + status = getattr(response_obj, "status", None) + if status == "incomplete": + stop_reason = "max_tokens" + usage = getattr(response_obj, "usage", None) + if usage is not None: + input_tokens = getattr(usage, "input_tokens", 0) or 0 + output_tokens = getattr(usage, "output_tokens", 0) or 0 + cache_creation_tokens = getattr(usage, "input_tokens_details", None) + cache_read_tokens = getattr(usage, "output_tokens_details", None) + # Prefer direct cache fields if present + cache_creation_tokens = getattr(usage, "cache_creation_input_tokens", 0) or 0 + cache_read_tokens = getattr(usage, "cache_read_input_tokens", 0) or 0 + + # Check if tool_use was in the output to override stop_reason + if response_obj is not None: + output = getattr(response_obj, "output", []) or [] + for out_item in output: + out_type = getattr(out_item, "type", None) or (out_item.get("type") if isinstance(out_item, dict) else None) + if out_type == "function_call": + stop_reason = "tool_use" + break + + usage_delta: Dict[str, Any] = { + "input_tokens": input_tokens, + "output_tokens": output_tokens, + } + if cache_creation_tokens: + usage_delta["cache_creation_input_tokens"] = cache_creation_tokens + if cache_read_tokens: + usage_delta["cache_read_input_tokens"] = cache_read_tokens + + self._chunk_queue.append({ + "type": "message_delta", + "delta": {"stop_reason": stop_reason, "stop_sequence": None}, + "usage": usage_delta, + }) + self._chunk_queue.append({"type": "message_stop"}) + self._sent_message_stop = True + return + + def __aiter__(self) -> "AnthropicResponsesStreamWrapper": + return self + + async def __anext__(self) -> Dict[str, Any]: + # Return any queued chunks first + if self._chunk_queue: + return self._chunk_queue.popleft() + + # Emit message_start if not yet done (fallback if response.created wasn't fired) + if not self._sent_message_start: + self._sent_message_start = True + self._chunk_queue.append(self._make_message_start()) + return self._chunk_queue.popleft() + + # Consume the upstream stream + try: + async for event in self.responses_stream: + self._process_event(event) + if self._chunk_queue: + return self._chunk_queue.popleft() + except StopAsyncIteration: + pass + except Exception as e: + verbose_logger.error( + f"AnthropicResponsesStreamWrapper error: {e}\n{traceback.format_exc()}" + ) + + # Drain any remaining queued chunks + if self._chunk_queue: + return self._chunk_queue.popleft() + + raise StopAsyncIteration + + async def async_anthropic_sse_wrapper(self) -> AsyncIterator[bytes]: + """Yield SSE-encoded bytes for each Anthropic event chunk.""" + async for chunk in self: + if isinstance(chunk, dict): + event_type: str = str(chunk.get("type", "message")) + payload = f"event: {event_type}\ndata: {json.dumps(chunk)}\n\n" + yield payload.encode() + else: + yield chunk diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py new file mode 100644 index 00000000000..c2752272905 --- /dev/null +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -0,0 +1,450 @@ +""" +Transformation layer: Anthropic /v1/messages <-> OpenAI Responses API. + +This module owns all format conversions for the direct v1/messages -> Responses API +path used for OpenAI and Azure models. +""" + +import json +from typing import Any, Dict, List, Optional, Union, cast + +from litellm.types.llms.anthropic import ( + AllAnthropicToolsValues, + AnthopicMessagesAssistantMessageParam, + AnthropicFinishReason, + AnthropicMessagesRequest, + AnthropicMessagesToolChoice, + AnthropicMessagesUserMessageParam, + AnthropicResponseContentBlockText, + AnthropicResponseContentBlockThinking, + AnthropicResponseContentBlockToolUse, +) +from litellm.types.llms.anthropic_messages.anthropic_response import ( + AnthropicMessagesResponse, + AnthropicUsage, +) +from litellm.types.llms.openai import ResponsesAPIResponse + + +class LiteLLMAnthropicToResponsesAPIAdapter: + """ + Converts Anthropic /v1/messages requests to OpenAI Responses API format and + converts Responses API responses back to Anthropic format. + """ + + # ------------------------------------------------------------------ # + # Request translation: Anthropic -> Responses API # + # ------------------------------------------------------------------ # + + @staticmethod + def _translate_anthropic_image_source_to_url(source: dict) -> Optional[str]: + """Convert Anthropic image source to a URL string.""" + source_type = source.get("type") + if source_type == "base64": + media_type = source.get("media_type", "image/jpeg") + data = source.get("data", "") + return f"data:{media_type};base64,{data}" if data else None + elif source_type == "url": + return source.get("url") + return None + + def translate_messages_to_responses_input( + self, + messages: List[ + Union[ + AnthropicMessagesUserMessageParam, + AnthopicMessagesAssistantMessageParam, + ] + ], + ) -> List[Dict[str, Any]]: + """ + Convert Anthropic messages list to Responses API `input` items. + + Mapping: + user text -> message(role=user, input_text) + user image -> message(role=user, input_image) + user tool_result -> function_call_output + assistant text -> message(role=assistant, output_text) + assistant tool_use -> function_call + """ + input_items: List[Dict[str, Any]] = [] + + for m in messages: + role = m["role"] + content = m.get("content") + + if role == "user": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": content}], + }) + elif isinstance(content, list): + user_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + user_parts.append({"type": "input_text", "text": block.get("text", "")}) + elif btype == "image": + url = self._translate_anthropic_image_source_to_url(block.get("source", {})) + if url: + user_parts.append({"type": "input_image", "image_url": url}) + elif btype == "tool_result": + tool_use_id = block.get("tool_use_id", "") + inner = block.get("content") + if inner is None: + output_text = "" + elif isinstance(inner, str): + output_text = inner + elif isinstance(inner, list): + parts = [ + c.get("text", "") + for c in inner + if isinstance(c, dict) and c.get("type") == "text" + ] + output_text = "\n".join(parts) + else: + output_text = str(inner) + # tool_result is a top-level item, not inside the message + input_items.append({ + "type": "function_call_output", + "call_id": tool_use_id, + "output": output_text, + }) + if user_parts: + input_items.append({ + "type": "message", + "role": "user", + "content": user_parts, + }) + + elif role == "assistant": + if isinstance(content, str): + input_items.append({ + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": content}], + }) + elif isinstance(content, list): + asst_parts: List[Dict[str, Any]] = [] + for block in content: + if not isinstance(block, dict): + continue + btype = block.get("type") + if btype == "text": + asst_parts.append({"type": "output_text", "text": block.get("text", "")}) + elif btype == "tool_use": + # tool_use becomes a top-level function_call item + input_items.append({ + "type": "function_call", + "call_id": block.get("id", ""), + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }) + elif btype == "thinking": + thinking_text = block.get("thinking", "") + if thinking_text: + asst_parts.append({"type": "output_text", "text": thinking_text}) + if asst_parts: + input_items.append({ + "type": "message", + "role": "assistant", + "content": asst_parts, + }) + + return input_items + + def translate_tools_to_responses_api( + self, + tools: List[AllAnthropicToolsValues], + ) -> List[Dict[str, Any]]: + """Convert Anthropic tool definitions to Responses API function tools.""" + result: List[Dict[str, Any]] = [] + for tool in tools: + tool_dict = cast(Dict[str, Any], tool) + tool_type = tool_dict.get("type", "") + tool_name = tool_dict.get("name", "") + # web_search tool + if (isinstance(tool_type, str) and tool_type.startswith("web_search")) or tool_name == "web_search": + result.append({"type": "web_search_preview"}) + continue + func_tool: Dict[str, Any] = {"type": "function", "name": tool_name} + if "description" in tool_dict: + func_tool["description"] = tool_dict["description"] + if "input_schema" in tool_dict: + func_tool["parameters"] = tool_dict["input_schema"] + result.append(func_tool) + return result + + @staticmethod + def translate_tool_choice_to_responses_api( + tool_choice: AnthropicMessagesToolChoice, + ) -> Dict[str, Any]: + """Convert Anthropic tool_choice to Responses API tool_choice.""" + tc_type = tool_choice.get("type") + if tc_type == "any": + return {"type": "required"} + elif tc_type == "tool": + return {"type": "function", "name": tool_choice.get("name", "")} + return {"type": "auto"} + + @staticmethod + def translate_context_management_to_responses_api( + context_management: Dict[str, Any], + ) -> Optional[List[Dict[str, Any]]]: + """ + Convert Anthropic context_management dict to OpenAI Responses API array format. + + Anthropic format: {"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]} + OpenAI format: [{"type": "compaction", "compact_threshold": 150000}] + """ + if not isinstance(context_management, dict): + return None + + edits = context_management.get("edits", []) + if not isinstance(edits, list): + return None + + result: List[Dict[str, Any]] = [] + for edit in edits: + if not isinstance(edit, dict): + continue + edit_type = edit.get("type", "") + if edit_type == "compact_20260112": + entry: Dict[str, Any] = {"type": "compaction"} + trigger = edit.get("trigger") + if isinstance(trigger, dict) and trigger.get("value") is not None: + entry["compact_threshold"] = int(trigger["value"]) + result.append(entry) + + return result if result else None + + @staticmethod + def translate_thinking_to_reasoning(thinking: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """ + Convert Anthropic thinking param to Responses API reasoning param. + + thinking.budget_tokens maps to reasoning effort: + >= 10000 -> high, >= 5000 -> medium, >= 2000 -> low, < 2000 -> minimal + """ + if not isinstance(thinking, dict) or thinking.get("type") != "enabled": + return None + budget = thinking.get("budget_tokens", 0) + if budget >= 10000: + effort = "high" + elif budget >= 5000: + effort = "medium" + elif budget >= 2000: + effort = "low" + else: + effort = "minimal" + return {"effort": effort, "summary": "detailed"} + + def translate_request( + self, + anthropic_request: AnthropicMessagesRequest, + ) -> Dict[str, Any]: + """ + Translate a full Anthropic /v1/messages request dict to + litellm.responses() / litellm.aresponses() kwargs. + """ + model: str = anthropic_request["model"] + messages_list = cast( + List[Union[AnthropicMessagesUserMessageParam, AnthopicMessagesAssistantMessageParam]], + anthropic_request["messages"], + ) + + responses_kwargs: Dict[str, Any] = { + "model": model, + "input": self.translate_messages_to_responses_input(messages_list), + } + + # system -> instructions + system = anthropic_request.get("system") + if system: + if isinstance(system, str): + responses_kwargs["instructions"] = system + elif isinstance(system, list): + text_parts = [ + b.get("text", "") + for b in system + if isinstance(b, dict) and b.get("type") == "text" + ] + responses_kwargs["instructions"] = "\n".join(filter(None, text_parts)) + + # max_tokens -> max_output_tokens + max_tokens = anthropic_request.get("max_tokens") + if max_tokens: + responses_kwargs["max_output_tokens"] = max_tokens + + # temperature / top_p passed through + if "temperature" in anthropic_request: + responses_kwargs["temperature"] = anthropic_request["temperature"] + if "top_p" in anthropic_request: + responses_kwargs["top_p"] = anthropic_request["top_p"] + + # tools + tools = anthropic_request.get("tools") + if tools: + responses_kwargs["tools"] = self.translate_tools_to_responses_api( + cast(List[AllAnthropicToolsValues], tools) + ) + + # tool_choice + tool_choice = anthropic_request.get("tool_choice") + if tool_choice: + responses_kwargs["tool_choice"] = self.translate_tool_choice_to_responses_api( + cast(AnthropicMessagesToolChoice, tool_choice) + ) + + # thinking -> reasoning + thinking = anthropic_request.get("thinking") + if isinstance(thinking, dict): + reasoning = self.translate_thinking_to_reasoning(thinking) + if reasoning: + responses_kwargs["reasoning"] = reasoning + + # output_format / output_config.format -> text format + # output_format: {"type": "json_schema", "schema": {...}} + # output_config: {"format": {"type": "json_schema", "schema": {...}}} + output_format = anthropic_request.get("output_format") + output_config = anthropic_request.get("output_config") + if not isinstance(output_format, dict) and isinstance(output_config, dict): + output_format = output_config.get("format") + if isinstance(output_format, dict) and output_format.get("type") == "json_schema": + schema = output_format.get("schema") + if schema: + responses_kwargs["text"] = { + "format": { + "type": "json_schema", + "name": "structured_output", + "schema": schema, + "strict": True, + } + } + + # context_management: Anthropic dict -> OpenAI array + context_management = anthropic_request.get("context_management") + if isinstance(context_management, dict): + openai_cm = self.translate_context_management_to_responses_api(context_management) + if openai_cm is not None: + responses_kwargs["context_management"] = openai_cm + + # metadata user_id -> user + metadata = anthropic_request.get("metadata") + if isinstance(metadata, dict) and "user_id" in metadata: + responses_kwargs["user"] = str(metadata["user_id"])[:64] + + return responses_kwargs + + # ------------------------------------------------------------------ # + # Response translation: Responses API -> Anthropic # + # ------------------------------------------------------------------ # + + def translate_response( + self, + response: ResponsesAPIResponse, + ) -> AnthropicMessagesResponse: + """ + Translate an OpenAI ResponsesAPIResponse to AnthropicMessagesResponse. + """ + from openai.types.responses import ( + ResponseFunctionToolCall, + ResponseOutputMessage, + ResponseReasoningItem, + ) + + from litellm.types.llms.openai import ResponseAPIUsage + + content: List[Dict[str, Any]] = [] + stop_reason: AnthropicFinishReason = "end_turn" + + for item in response.output: + if isinstance(item, ResponseReasoningItem): + for summary in item.summary: + text = getattr(summary, "text", "") + if text: + content.append( + AnthropicResponseContentBlockThinking( + type="thinking", + thinking=text, + signature=None, + ).model_dump() + ) + + elif isinstance(item, ResponseOutputMessage): + for part in item.content: + if getattr(part, "type", None) == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=getattr(part, "text", "") + ).model_dump() + ) + + elif isinstance(item, ResponseFunctionToolCall): + try: + input_data = json.loads(item.arguments) if item.arguments else {} + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.call_id or item.id, + name=item.name, + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + elif isinstance(item, dict): + item_type = item.get("type") + if item_type == "message": + for part in item.get("content", []): + if isinstance(part, dict) and part.get("type") == "output_text": + content.append( + AnthropicResponseContentBlockText( + type="text", text=part.get("text", "") + ).model_dump() + ) + elif item_type == "function_call": + try: + input_data = json.loads(item.get("arguments", "{}")) + except (json.JSONDecodeError, TypeError): + input_data = {} + content.append( + AnthropicResponseContentBlockToolUse( + type="tool_use", + id=item.get("call_id") or item.get("id", ""), + name=item.get("name", ""), + input=input_data, + ).model_dump() + ) + stop_reason = "tool_use" + + # status -> stop_reason override + if response.status == "incomplete": + stop_reason = "max_tokens" + + # usage + raw_usage: Optional[ResponseAPIUsage] = response.usage + input_tokens = int(getattr(raw_usage, "input_tokens", 0) or 0) + output_tokens = int(getattr(raw_usage, "output_tokens", 0) or 0) + + anthropic_usage = AnthropicUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + ) + + return AnthropicMessagesResponse( + id=response.id, + type="message", + role="assistant", + model=response.model or "unknown-model", + stop_sequence=None, + usage=anthropic_usage, # type: ignore + content=content, # type: ignore + stop_reason=stop_reason, + ) diff --git a/litellm/llms/base_llm/ocr/transformation.py b/litellm/llms/base_llm/ocr/transformation.py index fb13332c464..29929a2bf62 100644 --- a/litellm/llms/base_llm/ocr/transformation.py +++ b/litellm/llms/base_llm/ocr/transformation.py @@ -15,7 +15,9 @@ else: LiteLLMLoggingObj = Any -# DocumentType for OCR - Mistral format document dict +# DocumentType for OCR - providers always receive a dict with +# type="document_url" or type="image_url" (str values only). +# File-type inputs are preprocessed to this format in litellm/ocr/main.py. DocumentType = Dict[str, str] @@ -141,9 +143,13 @@ class BaseOCRConfig: Transform OCR request to provider-specific format. Override in provider-specific implementations. + Note: By the time this method is called, any file-type documents have already + been converted to document_url/image_url format with base64 data URIs by + the preprocessing in litellm/ocr/main.py. + Args: model: Model name - document: Document to process (Mistral format dict, or file path, bytes, etc.) + document: Document to process - always a dict with type="document_url" or type="image_url" optional_params: Optional parameters for the request headers: Request headers diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index d4fd0606302..306d63b77d0 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -511,7 +511,6 @@ class AmazonConverseConfig(BaseConfig): "response_format", "requestMetadata", "service_tier", - "parallel_tool_calls", ] if ( @@ -914,13 +913,6 @@ class AmazonConverseConfig(BaseConfig): ) if _tool_choice_value is not None: optional_params["tool_choice"] = _tool_choice_value - if param == "parallel_tool_calls": - disable_parallel = not value - optional_params["_parallel_tool_use_config"] = { - "tool_choice": { - "disable_parallel_tool_use": disable_parallel - } - } if param == "thinking": optional_params["thinking"] = value elif param == "reasoning_effort" and isinstance(value, str): @@ -1217,15 +1209,15 @@ class AmazonConverseConfig(BaseConfig): # Handle parallel_tool_calls configuration parallel_tool_use_config = additional_request_params.pop("_parallel_tool_use_config", None) - if parallel_tool_use_config is not None: - # Merge the tool_choice config from parallel_tool_calls into additional_request_params + if parallel_tool_use_config is not None and is_claude_4_5_on_bedrock(model): for key, value in parallel_tool_use_config.items(): if key in additional_request_params and isinstance(additional_request_params[key], dict) and isinstance(value, dict): - # Merge dictionaries additional_request_params[key].update(value) else: additional_request_params[key] = value + additional_request_params.pop("parallel_tool_calls", None) + # Only set the topK value in for models that support it additional_request_params.update( self._handle_top_k_value(model, inference_params) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b09a36be60f..d6fdc58099f 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -1,4 +1,5 @@ import json +import ssl from typing import ( TYPE_CHECKING, Any, @@ -4659,6 +4660,8 @@ class BaseLLMHTTPHandler: api_key: Optional[str] = None, client: Optional[Any] = None, timeout: Optional[float] = None, + user_api_key_dict: Optional[Any] = None, + litellm_metadata: Optional[Dict[str, Any]] = None, ): import websockets from websockets.asyncio.client import ClientConnection @@ -4672,6 +4675,11 @@ class BaseLLMHTTPHandler: try: ssl_context = get_shared_realtime_ssl_context() + if url.startswith("wss://") and ssl_context is False: + # Keep TLS for wss:// while honoring SSL_VERIFY=False semantics. + ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ssl_context.check_hostname = False + ssl_context.verify_mode = ssl.CERT_NONE async with websockets.connect( # type: ignore url, additional_headers=headers, @@ -4686,12 +4694,17 @@ class BaseLLMHTTPHandler: if _session_config: await backend_ws.send(_session_config) + _request_data: Dict[str, Any] = {} + if litellm_metadata: + _request_data["litellm_metadata"] = litellm_metadata realtime_streaming = RealTimeStreaming( websocket, cast(ClientConnection, backend_ws), logging_obj, provider_config, model, + user_api_key_dict=user_api_key_dict, + request_data=_request_data, ) if _session_config: realtime_streaming.session_configuration_request = _session_config diff --git a/litellm/llms/gemini/image_generation/cost_calculator.py b/litellm/llms/gemini/image_generation/cost_calculator.py index 0a9ca2e5276..941ab0d50f7 100644 --- a/litellm/llms/gemini/image_generation/cost_calculator.py +++ b/litellm/llms/gemini/image_generation/cost_calculator.py @@ -5,6 +5,9 @@ Google AI Image Generation Cost Calculator from typing import Any import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, +) from litellm.types.utils import ImageResponse @@ -13,13 +16,22 @@ def cost_calculator( image_response: Any, ) -> float: """ - Vertex AI Image Generation Cost Calculator + Google AI Image Generation Cost Calculator """ _model_info = litellm.get_model_info( model=model, custom_llm_provider="gemini", ) + if isinstance(image_response, ImageResponse): + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="gemini", + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if isinstance(image_response, ImageResponse): diff --git a/litellm/llms/gemini/realtime/transformation.py b/litellm/llms/gemini/realtime/transformation.py index 2e0e678e69f..d9465c95e3b 100644 --- a/litellm/llms/gemini/realtime/transformation.py +++ b/litellm/llms/gemini/realtime/transformation.py @@ -867,6 +867,52 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): ) returned_message: List[OpenAIRealtimeEvents] = [] + # Handle transcription events that arrive independently from model + # content. Gemini sends inputTranscription / outputTranscription + # inside serverContent, separately from modelTurn / turnComplete. + server_content = json_message.get("serverContent") + if isinstance(server_content, dict): + input_tx = server_content.get("inputTranscription") + if isinstance(input_tx, dict) and input_tx.get("text"): + returned_message.append( + cast(OpenAIRealtimeEvents, { + "type": "conversation.item.input_audio_transcription.completed", + "event_id": "event_{}".format(uuid.uuid4()), + "transcript": input_tx["text"], + "item_id": "item_{}".format(uuid.uuid4()), + "content_index": 0, + }) + ) + + output_tx = server_content.get("outputTranscription") + if isinstance(output_tx, dict) and output_tx.get("text"): + returned_message.append( + cast(OpenAIRealtimeEvents, { + "type": "response.audio_transcript.delta", + "event_id": "event_{}".format(uuid.uuid4()), + "delta": output_tx["text"], + "item_id": current_output_item_id or "item_{}".format(uuid.uuid4()), + "response_id": current_response_id or "resp_{}".format(uuid.uuid4()), + "output_index": 0, + "content_index": 0, + }) + ) + + # If serverContent only contained transcription(s) and no model + # content, return early — the main loop would fail on unknown keys. + _model_content_keys = {"modelTurn", "turnComplete", "interrupted", "generationComplete"} + if not any(k in server_content for k in _model_content_keys): + return { + "response": returned_message, + "current_output_item_id": current_output_item_id, + "current_response_id": current_response_id, + "current_delta_chunks": current_delta_chunks, + "current_conversation_id": current_conversation_id, + "current_item_chunks": current_item_chunks, + "current_delta_type": current_delta_type, + "session_configuration_request": session_configuration_request, + } + for key, value in json_message.items(): # Check if this key or any nested key matches our mapping openai_event = self.map_openai_event( @@ -974,6 +1020,8 @@ class GeminiRealtimeConfig(BaseRealtimeConfig): setup_config: BidiGenerateContentSetup = { "model": f"models/{model}", "generationConfig": {"responseModalities": response_modalities}, + # Return input transcript so guardrails can inspect user speech. + "inputAudioTranscription": {}, } if output_audio_transcription: setup_config["outputAudioTranscription"] = {} diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index 646c6080a2e..012de5498cb 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -3,6 +3,9 @@ Vertex AI Image Generation Cost Calculator """ import litellm +from litellm.litellm_core_utils.llm_cost_calc.utils import ( + calculate_image_response_cost_from_usage, +) from litellm.types.utils import ImageResponse @@ -18,6 +21,14 @@ def cost_calculator( custom_llm_provider="vertex_ai", ) + token_based_cost = calculate_image_response_cost_from_usage( + model=model, + image_response=image_response, + custom_llm_provider="vertex_ai", + ) + if token_based_cost is not None: + return token_based_cost + output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 num_images: int = 0 if image_response.data: diff --git a/litellm/llms/vertex_ai/realtime/transformation.py b/litellm/llms/vertex_ai/realtime/transformation.py index eaa9844f108..5eae143175b 100644 --- a/litellm/llms/vertex_ai/realtime/transformation.py +++ b/litellm/llms/vertex_ai/realtime/transformation.py @@ -124,6 +124,8 @@ class VertexAIRealtimeConfig(GeminiRealtimeConfig): "silenceDurationMs": 800, } }, + # Return input transcript so guardrails can inspect user speech. + "inputAudioTranscription": {}, # Return output transcript so clients can read what the model said. "outputAudioTranscription": {}, } diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 57563fc0bcc..b21f23ac022 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -31545,6 +31577,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 5acab8cbf2c..47cff8a2c0c 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -2,8 +2,14 @@ Main OCR function for LiteLLM. """ import asyncio +import base64 import contextvars +import mimetypes +import os +import re from functools import partial +from io import IOBase +from pathlib import Path from typing import Any, Coroutine, Dict, Optional, Union import httpx @@ -25,7 +31,7 @@ base_llm_http_handler = BaseLLMHTTPHandler() @client async def aocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -35,26 +41,27 @@ async def aocr( ) -> OCRResponse: """ Async OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -64,7 +71,7 @@ async def aocr( }, include_image_base64=True ) - + # OCR with image response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -73,7 +80,7 @@ async def aocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = await litellm.aocr( model="mistral/mistral-ocr-latest", @@ -82,6 +89,12 @@ async def aocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) + + # OCR with local file + response = await litellm.aocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) ``` """ local_vars = locals() @@ -135,7 +148,7 @@ async def aocr( @client def ocr( model: str, - document: Dict[str, str], + document: Dict[str, Any], api_key: Optional[str] = None, api_base: Optional[str] = None, timeout: Optional[Union[float, httpx.Timeout]] = None, @@ -145,26 +158,27 @@ def ocr( ) -> Union[OCRResponse, Coroutine[Any, Any, OCRResponse]]: """ Synchronous OCR function. - + Args: model: Model name (e.g., "mistral/mistral-ocr-latest") document: Document to process in Mistral format: - {"type": "document_url", "document_url": "https://..."} for PDFs/docs or - {"type": "image_url", "image_url": "https://..."} for images + {"type": "document_url", "document_url": "https://..."} for PDFs/docs, + {"type": "image_url", "image_url": "https://..."} for images, or + {"type": "file", "file": } for local files api_key: Optional API key api_base: Optional API base URL timeout: Optional timeout custom_llm_provider: Optional custom LLM provider extra_headers: Optional extra headers **kwargs: Additional parameters (e.g., include_image_base64, pages, image_limit) - + Returns: OCRResponse in Mistral OCR format with pages, model, usage_info, etc. - + Example: ```python import litellm - + # OCR with PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -174,7 +188,7 @@ def ocr( }, include_image_base64=True ) - + # OCR with image response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -183,7 +197,7 @@ def ocr( "image_url": "https://example.com/image.png" } ) - + # OCR with base64 encoded PDF response = litellm.ocr( model="mistral/mistral-ocr-latest", @@ -192,7 +206,13 @@ def ocr( "document_url": f"data:application/pdf;base64,{base64_pdf}" } ) - + + # OCR with local file + response = litellm.ocr( + model="mistral/mistral-ocr-latest", + document={"type": "file", "file": "/path/to/document.pdf"} + ) + # Access pages for page in response.pages: print(f"Page {page.index}: {page.markdown}") @@ -203,24 +223,38 @@ def ocr( litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) _is_async = kwargs.pop("aocr", False) is True - - # Validate document parameter format (Mistral spec) - if not isinstance(document, dict): - raise ValueError(f"document must be a dict with 'type' and URL field, got {type(document)}") - - doc_type = document.get("type") - if doc_type not in ["document_url", "image_url"]: - raise ValueError(f"Invalid document type: {doc_type}. Must be 'document_url' or 'image_url'") - model, custom_llm_provider, dynamic_api_key, dynamic_api_base = ( - litellm.get_llm_provider( - model=model, - custom_llm_provider=custom_llm_provider, - api_base=api_base, - api_key=api_key, + # Validate document parameter format + if not isinstance(document, dict): + raise ValueError( + f"document must be a dict with 'type' and URL/file field, got {type(document)}" ) + + doc_type = document.get("type") + + # Handle file type: convert to document_url/image_url with base64 data URI + if doc_type == "file": + document = convert_file_document_to_url_document(document) + doc_type = document.get("type") + + if doc_type not in ["document_url", "image_url"]: + raise ValueError( + f"Invalid document type: {doc_type}. " + "Must be 'document_url', 'image_url', or 'file'" + ) + + ( + model, + custom_llm_provider, + dynamic_api_key, + dynamic_api_base, + ) = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, ) - + # Update with dynamic values if available if dynamic_api_key: api_key = dynamic_api_key @@ -228,11 +262,11 @@ def ocr( api_base = dynamic_api_base # Get provider config - ocr_provider_config: Optional[BaseOCRConfig] = ( - ProviderConfigManager.get_provider_ocr_config( - model=model, - provider=litellm.LlmProviders(custom_llm_provider), - ) + ocr_provider_config: Optional[ + BaseOCRConfig + ] = ProviderConfigManager.get_provider_ocr_config( + model=model, + provider=litellm.LlmProviders(custom_llm_provider), ) if ocr_provider_config is None: @@ -246,21 +280,21 @@ def ocr( # Get litellm params using GenericLiteLLMParams (same as responses API) litellm_params = GenericLiteLLMParams(**kwargs) - + # Extract OCR-specific parameters from kwargs supported_params = ocr_provider_config.get_supported_ocr_params(model=model) non_default_params = {} for param in supported_params: if param in kwargs: non_default_params[param] = kwargs.pop(param) - + # Map parameters to provider-specific format optional_params = ocr_provider_config.map_ocr_params( non_default_params=non_default_params, optional_params={}, model=model, ) - + verbose_logger.debug(f"OCR optional_params after mapping: {optional_params}") # Pre Call logging @@ -300,3 +334,111 @@ def ocr( extra_kwargs=kwargs, ) + +################################################# +# Public utilities — used by the SDK and the proxy +################################################# + +_MIME_PATTERN = re.compile(r"^[\w.+-]+/[\w.+-]+$") + +_MIME_TYPE_MAP = { + ".pdf": "application/pdf", + ".png": "image/png", + ".jpg": "image/jpeg", + ".jpeg": "image/jpeg", + ".gif": "image/gif", + ".webp": "image/webp", + ".tiff": "image/tiff", + ".tif": "image/tiff", + ".bmp": "image/bmp", +} + + +def get_mime_type(file_path: str) -> str: + """ + Determine MIME type from file path extension. + + Falls back to mimetypes.guess_type, then to 'application/octet-stream'. + """ + ext = os.path.splitext(file_path)[1].lower() + mime = _MIME_TYPE_MAP.get(ext) + if mime: + return mime + guessed, _ = mimetypes.guess_type(file_path) + return guessed or "application/octet-stream" + + +def convert_file_document_to_url_document(document: Dict[str, Any]) -> Dict[str, str]: + """ + Convert a file-type document dict to a document_url-type document dict + with an inline base64 data URI. + + Accepts document dicts like: + {"type": "file", "file": "/path/to/document.pdf"} # file path string + {"type": "file", "file": Path("/path/to/doc.pdf")} # pathlib.Path + {"type": "file", "file": } # file-like object (BinaryIO) + {"type": "file", "file": b"raw bytes"} # raw bytes + + Returns: + {"type": "document_url", "document_url": "data:;base64,"} + or {"type": "image_url", "image_url": "data:;base64,"} + """ + file_input = document.get("file") + if file_input is None: + raise ValueError( + "document with type='file' must include a 'file' field containing " + "a file path (str), pathlib.Path, file-like object, or bytes" + ) + + file_bytes: bytes + mime_type: str = "application/octet-stream" + file_name: Optional[str] = None + + if isinstance(file_input, (str, Path)): + file_path = str(file_input) + if not os.path.isfile(file_path): + raise FileNotFoundError(f"File not found: {file_path}") + mime_type = get_mime_type(file_path) + file_name = os.path.basename(file_path) + with open(file_path, "rb") as f: + file_bytes = f.read() + elif isinstance(file_input, bytes): + file_bytes = file_input + elif isinstance(file_input, IOBase) or hasattr(file_input, "read"): + if hasattr(file_input, "name"): + file_name = getattr(file_input, "name", None) + if file_name: + mime_type = get_mime_type(file_name) + file_bytes = file_input.read() + if isinstance(file_bytes, str): + file_bytes = file_bytes.encode("utf-8") + else: + raise ValueError( + f"Unsupported file input type: {type(file_input)}. " + "Expected str (file path), pathlib.Path, bytes, or a file-like object." + ) + + if not file_bytes: + raise ValueError("File is empty or could not be read") + + if "mime_type" in document: + mime_type = document["mime_type"] + + if not _MIME_PATTERN.match(mime_type): + raise ValueError(f"Invalid MIME type: {mime_type}") + + base64_data = base64.b64encode(file_bytes).decode("utf-8") + data_uri = f"data:{mime_type};base64,{base64_data}" + + if mime_type.startswith("image/"): + verbose_logger.debug( + f"OCR file input: Converted file to image_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "image_url", "image_url": data_uri} + else: + verbose_logger.debug( + f"OCR file input: Converted file to document_url data URI " + f"(mime={mime_type}, size={len(file_bytes)} bytes, name={file_name})" + ) + return {"type": "document_url", "document_url": data_uri} diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json new file mode 100644 index 00000000000..fc79ba54759 --- /dev/null +++ b/litellm/provider_endpoints_support_backup.json @@ -0,0 +1,2748 @@ +{ + "_comment": "This file defines which endpoints are supported by each LiteLLM provider", + "_schema": { + "provider_slug": { + "display_name": "Display name shown in README (e.g., 'OpenAI (`openai`)')", + "url": "Link to provider documentation", + "endpoints": { + "chat_completions": "Supports /chat/completions endpoint", + "messages": "Supports /messages endpoint (Anthropic format)", + "responses": "Supports /responses endpoint (OpenAI/Anthropic unified)", + "embeddings": "Supports /embeddings endpoint", + "image_generations": "Supports /image/generations endpoint", + "audio_transcriptions": "Supports /audio/transcriptions endpoint", + "audio_speech": "Supports /audio/speech endpoint", + "moderations": "Supports /moderations endpoint", + "batches": "Supports /batches endpoint", + "rerank": "Supports /rerank endpoint", + "ocr": "Supports /ocr endpoint", + "search": "Supports /search endpoint", + "skills": "Supports /skills endpoint", + "interactions": "Supports /interactions endpoint (Google AI Interactions API)", + "a2a": "Supports /a2a/{agent}/message/send endpoint (A2A Protocol)", + "container": "Supports OpenAI's /containers endpoint", + "container_files": "Supports OpenAI's /containers/{id}/files endpoint", + "compact": "Supports /responses/compact endpoint", + "files": "Supports /files endpoint for file operations", + "image_edits": "Supports /images/edits endpoint for image editing", + "vector_stores_create": "Supports creating a new vector store via /vector_stores endpoint", + "vector_stores_search": "Supports searching a vector store via /vector_stores/{id}/search endpoint", + "video_generations": "Supports /videos/generations endpoint for video generation" + } + } + }, + "providers": { + "a2a": { + "display_name": "A2A (Agent-to-Agent) (`a2a`)", + "url": "https://docs.litellm.ai/docs/providers/a2a", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "abliteration": { + "display_name": "Abliteration (`abliteration`)", + "url": "https://docs.litellm.ai/docs/providers/abliteration", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "aiml": { + "display_name": "AI/ML API (`aiml`)", + "url": "https://docs.litellm.ai/docs/providers/aiml", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ai21": { + "display_name": "AI21 (`ai21`)", + "url": "https://docs.litellm.ai/docs/providers/ai21", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ai21_chat": { + "display_name": "AI21 Chat (`ai21_chat`)", + "url": "https://docs.litellm.ai/docs/providers/ai21", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "amazon_nova": { + "display_name": "Amazon Nova (`amazon_nova`)", + "url": "https://docs.litellm.ai/docs/providers/amazon_nova", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "anthropic": { + "display_name": "Anthropic (`anthropic`)", + "url": "https://docs.litellm.ai/docs/providers/anthropic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "rerank": false, + "skills": true, + "a2a": true, + "interactions": true, + "count_tokens": true + } + }, + "anthropic_text": { + "display_name": "Anthropic Text (`anthropic_text`)", + "url": "https://docs.litellm.ai/docs/providers/anthropic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "rerank": false, + "skills": true, + "a2a": true, + "interactions": true + } + }, + "apertis": { + "display_name": "Apertis (`apertis`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "assemblyai": { + "display_name": "AssemblyAI (`assemblyai`)", + "url": "https://docs.litellm.ai/docs/pass_through/assembly_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "auto_router": { + "display_name": "Auto Router (`auto_router`)", + "url": "https://docs.litellm.ai/docs/proxy/auto_routing", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "bedrock": { + "display_name": "AWS - Bedrock (`bedrock`)", + "url": "https://docs.litellm.ai/docs/providers/bedrock", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true, + "bedrock_invoke": true, + "bedrock_converse": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "rag_query": true + } + }, + "s3_vectors": { + "display_name": "AWS S3 Vectors (`s3_vectors`)", + "url": "https://docs.litellm.ai/docs/providers/s3_vectors", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false, + "vector_stores_create": true, + "vector_stores_search": true + } + }, + "sagemaker": { + "display_name": "AWS - Sagemaker (`sagemaker`)", + "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "aws_polly": { + "display_name": "AWS - Polly (`aws_polly`)", + "url": "https://docs.litellm.ai/docs/providers/aws_polly", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "azure": { + "display_name": "Azure (`azure`)", + "url": "https://docs.litellm.ai/docs/providers/azure", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true, + "vector_stores_search": true, + "assistants": true, + "fine_tuning": true, + "text_completion": true + } + }, + "azure_ai": { + "display_name": "Azure AI (`azure_ai`)", + "url": "https://docs.litellm.ai/docs/providers/azure_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true, + "vector_stores_create": true, + "vector_stores_search": true + } + }, + "azure_ai/doc-intelligence": { + "display_name": "Azure AI Document Intelligence (`azure_ai/doc-intelligence`)", + "url": "https://docs.litellm.ai/docs/providers/azure_document_intelligence", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true + } + }, + "azure_ai/agents": { + "display_name": "Azure AI Foundry Agents (`azure_ai/agents`)", + "url": "https://docs.litellm.ai/docs/providers/azure_ai_agents", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "azure_text": { + "display_name": "Azure Text (`azure_text`)", + "url": "https://docs.litellm.ai/docs/providers/azure", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "baseten": { + "display_name": "Baseten (`baseten`)", + "url": "https://docs.litellm.ai/docs/providers/baseten", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "bytez": { + "display_name": "Bytez (`bytez`)", + "url": "https://docs.litellm.ai/docs/providers/bytez", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cerebras": { + "display_name": "Cerebras (`cerebras`)", + "url": "https://docs.litellm.ai/docs/providers/cerebras", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "chutes": { + "display_name": "Chutes (`chutes`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "clarifai": { + "display_name": "Clarifai (`clarifai`)", + "url": "https://docs.litellm.ai/docs/providers/clarifai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cloudflare": { + "display_name": "Cloudflare AI Workers (`cloudflare`)", + "url": "https://docs.litellm.ai/docs/providers/cloudflare_workers", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "codestral": { + "display_name": "Codestral (`codestral`)", + "url": "https://docs.litellm.ai/docs/providers/codestral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cohere": { + "display_name": "Cohere (`cohere`)", + "url": "https://docs.litellm.ai/docs/providers/cohere", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "cohere_chat": { + "display_name": "Cohere Chat (`cohere_chat`)", + "url": "https://docs.litellm.ai/docs/providers/cohere", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "cometapi": { + "display_name": "CometAPI (`cometapi`)", + "url": "https://docs.litellm.ai/docs/providers/cometapi", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "compactifai": { + "display_name": "CompactifAI (`compactifai`)", + "url": "https://docs.litellm.ai/docs/providers/compactifai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "custom": { + "display_name": "Custom (`custom`)", + "url": "https://docs.litellm.ai/docs/providers/custom_llm_server", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "custom_openai": { + "display_name": "Custom OpenAI (`custom_openai`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "dashscope": { + "display_name": "Dashscope (`dashscope`)", + "url": "https://docs.litellm.ai/docs/providers/dashscope", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "databricks": { + "display_name": "Databricks (`databricks`)", + "url": "https://docs.litellm.ai/docs/providers/databricks", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "dataforseo": { + "display_name": "DataForSEO (`dataforseo`)", + "url": "https://docs.litellm.ai/docs/search/dataforseo", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "datarobot": { + "display_name": "DataRobot (`datarobot`)", + "url": "https://docs.litellm.ai/docs/providers/datarobot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepgram": { + "display_name": "Deepgram (`deepgram`)", + "url": "https://docs.litellm.ai/docs/providers/deepgram", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepinfra": { + "display_name": "DeepInfra (`deepinfra`)", + "url": "https://docs.litellm.ai/docs/providers/deepinfra", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "deepseek": { + "display_name": "Deepseek (`deepseek`)", + "url": "https://docs.litellm.ai/docs/providers/deepseek", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "duckduckgo": { + "display_name": "DuckDuckGo (`duckduckgo`)", + "url": "https://docs.litellm.ai/docs/search/duckduckgo", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "elevenlabs": { + "display_name": "ElevenLabs (`elevenlabs`)", + "url": "https://docs.litellm.ai/docs/providers/elevenlabs", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "exa_ai": { + "display_name": "Exa AI (`exa_ai`)", + "url": "https://docs.litellm.ai/docs/search/exa_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "brave": { + "display_name": "Brave Search (`brave`)", + "url": "https://docs.litellm.ai/docs/search/brave", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "empower": { + "display_name": "Empower (`empower`)", + "url": "https://docs.litellm.ai/docs/providers/empower", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "fal_ai": { + "display_name": "Fal AI (`fal_ai`)", + "url": "https://docs.litellm.ai/docs/providers/fal_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "featherless_ai": { + "display_name": "Featherless AI (`featherless_ai`)", + "url": "https://docs.litellm.ai/docs/providers/featherless_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "fireworks_ai": { + "display_name": "Fireworks AI (`fireworks_ai`)", + "url": "https://docs.litellm.ai/docs/providers/fireworks_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "firecrawl": { + "display_name": "Firecrawl (`firecrawl`)", + "url": "https://docs.litellm.ai/docs/search/firecrawl", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "linkup": { + "display_name": "Linkup (`linkup`)", + "url": "https://docs.litellm.ai/docs/search/linkup", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "friendliai": { + "display_name": "FriendliAI (`friendliai`)", + "url": "https://docs.litellm.ai/docs/providers/friendliai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "galadriel": { + "display_name": "Galadriel (`galadriel`)", + "url": "https://docs.litellm.ai/docs/providers/galadriel", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "github_copilot": { + "display_name": "GitHub Copilot (`github_copilot`)", + "url": "https://docs.litellm.ai/docs/providers/github_copilot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "chatgpt": { + "display_name": "ChatGPT Subscription (`chatgpt`)", + "url": "https://docs.litellm.ai/docs/providers/chatgpt", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, + "github": { + "display_name": "GitHub Models (`github`)", + "url": "https://docs.litellm.ai/docs/providers/github", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "gmi": { + "display_name": "GMI Cloud (`gmi`)", + "url": "https://docs.litellm.ai/docs/providers/gmi_cloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vertex_ai": { + "display_name": "Google - Vertex AI (`vertex_ai`)", + "url": "https://docs.litellm.ai/docs/providers/vertex", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true, + "vector_stores_search": true, + "count_tokens": true, + "fine_tuning": true, + "rag_ingest": true, + "rag_query": true, + "generateContent": true, + "realtime": true + } + }, + "gemini": { + "display_name": "Google AI Studio - Gemini (`gemini`)", + "url": "https://docs.litellm.ai/docs/providers/gemini", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "interactions": true, + "a2a": true, + "vector_stores_search": true, + "count_tokens": true, + "rag_ingest": true, + "realtime": true, + "generateContent": true + } + }, + "gradient_ai": { + "display_name": "GradientAI (`gradient_ai`)", + "url": "https://docs.litellm.ai/docs/providers/gradient_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "groq": { + "display_name": "Groq AI (`groq`)", + "url": "https://docs.litellm.ai/docs/providers/groq", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "heroku": { + "display_name": "Heroku (`heroku`)", + "url": "https://docs.litellm.ai/docs/providers/heroku", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "hosted_vllm": { + "display_name": "Hosted VLLM (`hosted_vllm`)", + "url": "https://docs.litellm.ai/docs/providers/vllm", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "files": true, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "huggingface": { + "display_name": "Huggingface (`huggingface`)", + "url": "https://docs.litellm.ai/docs/providers/huggingface", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "hyperbolic": { + "display_name": "Hyperbolic (`hyperbolic`)", + "url": "https://docs.litellm.ai/docs/providers/hyperbolic", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "watsonx": { + "display_name": "IBM - Watsonx.ai (`watsonx`)", + "url": "https://docs.litellm.ai/docs/providers/watsonx", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "infinity": { + "display_name": "Infinity (`infinity`)", + "url": "https://docs.litellm.ai/docs/providers/infinity", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "jina_ai": { + "display_name": "Jina AI (`jina_ai`)", + "url": "https://docs.litellm.ai/docs/providers/jina_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "lambda_ai": { + "display_name": "Lambda AI (`lambda_ai`)", + "url": "https://docs.litellm.ai/docs/providers/lambda_ai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "lemonade": { + "display_name": "Lemonade (`lemonade`)", + "url": "https://docs.litellm.ai/docs/providers/lemonade", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "litellm_proxy": { + "display_name": "LiteLLM Proxy (`litellm_proxy`)", + "url": "https://docs.litellm.ai/docs/providers/litellm_proxy", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "llamafile": { + "display_name": "Llamafile (`llamafile`)", + "url": "https://docs.litellm.ai/docs/providers/llamafile", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "lm_studio": { + "display_name": "LM Studio (`lm_studio`)", + "url": "https://docs.litellm.ai/docs/providers/lm_studio", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "maritalk": { + "display_name": "Maritalk (`maritalk`)", + "url": "https://docs.litellm.ai/docs/providers/maritalk", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "meta_llama": { + "display_name": "Meta - Llama API (`meta_llama`)", + "url": "https://docs.litellm.ai/docs/providers/meta_llama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "mistral": { + "display_name": "Mistral AI API (`mistral`)", + "url": "https://docs.litellm.ai/docs/providers/mistral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "ocr": true, + "a2a": true, + "interactions": true + } + }, + "moonshot": { + "display_name": "Moonshot (`moonshot`)", + "url": "https://docs.litellm.ai/docs/providers/moonshot", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "docker_model_runner": { + "display_name": "Docker Model Runner (`docker_model_runner`)", + "url": "https://docs.litellm.ai/docs/providers/docker_model_runner", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "morph": { + "display_name": "Morph (`morph`)", + "url": "https://docs.litellm.ai/docs/providers/morph", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nanogpt": { + "display_name": "NanoGPT (`nanogpt`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "nebius": { + "display_name": "Nebius AI Studio (`nebius`)", + "url": "https://docs.litellm.ai/docs/providers/nebius", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nlp_cloud": { + "display_name": "NLP Cloud (`nlp_cloud`)", + "url": "https://docs.litellm.ai/docs/providers/nlp_cloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "novita": { + "display_name": "Novita AI (`novita`)", + "url": "https://novita.ai/models/llm?utm_source=github_litellm&utm_medium=github_readme&utm_campaign=github_link", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nscale": { + "display_name": "Nscale (`nscale`)", + "url": "https://docs.litellm.ai/docs/providers/nscale", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "nvidia_nim": { + "display_name": "Nvidia NIM (`nvidia_nim`)", + "url": "https://docs.litellm.ai/docs/providers/nvidia_nim", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "oci": { + "display_name": "OCI (`oci`)", + "url": "https://docs.litellm.ai/docs/providers/oci", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ollama": { + "display_name": "Ollama (`ollama`)", + "url": "https://docs.litellm.ai/docs/providers/ollama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ollama_chat": { + "display_name": "Ollama Chat (`ollama_chat`)", + "url": "https://docs.litellm.ai/docs/providers/ollama", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "oobabooga": { + "display_name": "Oobabooga (`oobabooga`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "openai": { + "display_name": "OpenAI (`openai`)", + "url": "https://docs.litellm.ai/docs/providers/openai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "container": true, + "compact": true, + "a2a": true, + "interactions": true, + "vector_store_files": true, + "vector_stores_create": true, + "vector_stores_search": true, + "assistants": true, + "container_files": true, + "fine_tuning": true, + "image_variations": true, + "rag_ingest": true, + "rag_query": true, + "realtime": true, + "text_completion": true + } + }, + "openai_like": { + "display_name": "OpenAI-like (`openai_like`)", + "url": "https://docs.litellm.ai/docs/providers/openai_compatible", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "assistants": true + } + }, + "openrouter": { + "display_name": "OpenRouter (`openrouter`)", + "url": "https://docs.litellm.ai/docs/providers/openrouter", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ovhcloud": { + "display_name": "OVHCloud AI Endpoints (`ovhcloud`)", + "url": "https://docs.litellm.ai/docs/providers/ovhcloud", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "parallel_ai": { + "display_name": "Parallel AI (`parallel_ai`)", + "url": "https://docs.litellm.ai/docs/search/parallel_ai", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "perplexity": { + "display_name": "Perplexity AI (`perplexity`)", + "url": "https://docs.litellm.ai/docs/providers/perplexity", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true, + "a2a": true, + "interactions": true + } + }, + "petals": { + "display_name": "Petals (`petals`)", + "url": "https://docs.litellm.ai/docs/providers/petals", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "poe": { + "display_name": "Poe (`poe`)", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "publicai": { + "display_name": "PublicAI (`publicai`)", + "url": "https://docs.litellm.ai/docs/providers/publicai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "predibase": { + "display_name": "Predibase (`predibase`)", + "url": "https://docs.litellm.ai/docs/providers/predibase", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "recraft": { + "display_name": "Recraft (`recraft`)", + "url": "https://docs.litellm.ai/docs/providers/recraft", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "replicate": { + "display_name": "Replicate (`replicate`)", + "url": "https://docs.litellm.ai/docs/providers/replicate", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "runwayml": { + "display_name": "RunwayML (`runwayml`)", + "url": "https://docs.litellm.ai/docs/providers/runwayml/videos", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "audio_transcriptions": false, + "audio_speech": true, + "moderations": false, + "batches": false, + "rerank": false, + "video_generations": true + } + }, + "sagemaker_chat": { + "display_name": "Sagemaker Chat (`sagemaker_chat`)", + "url": "https://docs.litellm.ai/docs/providers/aws_sagemaker", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "searxng": { + "display_name": "SearXNG (`searxng`)", + "url": "https://docs.litellm.ai/docs/search/searxng", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "search": true + } + }, + "sambanova": { + "display_name": "Sambanova (`sambanova`)", + "url": "https://docs.litellm.ai/docs/providers/sambanova", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "sap": { + "display_name": "SAP Generative AI Hub (`sap`)", + "url": "https://docs.litellm.ai/docs/providers/sap", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "scaleway": { + "display_name": "Scaleway (`scaleway`)", + "url": "https://docs.litellm.ai/docs/providers/scaleway", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "snowflake": { + "display_name": "Snowflake (`snowflake`)", + "url": "https://docs.litellm.ai/docs/providers/snowflake", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "synthetic": { + "display_name": "Synthetic (`synthetic`)", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false + } + }, + "text-completion-codestral": { + "display_name": "Text Completion Codestral (`text-completion-codestral`)", + "url": "https://docs.litellm.ai/docs/providers/codestral", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "text-completion-openai": { + "display_name": "Text Completion OpenAI (`text-completion-openai`)", + "url": "https://docs.litellm.ai/docs/providers/text_completion_openai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, + "batches": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "together_ai": { + "display_name": "Together AI (`together_ai`)", + "url": "https://docs.litellm.ai/docs/providers/togetherai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "topaz": { + "display_name": "Topaz (`topaz`)", + "url": "https://docs.litellm.ai/docs/providers/topaz", + "endpoints": { + "image_variations": true + } + }, + "tavily": { + "display_name": "Tavily (`tavily`)", + "url": "https://docs.litellm.ai/docs/search/tavily", + "endpoints": { + "search": true + } + }, + "triton": { + "display_name": "Triton (`triton`)", + "url": "https://docs.litellm.ai/docs/providers/triton-inference-server", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "v0": { + "display_name": "V0 (`v0`)", + "url": "https://docs.litellm.ai/docs/providers/v0", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vercel_ai_gateway": { + "display_name": "Vercel AI Gateway (`vercel_ai_gateway`)", + "url": "https://docs.litellm.ai/docs/providers/vercel_ai_gateway", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vllm": { + "display_name": "VLLM (`vllm`)", + "url": "https://docs.litellm.ai/docs/providers/vllm", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": true, + "files": true, + "rerank": true, + "a2a": true, + "interactions": true + } + }, + "volcengine": { + "display_name": "Volcengine (`volcengine`)", + "url": "https://docs.litellm.ai/docs/providers/volcano", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "voyage": { + "display_name": "Voyage AI (`voyage`)", + "url": "https://docs.litellm.ai/docs/providers/voyage", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": true + } + }, + "wandb": { + "display_name": "WandB Inference (`wandb`)", + "url": "https://docs.litellm.ai/docs/providers/wandb_inference", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "watsonx_text": { + "display_name": "Watsonx Text (`watsonx_text`)", + "url": "https://docs.litellm.ai/docs/providers/watsonx", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "xai": { + "display_name": "xAI (`xai`)", + "url": "https://docs.litellm.ai/docs/providers/xai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true, + "realtime": true + } + }, + "xinference": { + "display_name": "Xinference (`xinference`)", + "url": "https://docs.litellm.ai/docs/providers/xinference", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": true, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "zai": { + "display_name": "Z.AI (Zhipu AI) (`zai`)", + "url": "https://docs.litellm.ai/docs/providers/zai", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "ragflow": { + "display_name": "RAGFlow (`ragflow`)", + "url": "https://docs.litellm.ai/docs/providers/ragflow", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "vector_stores_create": true, + "a2a": true, + "interactions": true + } + }, + "cursor": { + "display_name": "Cursor BYOK (`cursor`)", + "url": "https://docs.litellm.ai/docs/providers/cursor", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "langgraph": { + "display_name": "LangGraph (`langgraph`)", + "url": "https://docs.litellm.ai/docs/providers/langgraph", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "vertex_ai/agent_engine": { + "display_name": "Vertex AI Agent Engine (`vertex_ai/agent_engine`)", + "url": "https://docs.litellm.ai/docs/providers/vertex_ai_agent_engine", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "pydantic_ai_agents": { + "display_name": "Pydantic AI Agents (`pydantic_ai_agents`)", + "url": "https://docs.litellm.ai/docs/providers/pydantic_ai_agent", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true + } + }, + "stability": { + "display_name": "Stability AI (`stability`)", + "url": "https://docs.litellm.ai/docs/providers/stability", + "endpoints": { + "chat_completions": false, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": true, + "image_edits": true, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false + } + }, + "venice": { + "display_name": "Venice.ai (`venice`)", + "url": "https://docs.litellm.ai/docs/providers/venice", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "gigachat": { + "display_name": "GigaChat (`gigachat`)", + "url": "https://docs.litellm.ai/docs/providers/gigachat", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": true + } + }, + "google_pse": { + "display_name": "Google PSE (`google_pse`)", + "url": "https://docs.litellm.ai/docs/search/google_pse", + "endpoints": { + "search": true + } + }, + "milvus": { + "display_name": "Milvus (`milvus`)", + "url": "https://docs.litellm.ai/docs/providers/milvus_vector_stores", + "endpoints": { + "vector_stores_search": true + } + }, + "minimax": { + "display_name": "Minimax (`minimax`)", + "url": "https://docs.litellm.ai/docs/providers/minimax", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "pg_vector": { + "display_name": "PG Vector (`pg_vector`)", + "url": "https://docs.litellm.ai/docs/providers/pg_vector", + "endpoints": { + "vector_stores_search": true + } + }, + "helicone": { + "display_name": "Helicone (`helicone`)", + "url": "https://docs.litellm.ai/docs/providers/helicone", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "llamagate": { + "display_name": "LlamaGate (`llamagate`)", + "url": "https://docs.litellm.ai/docs/providers/llamagate", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "xiaomi_mimo": { + "display_name": "Xiaomi Mimo (`xiaomi_mimo`)", + "url": "https://docs.litellm.ai/docs/providers/xiaomi_mimo", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + }, + "manus": { + "display_name": "Manus (`manus`)", + "url": "https://docs.litellm.ai/docs/providers/manus", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "files": true, + "rerank": false, + "a2a": true, + "interactions": true + } + }, + "sarvam": { + "display_name": "Sarvam (`sarvam`)", + "url": "https://docs.litellm.ai/docs/providers/sarvam", + "endpoints": { + "chat_completions": true, + "messages": true, + "responses": true + } + } + }, + "endpoints": { + "a2a": { + "docs_label": "a2a", + "display_name": "A2A (Agent-to-Agent) protocol for agent communication", + "leftnav_label": "/a2a", + "provider_json_field": "a2a", + "url": "https://docs.litellm.ai/docs/a2a", + "bridges_to_chat_completion": true + }, + "messages": { + "docs_label": "anthropic_unified", + "display_name": "Anthropic Messages API", + "leftnav_label": "/messages", + "provider_json_field": "messages", + "url": "https://docs.litellm.ai/docs/anthropic_unified", + "bridges_to_chat_completion": true + }, + "anthropic_count_tokens": { + "docs_label": "anthropic_count_tokens", + "display_name": "Anthropic Count Tokens API", + "leftnav_label": "/count_tokens", + "provider_json_field": "count_tokens", + "url": "https://docs.litellm.ai/docs/anthropic_count_tokens" + }, + "apply_guardrail": { + "docs_label": "apply_guardrail", + "display_name": "Unified Apply Guardrail API", + "leftnav_label": "/guardrails/apply_guardrail", + "provider_json_field": "apply_guardrail", + "url": "https://docs.litellm.ai/docs/apply_guardrail" + }, + "assistants": { + "docs_label": "assistants", + "display_name": "OpenAI Assistants API", + "leftnav_label": "/assistants", + "provider_json_field": "assistants", + "url": "https://docs.litellm.ai/docs/assistants" + }, + "audio_transcription": { + "docs_label": "audio_transcription", + "display_name": "OpenAI Audio Transcription API", + "leftnav_label": "/audio/transcriptions", + "provider_json_field": "audio_transcriptions", + "url": "https://docs.litellm.ai/docs/audio_transcription" + }, + "batches": { + "docs_label": "batches", + "display_name": "OpenAI Batches API", + "leftnav_label": "/batches", + "provider_json_field": "batches", + "url": "https://docs.litellm.ai/docs/batches" + }, + "bedrock_invoke": { + "docs_label": "bedrock_invoke", + "display_name": "Bedrock Invoke API", + "leftnav_label": "/invoke", + "provider_json_field": "bedrock_invoke", + "url": "https://docs.litellm.ai/docs/bedrock_invoke" + }, + "bedrock_converse": { + "docs_label": "bedrock_converse", + "display_name": "Bedrock Converse API", + "leftnav_label": "/converse", + "provider_json_field": "bedrock_converse", + "url": "https://docs.litellm.ai/docs/bedrock_converse" + }, + "chat_completions": { + "docs_label": "chat_completions", + "display_name": "OpenAI Chat Completions API", + "leftnav_label": "/chat/completions", + "provider_json_field": "chat_completions", + "url": "https://docs.litellm.ai/docs/chat_completions" + }, + "container_files": { + "docs_label": "container_files", + "display_name": "OpenAI Container Files API", + "leftnav_label": "/create/container/files", + "provider_json_field": "container_files", + "url": "https://docs.litellm.ai/docs/container_files" + }, + "container": { + "docs_label": "containers", + "display_name": "OpenAI Containers API", + "leftnav_label": "/container", + "provider_json_field": "container", + "url": "https://docs.litellm.ai/docs/containers" + }, + "embeddings": { + "docs_label": "embedding/supported_embedding", + "display_name": "OpenAI Embeddings API", + "leftnav_label": "/embeddings", + "provider_json_field": "embeddings", + "url": "https://docs.litellm.ai/docs/embedding/supported_embedding" + }, + "files": { + "docs_label": "files", + "display_name": "OpenAI Files API", + "leftnav_label": "/files", + "provider_json_field": "files", + "url": "https://docs.litellm.ai/docs/proxy/litellm_managed_files" + }, + "fine_tuning": { + "docs_label": "fine_tuning", + "display_name": "OpenAI Fine-Tuning API", + "leftnav_label": "/fine_tuning", + "provider_json_field": "fine_tuning", + "url": "https://docs.litellm.ai/docs/proxy/managed_finetuning" + }, + "generateContent": { + "docs_label": "generateContent", + "display_name": "Google GenerateContent API", + "leftnav_label": "/generateContent", + "provider_json_field": "generateContent", + "url": "https://docs.litellm.ai/docs/generateContent", + "bridges_to_chat_completion": true + }, + "image_edits": { + "docs_label": "image_edits", + "display_name": "OpenAI Images Edits API", + "leftnav_label": "/images/edits", + "provider_json_field": "image_edits", + "url": "https://docs.litellm.ai/docs/image_edits" + }, + "image_generations": { + "docs_label": "image_generation", + "display_name": "OpenAI Images Generations API", + "leftnav_label": "/images/generations", + "provider_json_field": "image_generations", + "url": "https://docs.litellm.ai/docs/image_generation" + }, + "image_variations": { + "docs_label": "image_variations", + "display_name": "OpenAI Images Variations API", + "leftnav_label": "/images/variations", + "provider_json_field": "image_variations", + "url": "https://docs.litellm.ai/docs/image_variations" + }, + "interactions": { + "docs_label": "interactions", + "display_name": "Google Interactions API", + "leftnav_label": "/interactions", + "provider_json_field": "interactions", + "url": "https://docs.litellm.ai/docs/interactions", + "bridges_to_chat_completion": true + }, + "mcp": { + "docs_label": "mcp", + "display_name": "Model Context Protocol (MCP)", + "leftnav_label": "/mcp", + "provider_json_field": "mcp", + "url": "https://docs.litellm.ai/docs/mcp" + }, + "moderation": { + "docs_label": "moderation", + "display_name": "OpenAI Moderations API", + "leftnav_label": "/moderations", + "provider_json_field": "moderations", + "url": "https://docs.litellm.ai/docs/moderation" + }, + "ocr": { + "docs_label": "ocr", + "display_name": "Mistral OCR API", + "leftnav_label": "/ocr", + "provider_json_field": "ocr", + "url": "https://docs.litellm.ai/docs/ocr" + }, + "rag_ingest": { + "docs_label": "rag_ingest", + "display_name": "RAG Ingest API", + "leftnav_label": "/rag/ingest", + "provider_json_field": "rag_ingest", + "url": "https://docs.litellm.ai/docs/rag_ingest" + }, + "rag_query": { + "docs_label": "rag_query", + "display_name": "RAG Query API", + "leftnav_label": "/rag/query", + "provider_json_field": "rag_query", + "url": "https://docs.litellm.ai/docs/rag_query" + }, + "realtime": { + "docs_label": "realtime", + "display_name": "OpenAI Realtime API", + "leftnav_label": "/realtime", + "provider_json_field": "realtime", + "url": "https://docs.litellm.ai/docs/realtime" + }, + "rerank": { + "docs_label": "rerank", + "display_name": "Cohere Rerank API", + "leftnav_label": "/rerank", + "provider_json_field": "rerank", + "url": "https://docs.litellm.ai/docs/rerank" + }, + "responses": { + "docs_label": "response_api", + "display_name": "OpenAI Responses API", + "leftnav_label": "/responses", + "provider_json_field": "responses", + "url": "https://docs.litellm.ai/docs/response_api", + "bridges_to_chat_completion": true + }, + "response_api_compact": { + "docs_label": "response_api_compact", + "display_name": "OpenAI Responses API", + "leftnav_label": "/responses", + "provider_json_field": "compact", + "url": "https://docs.litellm.ai/docs/response_api" + }, + "search": { + "docs_label": "search", + "display_name": "Search API", + "leftnav_label": "/search", + "provider_json_field": "search", + "url": "https://docs.litellm.ai/docs/search" + }, + "skills": { + "docs_label": "skills", + "display_name": "Anthropic Skills API", + "leftnav_label": "/skills", + "provider_json_field": "skills", + "url": "https://docs.litellm.ai/docs/skills" + }, + "text_completion": { + "docs_label": "text_completion", + "display_name": "OpenAI Completions API", + "leftnav_label": "/completions", + "provider_json_field": "text_completion", + "url": "https://docs.litellm.ai/docs/text_completion", + "bridges_to_chat_completion": true + }, + "text_to_speech": { + "docs_label": "text_to_speech", + "display_name": "OpenAI Text-to-Speech API", + "leftnav_label": "/audio/speech", + "provider_json_field": "audio_speech", + "url": "https://docs.litellm.ai/docs/text_to_speech" + }, + "vector_store_files": { + "docs_label": "vector_store_files", + "display_name": "OpenAI Vector Store Files API", + "leftnav_label": "/vector_stores/files", + "provider_json_field": "vector_store_files", + "url": "https://docs.litellm.ai/docs/vector_store_files" + }, + "vector_stores_create": { + "docs_label": "vector_stores_create", + "display_name": "OpenAI Vector Stores Create API", + "leftnav_label": "/vector_stores/create", + "provider_json_field": "vector_stores_create", + "url": "https://docs.litellm.ai/docs/vector_stores/create" + }, + "vector_stores_search": { + "docs_label": "vector_stores_search", + "display_name": "OpenAI Vector Stores Search API", + "leftnav_label": "/vector_stores/search", + "provider_json_field": "vector_stores_search", + "url": "https://docs.litellm.ai/docs/vector_stores/search" + }, + "videos": { + "docs_label": "videos", + "display_name": "OpenAI Videos API", + "leftnav_label": "/videos", + "provider_json_field": "video_generations", + "url": "https://docs.litellm.ai/docs/videos" + } + } +} diff --git a/litellm/proxy/_experimental/mcp_server/server.py b/litellm/proxy/_experimental/mcp_server/server.py index 48c837c1e4f..5b3d5bd60e2 100644 --- a/litellm/proxy/_experimental/mcp_server/server.py +++ b/litellm/proxy/_experimental/mcp_server/server.py @@ -5,6 +5,7 @@ LiteLLM MCP Server Routes import asyncio import contextlib + import traceback import uuid from datetime import datetime @@ -84,6 +85,7 @@ except ImportError as e: _SESSION_MANAGERS_INITIALIZED = False _INITIALIZATION_LOCK = asyncio.Lock() + if MCP_AVAILABLE: from mcp.server import Server @@ -1919,65 +1921,86 @@ if MCP_AVAILABLE: mgr: "StreamableHTTPSessionManager", ) -> bool: """ - Handle stale MCP session IDs to prevent "Session not found" errors. - - When clients reconnect after a server restart or session cleanup, they may - send a session ID that no longer exists. This function handles two scenarios: - - 1. Non-DELETE requests: Strip the stale session ID header so the session - manager creates a fresh session transparently. - - 2. DELETE requests: Return success (200) immediately for idempotent behavior, - since the desired state (session doesn't exist) is already achieved. + Inspect the incoming ``mcp-session-id`` header **before** the + request reaches the MCP SDK. If the session is stale (not known + to this worker), strip the header so the SDK creates a fresh + stateless session instead of returning a 400. Returns: - True if the request was handled (DELETE on non-existent session) - False if the request should continue to the session manager + True if the request was fully handled (e.g. DELETE on + non-existent session). False if the request should continue + to the session manager. - Fixes https://github.com/BerriAI/litellm/issues/20292 + Fixes https://github.com/BerriAI/litellm/issues/20992 """ _mcp_session_header = b"mcp-session-id" + _headers = scope.get("headers", []) + + def _normalize_header_name(header_name: Any) -> Optional[bytes]: + if isinstance(header_name, bytes): + return header_name.lower() + if isinstance(header_name, str): + return header_name.lower().encode("utf-8", errors="replace") + return None + _session_id: Optional[str] = None - for header_name, header_value in scope.get("headers", []): - if header_name == _mcp_session_header: - _session_id = header_value.decode("utf-8", errors="replace") + for header_name, header_value in _headers: + if _normalize_header_name(header_name) == _mcp_session_header: + if isinstance(header_value, bytes): + _session_id = header_value.decode("utf-8", errors="replace") + else: + _session_id = str(header_value) break if _session_id is None: return False + # Check in-memory session tracking known_sessions = getattr(mgr, "_server_instances", None) - if known_sessions is None or _session_id in known_sessions: - # Session exists or we can't check - let the session manager handle it + # If we cannot inspect known_sessions, let the manager handle it + if known_sessions is None: return False - # Session doesn't exist - handle based on request method + # If session exists in this worker's memory, let the manager handle it + try: + if _session_id in known_sessions: + return False + except Exception: + verbose_logger.debug( + "Unable to inspect active MCP sessions for '%s'. " + "Deferring to session manager.", + _session_id, + ) + return False + + # --- Session not in this worker's memory --- method = scope.get("method", "").upper() - + if method == "DELETE": - # Idempotent DELETE: session doesn't exist, return success verbose_logger.info( - f"DELETE request for non-existent MCP session '{_session_id}'. " - "Returning success (idempotent DELETE)." + "DELETE request for non-existent MCP session '%s'. " + "Returning success (idempotent DELETE).", + _session_id, ) success_response = JSONResponse( status_code=200, - content={"message": "Session terminated successfully"} + content={"message": "Session terminated successfully"}, ) await success_response(scope, receive, send) return True - else: - # Non-DELETE: strip stale session ID to allow new session creation - verbose_logger.warning( - "MCP session ID '%s' not found in active sessions. " - "Stripping stale header to force new session creation.", - _session_id, - ) - scope["headers"] = [ - (k, v) for k, v in scope["headers"] - if k != _mcp_session_header - ] - return False + + # Non-DELETE: strip stale session ID to allow new session creation + verbose_logger.warning( + "MCP session ID '%s' not found in this worker's memory. " + "Stripping stale header to force new session creation.", + _session_id, + ) + scope["headers"] = [ + (k, v) + for k, v in _headers + if _normalize_header_name(k) != _mcp_session_header + ] + return False async def handle_streamable_http_mcp( scope: Scope, receive: Receive, send: Send @@ -2055,7 +2078,9 @@ if MCP_AVAILABLE: # Handle stale session IDs - either strip them for reconnection # or return success for idempotent DELETE operations - handled = await _handle_stale_mcp_session(scope, receive, send, session_manager) + handled = await _handle_stale_mcp_session( + scope, receive, send, session_manager + ) if handled: # Request was fully handled (e.g., DELETE on non-existent session) return diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 53513f7f522..28311ab1b3b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2280,6 +2280,9 @@ class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): organization_rpm_limit: Optional[int] = None organization_metadata: Optional[dict] = None + # Project Params + project_metadata: Optional[dict] = None + # Time stamps last_refreshed_at: Optional[float] = None # last time joint view was pulled from db @@ -2581,6 +2584,7 @@ class NewProjectRequest(LiteLLM_BudgetTable): team_id: str budget_id: Optional[str] = None metadata: Optional[dict] = None + tags: Optional[List[str]] = None models: List[str] = [] model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2590,6 +2594,11 @@ class NewProjectRequest(LiteLLM_BudgetTable): @model_validator(mode="before") @classmethod def set_model_info(cls, values): + if "tags" in values and values["tags"] is not None: + if not isinstance(values["tags"], list): + raise ValueError( + f"tags must be a list of strings, got {type(values['tags']).__name__}" + ) for field in LiteLLM_ManagementEndpoint_MetadataFields: if values.get(field) is not None: if values.get("metadata") is None: @@ -2607,6 +2616,7 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): description: Optional[str] = None team_id: Optional[str] = None metadata: Optional[dict] = None + tags: Optional[List[str]] = None models: Optional[List[str]] = None model_rpm_limit: Optional[dict] = None model_tpm_limit: Optional[dict] = None @@ -2617,6 +2627,11 @@ class UpdateProjectRequest(LiteLLM_BudgetTable): @model_validator(mode="before") @classmethod def set_model_info(cls, values): + if "tags" in values and values["tags"] is not None: + if not isinstance(values["tags"], list): + raise ValueError( + f"tags must be a list of strings, got {type(values['tags']).__name__}" + ) for field in LiteLLM_ManagementEndpoint_MetadataFields: if values.get(field) is not None: if values.get("metadata") is None: diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 3e2378ada60..8ad3b83c043 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -212,10 +212,12 @@ async def user_api_key_auth_websocket(websocket: WebSocket): api_key = websocket.headers.get("api-key") if not api_key: # Try extracting from WebSocket subprotocol (browser clients) - for protocol in websocket.headers.get("sec-websocket-protocol", "").split(","): + for protocol in websocket.headers.get("sec-websocket-protocol", "").split( + "," + ): protocol = protocol.strip() if protocol.startswith("openai-insecure-api-key."): - api_key = protocol[len("openai-insecure-api-key."):] + api_key = protocol[len("openai-insecure-api-key.") :] break if not api_key: await websocket.close(code=status.WS_1008_POLICY_VIOLATION) @@ -704,6 +706,8 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _jwt_project_obj is not None: + valid_token.project_metadata = _jwt_project_obj.metadata # run through common checks _ = await common_checks( @@ -1294,6 +1298,8 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _project_obj is not None: + valid_token.project_metadata = _project_obj.metadata global_proxy_spend = None if ( @@ -1743,6 +1749,8 @@ async def _run_post_custom_auth_checks( user_api_key_cache=user_api_key_cache, proxy_logging_obj=proxy_logging_obj, ) + if _project_obj is not None: + valid_token.project_metadata = _project_obj.metadata _ = await common_checks( request=request, diff --git a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py index 02fa84bae30..8c59c79ff0a 100644 --- a/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py +++ b/litellm/proxy/db/db_transaction_queue/spend_log_cleanup.py @@ -76,27 +76,29 @@ class SpendLogCleanup: "Max logs deleted - 1,00,000, rest of the logs will be deleted in next run" ) break - # Step 1: Find logs to delete - logs_to_delete = await prisma_client.db.litellm_spendlogs.find_many( - where={"startTime": {"lt": cutoff_date}}, - take=self.batch_size, + # Step 1: Find logs and delete them in one go without fetching to application + # Delete in batches, limited by self.batch_size + deleted_count = await prisma_client.db.execute_raw( + """ + DELETE FROM "LiteLLM_SpendLogs" + WHERE "request_id" IN ( + SELECT "request_id" FROM "LiteLLM_SpendLogs" + WHERE "startTime" < $1::timestamptz + LIMIT $2 + ) + """, + cutoff_date, + self.batch_size, ) - verbose_proxy_logger.info(f"Found {len(logs_to_delete)} logs in this batch") + verbose_proxy_logger.info(f"Deleted {deleted_count} logs in this batch") - if not logs_to_delete: + if deleted_count == 0: verbose_proxy_logger.info( f"No more logs to delete. Total deleted: {total_deleted}" ) break - request_ids = [log.request_id for log in logs_to_delete] - - # Step 2: Delete them in one go - await prisma_client.db.litellm_spendlogs.delete_many( - where={"request_id": {"in": request_ids}} - ) - - total_deleted += len(logs_to_delete) + total_deleted += deleted_count run_count += 1 # Add a small sleep to prevent overwhelming the database diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 7ebf9a4caee..3168bcd812f 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -248,13 +248,15 @@ def clean_headers( clean_headers = {} litellm_key_lower = ( litellm_key_header_name.lower() if litellm_key_header_name is not None else None - ) + ) for header, value in headers.items(): header_lower = header.lower() - + if header_lower == "authorization" and is_anthropic_oauth_key(value): clean_headers[header] = value - elif forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE: + elif ( + forward_llm_provider_auth_headers and header_lower in _SPECIAL_HEADERS_CACHE + ): if litellm_key_lower and header_lower == litellm_key_lower: continue if header_lower == "authorization": @@ -840,11 +842,13 @@ async def add_litellm_data_to_request( # noqa: PLR0915 from litellm.types.proxy.litellm_pre_call_utils import SecretFields _raw_headers: Dict[str, str] = _safe_get_request_headers(request) - + forward_llm_auth = False if general_settings: - forward_llm_auth = general_settings.get("forward_llm_provider_auth_headers", False) - + forward_llm_auth = general_settings.get( + "forward_llm_provider_auth_headers", False + ) + _headers: Dict[str, str] = clean_headers( request.headers, litellm_key_header_name=( @@ -1019,6 +1023,14 @@ async def add_litellm_data_to_request( # noqa: PLR0915 "spend_logs_metadata" ] + ## PROJECT-LEVEL TAGS + project_metadata = user_api_key_dict.project_metadata or {} + if "tags" in project_metadata and project_metadata["tags"] is not None: + data[_metadata_variable_name]["tags"] = LiteLLMProxyRequestSetup._merge_tags( + request_tags=data[_metadata_variable_name].get("tags"), + tags_to_add=project_metadata["tags"], + ) + ## TEAM-LEVEL METADATA data = ( LiteLLMProxyRequestSetup.add_management_endpoint_metadata_to_request_metadata( diff --git a/litellm/proxy/management_endpoints/access_group_endpoints.py b/litellm/proxy/management_endpoints/access_group_endpoints.py index 12aa748bbc3..d58dca5aec0 100644 --- a/litellm/proxy/management_endpoints/access_group_endpoints.py +++ b/litellm/proxy/management_endpoints/access_group_endpoints.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Set from fastapi import APIRouter, Depends, HTTPException, status @@ -94,6 +94,183 @@ async def _invalidate_cache_access_group(access_group_id: str) -> None: ) +# --------------------------------------------------------------------------- +# DB sync helpers (called inside a Prisma transaction) +# --------------------------------------------------------------------------- + + +async def _sync_add_access_group_to_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Add access_group_id to each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id not in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": list(team.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_teams( + tx, team_ids: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each team's access_group_ids (idempotent).""" + for team_id in team_ids: + team = await tx.litellm_teamtable.find_unique(where={"team_id": team_id}) + if team is not None and access_group_id in (team.access_group_ids or []): + await tx.litellm_teamtable.update( + where={"team_id": team_id}, + data={"access_group_ids": [ag for ag in team.access_group_ids if ag != access_group_id]}, + ) + + +async def _sync_add_access_group_to_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Add access_group_id to each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id not in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": list(key.access_group_ids or []) + [access_group_id]}, + ) + + +async def _sync_remove_access_group_from_keys( + tx, key_tokens: List[str], access_group_id: str +) -> None: + """Remove access_group_id from each key's access_group_ids (idempotent).""" + for token in key_tokens: + key = await tx.litellm_verificationtoken.find_unique(where={"token": token}) + if key is not None and access_group_id in (key.access_group_ids or []): + await tx.litellm_verificationtoken.update( + where={"token": token}, + data={"access_group_ids": [ag for ag in key.access_group_ids if ag != access_group_id]}, + ) + + +# --------------------------------------------------------------------------- +# Cache patch helpers +# --------------------------------------------------------------------------- + + +async def _patch_team_caches_add_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to include access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is None: + continue + if cached_team.access_group_ids is None: + cached_team.access_group_ids = [access_group_id] + elif access_group_id not in cached_team.access_group_ids: + cached_team.access_group_ids = list(cached_team.access_group_ids) + [access_group_id] + else: + continue + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_team_caches_remove_access_group( + team_ids: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached team objects to remove access_group_id.""" + for team_id in team_ids: + cached_team = await _get_team_object_from_cache( + key="team_id:{}".format(team_id), + proxy_logging_obj=proxy_logging_obj, + user_api_key_cache=user_api_key_cache, + parent_otel_span=None, + ) + if cached_team is not None and cached_team.access_group_ids: + cached_team.access_group_ids = [ + ag for ag in cached_team.access_group_ids if ag != access_group_id + ] + await _cache_team_object( + team_id=team_id, + team_table=cached_team, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_add_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to include access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if not isinstance(cached_key, UserAPIKeyAuth): + continue + if cached_key.access_group_ids is None: + cached_key.access_group_ids = [access_group_id] + elif access_group_id not in cached_key.access_group_ids: + cached_key.access_group_ids = list(cached_key.access_group_ids) + [access_group_id] + else: + continue + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +async def _patch_key_caches_remove_access_group( + key_tokens: List[str], + access_group_id: str, + user_api_key_cache, + proxy_logging_obj, +) -> None: + """Patch cached key objects to remove access_group_id.""" + for token in key_tokens: + cached_key = await user_api_key_cache.async_get_cache(key=token) + if cached_key is None: + continue + if isinstance(cached_key, dict): + cached_key = UserAPIKeyAuth(**cached_key) + if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: + cached_key.access_group_ids = [ + ag for ag in cached_key.access_group_ids if ag != access_group_id + ] + await _cache_key_object( + hashed_token=token, + user_api_key_obj=cached_key, + user_api_key_cache=user_api_key_cache, + proxy_logging_obj=proxy_logging_obj, + ) + + +# --------------------------------------------------------------------------- +# CRUD endpoints +# --------------------------------------------------------------------------- + + @router.post( "/v1/access_group", response_model=AccessGroupResponse, @@ -106,32 +283,42 @@ async def create_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_name": data.access_group_name} - ) - if existing is not None: - raise HTTPException( - status_code=status.HTTP_409_CONFLICT, - detail=f"Access group '{data.access_group_name}' already exists", - ) - try: - record = await prisma_client.db.litellm_accessgrouptable.create( - data={ - "access_group_name": data.access_group_name, - "description": data.description, - "access_model_names": data.access_model_names or [], - "access_mcp_server_ids": data.access_mcp_server_ids or [], - "access_agent_ids": data.access_agent_ids or [], - "assigned_team_ids": data.assigned_team_ids or [], - "assigned_key_ids": data.assigned_key_ids or [], - "created_by": user_api_key_dict.user_id, - "updated_by": user_api_key_dict.user_id, - } - ) + async with prisma_client.db.tx() as tx: + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_name": data.access_group_name} + ) + if existing is not None: + raise HTTPException( + status_code=status.HTTP_409_CONFLICT, + detail=f"Access group '{data.access_group_name}' already exists", + ) + + record = await tx.litellm_accessgrouptable.create( + data={ + "access_group_name": data.access_group_name, + "description": data.description, + "access_model_names": data.access_model_names or [], + "access_mcp_server_ids": data.access_mcp_server_ids or [], + "access_agent_ids": data.access_agent_ids or [], + "assigned_team_ids": data.assigned_team_ids or [], + "assigned_key_ids": data.assigned_key_ids or [], + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + + # Sync team and key tables to reference the new access group + await _sync_add_access_group_to_teams( + tx, data.assigned_team_ids or [], record.access_group_id + ) + await _sync_add_access_group_to_keys( + tx, data.assigned_key_ids or [], record.access_group_id + ) + except HTTPException: + raise except Exception as e: # Race condition: another request created the same name between find_unique and create. - # Prisma raises UniqueViolationError (P2002) or similar for unique constraint. if "unique constraint" in str(e).lower() or "P2002" in str(e): raise HTTPException( status_code=status.HTTP_409_CONFLICT, @@ -139,8 +326,15 @@ async def create_access_group( ) raise - # Cache the newly created access group for read-heavy access patterns + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group( + data.assigned_team_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_add_access_group( + data.assigned_key_ids or [], record.access_group_id, user_api_key_cache, proxy_logging_obj + ) return _record_to_response(record) @@ -195,24 +389,54 @@ async def update_access_group( _require_proxy_admin(user_api_key_dict) prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) - existing = await prisma_client.db.litellm_accessgrouptable.find_unique( - where={"access_group_id": access_group_id} - ) - if existing is None: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, - detail=f"Access group '{access_group_id}' not found", - ) - + update_fields = data.model_dump(exclude_unset=True) update_data: dict = {"updated_by": user_api_key_dict.user_id} - for field, value in data.model_dump(exclude_unset=True).items(): + for field, value in update_fields.items(): + if field in ("assigned_team_ids", "assigned_key_ids", "access_model_names", "access_mcp_server_ids", "access_agent_ids") and value is None: + value = [] update_data[field] = value + # Initialize delta lists before the try block so they remain accessible + # for cache updates after the transaction, even if an error path is added later. + teams_to_add: List[str] = [] + teams_to_remove: List[str] = [] + keys_to_add: List[str] = [] + keys_to_remove: List[str] = [] + try: - record = await prisma_client.db.litellm_accessgrouptable.update( - where={"access_group_id": access_group_id}, - data=update_data, - ) + async with prisma_client.db.tx() as tx: + # Read inside the transaction so delta computation is consistent with the write, + # avoiding a TOCTOU race where a concurrent update could make deltas stale. + existing = await tx.litellm_accessgrouptable.find_unique( + where={"access_group_id": access_group_id} + ) + if existing is None: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, + detail=f"Access group '{access_group_id}' not found", + ) + + old_team_ids: Set[str] = set(existing.assigned_team_ids or []) + old_key_ids: Set[str] = set(existing.assigned_key_ids or []) + new_team_ids: Set[str] = set(update_fields["assigned_team_ids"] or []) if "assigned_team_ids" in update_fields else old_team_ids + new_key_ids: Set[str] = set(update_fields["assigned_key_ids"] or []) if "assigned_key_ids" in update_fields else old_key_ids + + teams_to_add = list(new_team_ids - old_team_ids) + teams_to_remove = list(old_team_ids - new_team_ids) + keys_to_add = list(new_key_ids - old_key_ids) + keys_to_remove = list(old_key_ids - new_key_ids) + + record = await tx.litellm_accessgrouptable.update( + where={"access_group_id": access_group_id}, + data=update_data, + ) + + await _sync_add_access_group_to_teams(tx, teams_to_add, access_group_id) + await _sync_remove_access_group_from_teams(tx, teams_to_remove, access_group_id) + await _sync_add_access_group_to_keys(tx, keys_to_add, access_group_id) + await _sync_remove_access_group_from_keys(tx, keys_to_remove, access_group_id) + except HTTPException: + raise except Exception as e: # Unique constraint violation (e.g. access_group_name already exists). if "unique constraint" in str(e).lower() or "P2002" in str(e): @@ -222,8 +446,13 @@ async def update_access_group( ) raise - # Write the updated record into cache (same key, overwrites stale entry) + from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache + await _cache_access_group_record(record) + await _patch_team_caches_add_access_group(teams_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_team_caches_remove_access_group(teams_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_add_access_group(keys_to_add, access_group_id, user_api_key_cache, proxy_logging_obj) + await _patch_key_caches_remove_access_group(keys_to_remove, access_group_id, user_api_key_cache, proxy_logging_obj) return _record_to_response(record) @@ -240,9 +469,8 @@ async def delete_access_group( prisma_client = get_prisma_client_or_throw(CommonProxyErrors.db_not_connected_error.value) try: - # Track affected team IDs and key tokens for cache invalidation - affected_team_ids: list = [] - affected_key_tokens: list = [] + affected_team_ids: List[str] = [] + affected_key_tokens: List[str] = [] async with prisma_client.db.tx() as tx: existing = await tx.litellm_accessgrouptable.find_unique( @@ -254,73 +482,61 @@ async def delete_access_group( detail=f"Access group '{access_group_id}' not found", ) - # Remove access_group_id from teams and keys that reference it + # Union of: teams that have this access_group_id in their own access_group_ids + # AND teams listed in assigned_team_ids (handles out-of-sync data from before this sync was added) teams_with_group = await tx.litellm_teamtable.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) - for team in teams_with_group: - affected_team_ids.append(team.team_id) - updated_ids = [tid for tid in (team.access_group_ids or []) if tid != access_group_id] - await tx.litellm_teamtable.update( - where={"team_id": team.team_id}, - data={"access_group_ids": updated_ids}, - ) + all_affected_team_ids: Set[str] = ( + {team.team_id for team in teams_with_group} + | set(existing.assigned_team_ids or []) + ) + affected_team_ids = list(all_affected_team_ids) + # Union of: keys that have this access_group_id in their own access_group_ids + # AND keys listed in assigned_key_ids (handles out-of-sync data) keys_with_group = await tx.litellm_verificationtoken.find_many( where={"access_group_ids": {"hasSome": [access_group_id]}} ) + all_affected_key_tokens: Set[str] = ( + {key.token for key in keys_with_group} + | set(existing.assigned_key_ids or []) + ) + affected_key_tokens = list(all_affected_key_tokens) + + # Update teams returned by find_many directly — we already have their data. + for team in teams_with_group: + await tx.litellm_teamtable.update( + where={"team_id": team.team_id}, + data={"access_group_ids": [ag for ag in (team.access_group_ids or []) if ag != access_group_id]}, + ) + # Use _sync_remove only for out-of-sync teams not found by the hasSome query. + out_of_sync_team_ids = set(existing.assigned_team_ids or []) - {t.team_id for t in teams_with_group} + await _sync_remove_access_group_from_teams(tx, list(out_of_sync_team_ids), access_group_id) + + # Update keys returned by find_many directly — we already have their data. for key in keys_with_group: - affected_key_tokens.append(key.token) - updated_ids = [kid for kid in (key.access_group_ids or []) if kid != access_group_id] await tx.litellm_verificationtoken.update( where={"token": key.token}, - data={"access_group_ids": updated_ids}, + data={"access_group_ids": [ag for ag in (key.access_group_ids or []) if ag != access_group_id]}, ) + # Use _sync_remove only for out-of-sync keys not found by the hasSome query. + out_of_sync_key_tokens = set(existing.assigned_key_ids or []) - {k.token for k in keys_with_group} + await _sync_remove_access_group_from_keys(tx, list(out_of_sync_key_tokens), access_group_id) await tx.litellm_accessgrouptable.delete( where={"access_group_id": access_group_id} ) - # Invalidate the deleted access group from cache - await _invalidate_cache_access_group(access_group_id) - - # Patch cached team and key objects to remove the deleted access_group_id - # instead of fully invalidating them (keeps cache warm, avoids DB re-fetch) from litellm.proxy.proxy_server import proxy_logging_obj, user_api_key_cache - for team_id in affected_team_ids: - cached_team = await _get_team_object_from_cache( - key="team_id:{}".format(team_id), - proxy_logging_obj=proxy_logging_obj, - user_api_key_cache=user_api_key_cache, - parent_otel_span=None, - ) - if cached_team is not None and cached_team.access_group_ids: - cached_team.access_group_ids = [ - ag_id for ag_id in cached_team.access_group_ids if ag_id != access_group_id - ] - await _cache_team_object( - team_id=team_id, - team_table=cached_team, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) - - for token in affected_key_tokens: - cached_key = await user_api_key_cache.async_get_cache(key=token) - if cached_key is not None: - if isinstance(cached_key, dict): - cached_key = UserAPIKeyAuth(**cached_key) - if isinstance(cached_key, UserAPIKeyAuth) and cached_key.access_group_ids: - cached_key.access_group_ids = [ - ag_id for ag_id in cached_key.access_group_ids if ag_id != access_group_id - ] - await _cache_key_object( - hashed_token=token, - user_api_key_obj=cached_key, - user_api_key_cache=user_api_key_cache, - proxy_logging_obj=proxy_logging_obj, - ) + await _invalidate_cache_access_group(access_group_id) + await _patch_team_caches_remove_access_group( + affected_team_ids, access_group_id, user_api_key_cache, proxy_logging_obj + ) + await _patch_key_caches_remove_access_group( + affected_key_tokens, access_group_id, user_api_key_cache, proxy_logging_obj + ) except HTTPException: raise diff --git a/litellm/proxy/management_endpoints/project_endpoints.py b/litellm/proxy/management_endpoints/project_endpoints.py index ba3238ebfd5..8f48f9def78 100644 --- a/litellm/proxy/management_endpoints/project_endpoints.py +++ b/litellm/proxy/management_endpoints/project_endpoints.py @@ -284,6 +284,7 @@ async def new_project( - model_tpm_limit: *Optional[dict]* - TPM limits per model. Example: {"gpt-4": 50000, "gpt-3.5-turbo": 100000} - budget_duration: *Optional[str]* - Frequency of reseting project budget - metadata: *Optional[dict]* - Metadata for project, store information for project. Example metadata - {"use_case_id": "SNOW-12345", "responsible_ai_id": "RAI-67890"} + - tags: *Optional[list]* - Tags for the project. Example: ["production", "api"] - blocked: *bool* - Flag indicating if the project is blocked or not - will stop all calls from keys with this project_id. - object_permission: Optional[LiteLLM_ObjectPermissionBase] - project-specific object permission. Example - {"vector_stores": ["vector_store_1", "vector_store_2"]}. IF null or {} then no object permission. @@ -339,6 +340,15 @@ async def new_project( ) try: + if getattr(data, "tags", None) is not None and not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Only premium users can add tags to projects. " + + CommonProxyErrors.not_premium_user.value + }, + ) + if not premium_user: raise HTTPException( status_code=403, @@ -348,6 +358,16 @@ async def new_project( }, ) + # ADD METADATA FIELDS + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) + delattr(data, field) + if prisma_client is None: raise HTTPException( status_code=500, @@ -463,7 +483,7 @@ async def new_project( response_model=LiteLLM_ProjectTable, ) @management_endpoint_wrapper -async def update_project( +async def update_project( # noqa: PLR0915 data: UpdateProjectRequest, http_request: Request, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -485,6 +505,7 @@ async def update_project( - model_rpm_limit: *Optional[dict]* - Updated RPM limits per model - model_tpm_limit: *Optional[dict]* - Updated TPM limits per model - budget_duration: *Optional[str]* - Updated budget duration + - tags: *Optional[list]* - Updated list of tags for the project - object_permission: Optional[LiteLLM_ObjectPermissionBase] - Updated object permission Example: @@ -514,6 +535,15 @@ async def update_project( ) try: + if getattr(data, "tags", None) is not None and not premium_user: + raise HTTPException( + status_code=403, + detail={ + "error": "Only premium users can add tags to projects. " + + CommonProxyErrors.not_premium_user.value + }, + ) + if not premium_user: raise HTTPException( status_code=403, @@ -523,6 +553,16 @@ async def update_project( }, ) + # ADD METADATA FIELDS + for field in LiteLLM_ManagementEndpoint_MetadataFields_Premium: + if getattr(data, field, None) is not None: + _set_object_metadata_field( + object_data=data, + field_name=field, + value=getattr(data, field), + ) + delattr(data, field) + if prisma_client is None: raise HTTPException( status_code=500, diff --git a/litellm/proxy/ocr_endpoints/endpoints.py b/litellm/proxy/ocr_endpoints/endpoints.py index c1092a06b48..4f31c762df1 100644 --- a/litellm/proxy/ocr_endpoints/endpoints.py +++ b/litellm/proxy/ocr_endpoints/endpoints.py @@ -1,9 +1,14 @@ #### OCR Endpoints ##### +import json +from typing import Any, Dict, Optional, cast + import orjson -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, Request, Response, UploadFile from fastapi.responses import ORJSONResponse +from litellm._logging import verbose_proxy_logger +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -11,6 +16,171 @@ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessin router = APIRouter() +def _build_document_from_upload( + file_content: bytes, + filename: Optional[str], + content_type: Optional[str], +) -> Dict[str, str]: + """ + Convert uploaded file bytes into a Mistral-format document dict with base64 data URI. + + Delegates to convert_file_document_to_url_document after resolving MIME type + from the upload's content_type header or filename. + """ + mime_type = content_type.split(";")[0].strip() if content_type else None + if not mime_type or mime_type == "application/octet-stream": + if filename: + mime_type = get_mime_type(filename) + + return convert_file_document_to_url_document( + { + "type": "file", + "file": file_content, + "mime_type": mime_type or "application/octet-stream", + } + ) + + +async def _parse_multipart_form(request: Request) -> Dict[str, Any]: + """ + Extract OCR data from a multipart form request. + + Uses the cached form if already parsed by auth middleware, + otherwise parses the form from the request. + + Returns: + A dict with 'document', 'model', and any other OCR params. + """ + try: + form = await request.form() + except Exception as e: + raise ValueError( + f"Failed to parse multipart form data: {str(e)}. " + "When using curl with --form/-F, do NOT set the Content-Type header " + "manually — curl will set it automatically with the required boundary." + ) + + uploaded_file = form.get("file") + # request.form() may return either a FastAPI or Starlette UploadFile + # depending on middleware; check both via isinstance (FastAPI's UploadFile + # is a subclass of Starlette's) and fall back to duck-type check. + if uploaded_file is None or ( + not isinstance(uploaded_file, UploadFile) and not hasattr(uploaded_file, "read") + ): + raise ValueError( + "Multipart OCR request must include a 'file' field with the document to process" + ) + + uploaded_file = cast(UploadFile, uploaded_file) + + # Seek to start in case the file was already partially read by middleware + await uploaded_file.seek(0) + file_content = await uploaded_file.read() + if not file_content: + raise ValueError("Uploaded file is empty") + + document = _build_document_from_upload( + file_content=file_content, + filename=uploaded_file.filename, + content_type=uploaded_file.content_type, + ) + + data: Dict[str, Any] = {"document": document} + + for field_name, field_value in form.items(): + if field_name in ("file", "document"): + continue + # Try to parse JSON values (e.g. pages=[0,1,2]) + if isinstance(field_value, str): + try: + data[field_name] = json.loads(field_value) + except (json.JSONDecodeError, ValueError): + data[field_name] = field_value + else: + data[field_name] = field_value + + verbose_proxy_logger.debug( + f"OCR multipart form request parsed - model: {data.get('model')}, " + f"document_type: {document['type']}, " + f"filename: {uploaded_file.filename}" + ) + + return data + + +async def _parse_ocr_request(request: Request) -> Dict[str, Any]: + """ + Parse an OCR request, supporting both JSON and multipart form data. + + JSON body (existing behavior): + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://..."} + } + + Multipart form data (new): + - file: the uploaded file + - model: model name (form field) + - Any other OCR params as form fields (pages, include_image_base64, etc.) + + Returns: + A dict suitable for passing to the OCR processing pipeline. + """ + content_type = request.headers.get("content-type", "") + + if "multipart/form-data" in content_type.lower(): + return await _parse_multipart_form(request) + + # --- JSON body (existing behavior) --- + try: + body = await request.body() + except RuntimeError: + # Body stream was consumed by auth middleware (e.g., form parsing). + body = b"" + + if not body: + # The body may be empty because the auth middleware already parsed + # it as form data (e.g., _read_request_body called request.form()). + # Check if form data is available. + if getattr(request, "_form", None) is not None: + verbose_proxy_logger.debug( + "OCR request body is empty but form data is available from middleware — " + "processing as multipart form." + ) + return await _parse_multipart_form(request) + + raise ValueError( + "Empty request body. For file uploads, use multipart/form-data content type " + "with a file field. When using curl with --form/-F, do NOT set the Content-Type " + "header manually." + ) + + try: + data = orjson.loads(body) + except orjson.JSONDecodeError as e: + raise ValueError( + f"Invalid JSON in request body: {e}. " + "Ensure the request body is valid JSON with Content-Type: application/json, " + "or use multipart/form-data for file uploads." + ) + + # Security: reject type="file" documents received via JSON. + # The "file" document type is designed for local SDK usage where the + # caller and the process share a filesystem. In the proxy context the + # caller is remote, so allowing a file-path string would let an + # authenticated user read arbitrary files from the server's filesystem. + # File uploads must go through multipart/form-data instead. + doc = data.get("document") if isinstance(data, dict) else None + if isinstance(doc, dict) and doc.get("type") == "file": + raise ValueError( + "document type 'file' is not supported through the JSON API. " + "To upload a local file, use multipart/form-data with a 'file' field. " + "For JSON requests, use 'document_url' or 'image_url' document types." + ) + + return data + + @router.post( "/v1/ocr", dependencies=[Depends(user_api_key_auth)], @@ -30,23 +200,30 @@ async def ocr( ): """ OCR endpoint for extracting text from documents and images. - - Follows the Mistral OCR API spec: - https://docs.mistral.ai/capabilities/vision/#optical-character-recognition-ocr - - Example: + + Supports two input modes: + + **1. JSON body** (Mistral OCR API compatible): ```bash curl -X POST "http://localhost:4000/v1/ocr" \ -H "Authorization: Bearer sk-1234" \ -H "Content-Type: application/json" \ -d '{ - "model": "mistral/mistral-ocr-latest", + "model": "mistral-ocr", "document": { "type": "document_url", "document_url": "https://arxiv.org/pdf/2201.04234" } }' ``` + + **2. Multipart form file upload**: + ```bash + curl -X POST "http://localhost:4000/v1/ocr" \ + -H "Authorization: Bearer sk-1234" \ + -F "model=mistral-ocr" \ + -F "file=@document.pdf" + ``` """ from litellm.proxy.proxy_server import ( general_settings, @@ -62,13 +239,14 @@ async def ocr( version, ) - # Read request body - body = await request.body() - data = orjson.loads(body) - - # Process request using ProxyBaseLLMRequestProcessing - processor = ProxyBaseLLMRequestProcessing(data=data) + data: dict = {} try: + # Parse request body (JSON or multipart form) + data = await _parse_ocr_request(request) + + # Process request using ProxyBaseLLMRequestProcessing + processor = ProxyBaseLLMRequestProcessing(data=data) + return await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, @@ -88,10 +266,10 @@ async def ocr( version=version, ) except Exception as e: + processor = ProxyBaseLLMRequestProcessing(data=data) raise await processor._handle_llm_api_exception( e=e, user_api_key_dict=user_api_key_dict, proxy_logging_obj=proxy_logging_obj, version=version, ) - diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 54fc753e768..ac5d9126145 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -1,7 +1,8 @@ import json import os import re -from typing import List +from importlib.resources import files +from typing import Any, Dict, List, Optional import litellm from fastapi import APIRouter, Depends, HTTPException @@ -24,15 +25,106 @@ from litellm.types.proxy.public_endpoints.public_endpoints import ( AgentCreateInfo, ProviderCreateInfo, PublicModelHubInfo, - SupportedEndpointInfo, SupportedEndpointsResponse, - SupportedProviderInfo, ) from litellm.types.utils import LlmProviders router = APIRouter() -_supported_endpoints_cache: SupportedEndpointsResponse | None = None +# --------------------------------------------------------------------------- +# /public/endpoints — helpers +# --------------------------------------------------------------------------- + +_ENDPOINT_METADATA: Dict[str, Dict[str, str]] = { + "chat_completions": {"label": "Chat Completions", "endpoint": "/chat/completions"}, + "messages": {"label": "Messages", "endpoint": "/messages"}, + "responses": {"label": "Responses", "endpoint": "/responses"}, + "embeddings": {"label": "Embeddings", "endpoint": "/embeddings"}, + "image_generations": {"label": "Image Generations", "endpoint": "/images/generations"}, + "audio_transcriptions": {"label": "Audio Transcriptions", "endpoint": "/audio/transcriptions"}, + "audio_speech": {"label": "Audio Speech", "endpoint": "/audio/speech"}, + "moderations": {"label": "Moderations", "endpoint": "/moderations"}, + "batches": {"label": "Batches", "endpoint": "/batches"}, + "rerank": {"label": "Rerank", "endpoint": "/rerank"}, + "ocr": {"label": "OCR", "endpoint": "/ocr"}, + "search": {"label": "Search", "endpoint": "/search"}, + "skills": {"label": "Skills", "endpoint": "/skills"}, + "interactions": {"label": "Interactions", "endpoint": "/interactions"}, + "a2a": {"label": "A2A (Agent Gateway)", "endpoint": "/a2a/{agent}/message/send"}, + "container": {"label": "Containers", "endpoint": "/containers"}, + "container_files": {"label": "Container Files", "endpoint": "/containers/{id}/files"}, + "compact": {"label": "Compact", "endpoint": "/responses/compact"}, + "files": {"label": "Files", "endpoint": "/files"}, + "image_edits": {"label": "Image Edits", "endpoint": "/images/edits"}, + "vector_stores_create": {"label": "Vector Stores (Create)", "endpoint": "/vector_stores"}, + "vector_stores_search": {"label": "Vector Stores (Search)", "endpoint": "/vector_stores/{id}/search"}, + "vector_store_files": {"label": "Vector Store Files", "endpoint": "/vector_stores/{id}/files"}, + "video_generations": {"label": "Video Generations", "endpoint": "/videos/generations"}, + "assistants": {"label": "Assistants", "endpoint": "/assistants"}, + "fine_tuning": {"label": "Fine Tuning", "endpoint": "/fine_tuning/jobs"}, + "text_completion": {"label": "Text Completion", "endpoint": "/completions"}, + "realtime": {"label": "Realtime", "endpoint": "/realtime"}, + "count_tokens": {"label": "Count Tokens", "endpoint": "/utils/token_counter"}, + "image_variations": {"label": "Image Variations", "endpoint": "/images/variations"}, + "generateContent": {"label": "Generate Content", "endpoint": "/generateContent"}, + "bedrock_invoke": {"label": "Bedrock Invoke", "endpoint": "/bedrock/invoke"}, + "bedrock_converse": {"label": "Bedrock Converse", "endpoint": "/bedrock/converse"}, + "rag_ingest": {"label": "RAG Ingest", "endpoint": "/rag/ingest"}, + "rag_query": {"label": "RAG Query", "endpoint": "/rag/query"}, +} + +_SLUG_SUFFIX_RE = re.compile(r"\s*\(`[^`]+`\)\s*$") + +# Loaded once on first request; never invalidated (local file, no TTL needed). +_cached_endpoints: Optional[List[Dict[str, Any]]] = None + + +def _clean_display_name(raw: str) -> str: + return _SLUG_SUFFIX_RE.sub("", raw).strip() + + +def _build_endpoints(raw: Dict[str, Any]) -> List[Dict[str, Any]]: + """Transform raw provider_endpoints_support_backup.json into the response shape.""" + providers: Dict[str, Any] = raw.get("providers", {}) + + # Collect endpoint keys in insertion order (union across all providers). + seen: set = set() + all_keys: List[str] = [] + for provider_data in providers.values(): + for key in provider_data.get("endpoints", {}): + if key not in seen: + seen.add(key) + all_keys.append(key) + + result: List[Dict[str, Any]] = [] + for key in all_keys: + meta = _ENDPOINT_METADATA.get(key) + label = meta["label"] if meta else key.replace("_", " ").title() + path = meta["endpoint"] if meta else "/" + key.replace("_", "/") + + supporting: List[Dict[str, str]] = [ + { + "slug": slug, + "display_name": _clean_display_name(pd.get("display_name", slug)), + } + for slug, pd in providers.items() + if pd.get("endpoints", {}).get(key) + ] + result.append({"key": key, "label": label, "endpoint": path, "providers": supporting}) + + return result + + +def _load_endpoints() -> List[Dict[str, Any]]: + raw = json.loads( + files("litellm") + .joinpath("provider_endpoints_support_backup.json") + .read_text(encoding="utf-8") + ) + return _build_endpoints(raw) + + +# --------------------------------------------------------------------------- @router.get( @@ -232,65 +324,21 @@ async def get_litellm_blog_posts(): @router.get( - "/public/supported_endpoints", - tags=["public", "providers"], + "/public/endpoints", + tags=["public"], response_model=SupportedEndpointsResponse, ) -async def get_provider_supported_endpoints() -> SupportedEndpointsResponse: +async def get_supported_endpoints() -> SupportedEndpointsResponse: """ - Return all supported endpoints and which providers support them. + Return the list of LiteLLM proxy endpoints and which providers support each one. - Reads from provider_endpoints_support.json at the repo root. - Result is cached for the lifetime of the process. + Reads from the bundled local backup file. Result is cached in-process for + the lifetime of the server process. """ - global _supported_endpoints_cache - if _supported_endpoints_cache is not None: - return _supported_endpoints_cache - - provider_endpoints_support_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(__file__))), - "proxy", - "public_endpoints", - "provider_endpoints_support.json", - ) - - with open(provider_endpoints_support_path, "r") as f: - data = json.load(f) - - schema_endpoints = data["_schema"]["provider_slug"]["endpoints"] - - endpoints = [] - for key, description in schema_endpoints.items(): - path_match = re.search(r"(/[\w/{}.()*-]+)", description) - endpoint_path = path_match.group(1) if path_match else f"/{key}" - display_name = key.replace("_", " ").title() - endpoints.append( - SupportedEndpointInfo( - key=key, - display_name=display_name, - endpoint=endpoint_path, - ) - ) - - providers = [] - for slug, provider_data in data["providers"].items(): - supported = [ - endpoint_key - for endpoint_key, supported in provider_data["endpoints"].items() - if supported - ] - providers.append( - SupportedProviderInfo( - slug=slug, - display_name=provider_data["display_name"], - supported=supported, - ) - ) - - _supported_endpoints_cache = SupportedEndpointsResponse( - endpoints=endpoints, providers=providers - ) - return _supported_endpoints_cache + global _cached_endpoints + if _cached_endpoints is None: + _cached_endpoints = SupportedEndpointsResponse(endpoints=_load_endpoints()) + return _cached_endpoints @router.get( @@ -301,7 +349,7 @@ async def get_provider_supported_endpoints() -> SupportedEndpointsResponse: async def get_agent_fields() -> List[AgentCreateInfo]: """ Return agent type metadata required by the dashboard create-agent flow. - + If an agent has `inherit_credentials_from_provider`, the provider's credential fields are automatically appended to the agent's credential_fields. """ @@ -310,19 +358,19 @@ async def get_agent_fields() -> List[AgentCreateInfo]: "proxy", "public_endpoints", ) - + agent_create_fields_path = os.path.join(base_path, "agent_create_fields.json") provider_create_fields_path = os.path.join(base_path, "provider_create_fields.json") with open(agent_create_fields_path, "r") as f: agent_create_fields = json.load(f) - + with open(provider_create_fields_path, "r") as f: provider_create_fields = json.load(f) - + # Build a lookup map for providers by name provider_map = {p["provider"]: p for p in provider_create_fields} - + # Merge inherited credential fields for agent in agent_create_fields: inherit_from = agent.get("inherit_credentials_from_provider") diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 440c9c1d829..13461be3e7c 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? @@ -504,6 +505,7 @@ model LiteLLM_SpendLogs { agent_id String? proxy_server_request Json? @default("{}") @@index([startTime]) + @@index([startTime, request_id]) @@index([end_user]) @@index([session_id]) } diff --git a/litellm/realtime_api/main.py b/litellm/realtime_api/main.py index ac597fc623d..3e64f61abdb 100644 --- a/litellm/realtime_api/main.py +++ b/litellm/realtime_api/main.py @@ -106,6 +106,8 @@ async def _arealtime( # noqa: PLR0915 client=client, timeout=timeout, headers=headers, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) elif _custom_llm_provider == "azure": api_base = ( @@ -277,6 +279,8 @@ async def _arealtime( # noqa: PLR0915 client=client, timeout=timeout, headers=headers, + user_api_key_dict=kwargs.get("user_api_key_dict"), + litellm_metadata=_build_litellm_metadata(kwargs), ) else: raise ValueError(f"Unsupported model: {model}") diff --git a/litellm/router.py b/litellm/router.py index cbe5b414040..d89a5099b01 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -6677,6 +6677,22 @@ class Router: # initialize client self._add_deployment(deployment=deployment) + # Register custom pricing in litellm.model_cost. + # Mirrors _create_deployment() logic to ensure dynamically-added deployments + # (e.g., loaded from DB) also have their custom pricing registered. + # Without this, _is_model_cost_zero() cannot detect explicitly-configured + # zero-cost models, causing budget checks to block free models. + _model_id = deployment.model_info.id + if _model_id is not None: + _model_info_dict: dict = deployment.model_info.model_dump( + exclude_none=True + ) + for field in CustomPricingLiteLLMParams.model_fields.keys(): + field_value = deployment.litellm_params.get(field) + if field_value is not None: + _model_info_dict[field] = field_value + litellm.register_model(model_cost={_model_id: _model_info_dict}) + # add to model names self._add_model_to_list_and_index_map( model=_deployment, model_id=deployment.model_info.id diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 15e8d1be930..c0aae9bc2de 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -699,7 +699,15 @@ class OpenAIChatCompletionAssistantMessage(TypedDict, total=False): role: Required[Literal["assistant"]] content: Optional[ Union[ - str, Iterable[Union[ChatCompletionTextObject, ChatCompletionThinkingBlock]] + str, + Iterable[ + Union[ + ChatCompletionTextObject, + ChatCompletionThinkingBlock, + ChatCompletionRedactedThinkingBlock, + ChatCompletionImageObject, + ] + ], ] ] name: Optional[str] @@ -786,17 +794,19 @@ ValidUserMessageContentTypes = [ "file", ] # used for validating user messages. Prevent users from accidentally sending anthropic messages. -# Assistant message content types (text, thinking, redacted_thinking) +# Assistant message content types (text, thinking, redacted_thinking, image_url) ValidAssistantMessageContentTypesLiteral = Literal[ "text", "thinking", "redacted_thinking", + "image_url", ] ValidAssistantMessageContentTypes = [ "text", "thinking", "redacted_thinking", + "image_url", ] # Combined valid content types for chat completion messages diff --git a/litellm/types/proxy/public_endpoints/public_endpoints.py b/litellm/types/proxy/public_endpoints/public_endpoints.py index a167eeec6a2..caa9a978530 100644 --- a/litellm/types/proxy/public_endpoints/public_endpoints.py +++ b/litellm/types/proxy/public_endpoints/public_endpoints.py @@ -54,18 +54,17 @@ class AgentCreateInfo(BaseModel): model_template: Optional[str] = None -class SupportedEndpointInfo(BaseModel): - key: str - display_name: str - endpoint: str - - -class SupportedProviderInfo(BaseModel): +class EndpointProvider(BaseModel): slug: str display_name: str - supported: List[str] + + +class SupportedEndpoint(BaseModel): + key: str + label: str + endpoint: str + providers: List[EndpointProvider] class SupportedEndpointsResponse(BaseModel): - endpoints: List[SupportedEndpointInfo] - providers: List[SupportedProviderInfo] + endpoints: List[SupportedEndpoint] diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 57563fc0bcc..d5b79bbeb5b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -14194,6 +14194,38 @@ "supports_vision": true, "supports_web_search": true }, + "gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models", + "supported_endpoints": [ + "/v1/chat/completions", + "/v1/completions", + "/v1/batch" + ], + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "text", + "image" + ], + "supports_function_calling": false, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_system_messages": true, + "supports_vision": true, + "supports_web_search": true + }, "deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, @@ -19178,6 +19210,39 @@ "supports_tool_choice": true, "supports_vision": false }, + "gpt-audio-1.5": { + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_token": 2.5e-06, + "litellm_provider": "openai", + "max_input_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1e-05, + "supported_endpoints": [ + "/v1/chat/completions" + ], + "supported_modalities": [ + "text", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_native_streaming": true, + "supports_parallel_function_calling": true, + "supports_prompt_caching": false, + "supports_reasoning": false, + "supports_response_schema": false, + "supports_system_messages": true, + "supports_tool_choice": true, + "supports_vision": false + }, "gpt-audio-2025-08-28": { "input_cost_per_audio_token": 3.2e-05, "input_cost_per_token": 2.5e-06, @@ -20895,6 +20960,38 @@ "supports_system_messages": true, "supports_tool_choice": true }, + "gpt-realtime-1.5": { + "cache_creation_input_audio_token_cost": 4e-07, + "cache_read_input_token_cost": 4e-07, + "input_cost_per_audio_token": 3.2e-05, + "input_cost_per_image": 5e-06, + "input_cost_per_token": 4e-06, + "litellm_provider": "openai", + "max_input_tokens": 32000, + "max_output_tokens": 4096, + "max_tokens": 4096, + "mode": "chat", + "output_cost_per_audio_token": 6.4e-05, + "output_cost_per_token": 1.6e-05, + "supported_endpoints": [ + "/v1/realtime" + ], + "supported_modalities": [ + "text", + "image", + "audio" + ], + "supported_output_modalities": [ + "text", + "audio" + ], + "supports_audio_input": true, + "supports_audio_output": true, + "supports_function_calling": true, + "supports_parallel_function_calling": true, + "supports_system_messages": true, + "supports_tool_choice": true + }, "gpt-realtime-mini": { "cache_creation_input_audio_token_cost": 3e-07, "cache_read_input_audio_token_cost": 3e-07, @@ -26618,8 +26715,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/swiss-ai/apertus-70b-instruct": { "input_cost_per_token": 0.0, @@ -26630,8 +26727,8 @@ "mode": "chat", "output_cost_per_token": 0.0, "source": "https://platform.publicai.co/docs", - "supports_function_calling": true, - "supports_tool_choice": true + "supports_function_calling": false, + "supports_tool_choice": false }, "publicai/aisingapore/Gemma-SEA-LION-v4-27B-IT": { "input_cost_per_token": 0.0, @@ -31545,6 +31642,19 @@ "output_cost_per_token_batches": 6e-06, "source": "https://docs.cloud.google.com/vertex-ai/generative-ai/docs/models/gemini/3-pro-image" }, + "vertex_ai/gemini-3.1-flash-image-preview": { + "input_cost_per_image": 0.00056, + "input_cost_per_token": 5e-07, + "litellm_provider": "vertex_ai-language-models", + "max_input_tokens": 65536, + "max_output_tokens": 32768, + "max_tokens": 32768, + "mode": "image_generation", + "output_cost_per_image": 0.0672, + "output_cost_per_image_token": 6e-05, + "output_cost_per_token": 3e-06, + "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#gemini-models" + }, "vertex_ai/deep-research-pro-preview-12-2025": { "input_cost_per_image": 0.0011, "input_cost_per_token": 2e-06, diff --git a/poetry.lock b/poetry.lock index 34227a69ccb..0314a360542 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3222,15 +3222,15 @@ files = [ [[package]] name = "litellm-proxy-extras" -version = "0.4.48" +version = "0.4.49" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." optional = true python-versions = "!=2.7.*,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,!=3.7.*,>=3.8" groups = ["main"] markers = "extra == \"proxy\"" files = [ - {file = "litellm_proxy_extras-0.4.48-py3-none-any.whl", hash = "sha256:097001fccec5dbf4cffd902114898a9cfeba62673202447d55d2d0286cf93126"}, - {file = "litellm_proxy_extras-0.4.48.tar.gz", hash = "sha256:5d5d8acf31b92d0cd6738555fb4a2411819755155438de9fb23c724c356400a2"}, + {file = "litellm_proxy_extras-0.4.49-py3-none-any.whl", hash = "sha256:aeb0e08b4705c19fdc5b75a43c608a82fc36032f6d83be509dbf37baea62f2cd"}, + {file = "litellm_proxy_extras-0.4.49.tar.gz", hash = "sha256:d9bdae54d1e3398f2e2025c9d8b98a19e226874337d540d5415922d7dbbc97bb"}, ] [[package]] @@ -7989,4 +7989,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.9,<4.0" -content-hash = "b9b1e47b3b84748c0053be6a544c2399bf2601746a4f88dcb1be7c5e4eeab359" +content-hash = "bbc7d43f5484af4c8877fe66e34f8283069528379af49d573036ba144cc2eb7a" diff --git a/litellm/proxy/public_endpoints/provider_endpoints_support.json b/provider_endpoints_support.json similarity index 100% rename from litellm/proxy/public_endpoints/provider_endpoints_support.json rename to provider_endpoints_support.json diff --git a/pyproject.toml b/pyproject.toml index 8eb433fb064..07f3ea30fc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,7 +61,7 @@ boto3 = { version = "1.40.76", optional = true } redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = ">=1.25.0,<2.0.0", optional = true, python = ">=3.10"} a2a-sdk = {version = "^0.3.22", optional = true, python = ">=3.10"} -litellm-proxy-extras = {version = "0.4.48", optional = true} +litellm-proxy-extras = {version = "0.4.49", optional = true} rich = {version = "13.7.1", optional = true} litellm-enterprise = {version = "0.1.32", optional = true} diskcache = {version = "^5.6.1", optional = true} diff --git a/requirements.txt b/requirements.txt index 6cdb2f63a33..67f390cf272 100644 --- a/requirements.txt +++ b/requirements.txt @@ -57,7 +57,7 @@ grpcio>=1.75.0; python_version >= "3.14" sentry_sdk==2.21.0 # for sentry error handling detect-secrets==1.5.0 # Enterprise - secret detection / masking in LLM requests tzdata==2025.1 # IANA time zone database -litellm-proxy-extras==0.4.48 # for proxy extras - e.g. prisma migrations +litellm-proxy-extras==0.4.49 # for proxy extras - e.g. prisma migrations llm-sandbox==0.3.31 # for skill execution in sandbox ### LITELLM PACKAGE DEPENDENCIES python-dotenv==1.0.1 # for env diff --git a/schema.prisma b/schema.prisma index 440c9c1d829..34308b29ebf 100644 --- a/schema.prisma +++ b/schema.prisma @@ -390,6 +390,7 @@ model LiteLLM_DeletedVerificationToken { config Json @default("{}") user_id String? team_id String? + agent_id String? project_id String? permissions Json @default("{}") max_parallel_requests Int? diff --git a/tests/code_coverage_tests/check_endpoint_coverage.py b/tests/code_coverage_tests/check_endpoint_coverage.py index 25f181aa3a5..2d46d1ab469 100644 --- a/tests/code_coverage_tests/check_endpoint_coverage.py +++ b/tests/code_coverage_tests/check_endpoint_coverage.py @@ -99,7 +99,7 @@ def extract_endpoints_from_sidebars() -> Dict[str, str]: def load_provider_endpoints_file() -> Dict: """Load the provider_endpoints_support.json file.""" repo_root = get_repo_root() - file_path = repo_root / "litellm" / "proxy" / "public_endpoints" / "provider_endpoints_support.json" + file_path = repo_root / "provider_endpoints_support.json" if not file_path.exists(): print( diff --git a/tests/code_coverage_tests/check_provider_folders_documented.py b/tests/code_coverage_tests/check_provider_folders_documented.py index d5f7c7f7a07..60afc55331f 100644 --- a/tests/code_coverage_tests/check_provider_folders_documented.py +++ b/tests/code_coverage_tests/check_provider_folders_documented.py @@ -65,7 +65,7 @@ def get_llm_provider_folders() -> Set[str]: def load_provider_endpoints_file() -> Dict: """Load the provider_endpoints_support.json file.""" repo_root = get_repo_root() - file_path = repo_root / "litellm" / "proxy" / "public_endpoints" / "provider_endpoints_support.json" + file_path = repo_root / "provider_endpoints_support.json" if not file_path.exists(): print( diff --git a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py index 08b9351f9a3..61b1b1f8185 100644 --- a/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py +++ b/tests/enterprise/litellm_enterprise/enterprise_callbacks/test_prometheus_logging_callbacks.py @@ -2316,5 +2316,3 @@ async def test_prometheus_token_metrics_with_prometheus_config(): raise AssertionError(f"Metric {metric_name} not found in registry") print("✓ All token metrics validated successfully!") - - # check final value of metrics in registry diff --git a/tests/llm_translation/realtime/test_realtime_guardrails_openai.py b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py new file mode 100644 index 00000000000..a7913e6d761 --- /dev/null +++ b/tests/llm_translation/realtime/test_realtime_guardrails_openai.py @@ -0,0 +1,355 @@ +""" +Integration tests for RealTimeStreaming guardrails against a live OpenAI backend. + +These tests require OPENAI_API_KEY and are skipped if not set. + +They verify end-to-end that: + 1. A text message blocked by a guardrail -> error event sent to client, NO AI response. + 2. A voice transcript blocked by a guardrail -> error event sent, response.create NOT sent. + 3. A clean text message passes through and triggers a real OpenAI response. + +Run with: + poetry run pytest tests/llm_translation/realtime/test_realtime_guardrails_openai.py -v -s +""" + +import asyncio +import json +import os +from typing import List +from unittest.mock import AsyncMock, MagicMock + +import pytest + +import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming +from litellm.types.guardrails import GuardrailEventHooks + +OPENAI_API_KEY = os.environ.get("OPENAI_API_KEY") +OPENAI_REALTIME_URL = ( + "wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-12-17" +) + +pytestmark = pytest.mark.skipif( + not OPENAI_API_KEY, + reason="OPENAI_API_KEY not set - skipping OpenAI realtime integration tests", +) + +# A unique phrase guaranteed NOT to appear in normal assistant output. +BLOCKED_PHRASE = "XSECRETBLOCKTESTPHRASEX" + + +class PhraseBlockingGuardrail(CustomGuardrail): + """Blocks any message containing BLOCKED_PHRASE.""" + + async def apply_guardrail( + self, inputs, request_data, input_type, logging_obj=None + ): + for text in inputs.get("texts", []): + if BLOCKED_PHRASE in text: + raise ValueError( + "Content blocked: contains forbidden test phrase." + ) + return inputs + + +def _make_guardrail(event_hook=GuardrailEventHooks.pre_call): + return PhraseBlockingGuardrail( + guardrail_name="integration-test-guard", + event_hook=event_hook, + default_on=True, + ) + + +async def _wait_for_event( + client_events: List[dict], event_type: str, timeout: float = 15.0 +) -> dict: + """Poll client_events list until an event with matching type appears.""" + deadline = asyncio.get_event_loop().time() + timeout + while asyncio.get_event_loop().time() < deadline: + matching = [e for e in client_events if e.get("type") == event_type] + if matching: + return matching[0] + await asyncio.sleep(0.05) + raise TimeoutError( + f"Timed out waiting for '{event_type}'. Got so far: {[e.get('type') for e in client_events]}" + ) + + +async def _build_streaming(client_events: List[dict], backend_ws, request_data=None): + """Create a RealTimeStreaming with a mock client WebSocket that captures events.""" + client_ws = MagicMock() + input_queue: asyncio.Queue = asyncio.Queue() + + async def send_text(data: str): + client_events.append(json.loads(data)) + + client_ws.send_text = send_text + client_ws.receive_text = input_queue.get + + logging_obj = MagicMock() + logging_obj.pre_call = MagicMock() + logging_obj.async_success_handler = AsyncMock() + logging_obj.success_handler = MagicMock() + logging_obj.model_call_details = {} + + streaming = RealTimeStreaming( + websocket=client_ws, + backend_ws=backend_ws, + logging_obj=logging_obj, + request_data=request_data or {"guardrails": ["integration-test-guard"]}, + ) + return streaming, input_queue + + +@pytest.mark.asyncio +async def test_text_message_blocked_by_guardrail_no_ai_response(): + """ + Send a text message containing the blocked phrase. + Guardrail must: + - Send error event (guardrail_violation) to client. + - Send response.audio_transcript.delta with the block message to client. + - NOT forward response.create to OpenAI (no AI response). + """ + import websockets + + guardrail = _make_guardrail(GuardrailEventHooks.pre_call) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + try: + async with websockets.connect( + OPENAI_REALTIME_URL, + additional_headers={ + "Authorization": f"Bearer {OPENAI_API_KEY}", + "OpenAI-Beta": "realtime=v1", + }, + ) as backend_ws: + streaming, input_queue = await _build_streaming(client_events, backend_ws) + + # Start backend -> client forwarding + backend_task = asyncio.create_task( + streaming.backend_to_client_send_messages() + ) + # Start client -> backend forwarding (reads from input_queue) + client_task = asyncio.create_task(streaming.client_ack_messages()) + + try: + # Wait until session is ready + await _wait_for_event(client_events, "session.created", timeout=15) + + # Send the blocked message + response.create + blocked_item = json.dumps( + { + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [ + { + "type": "input_text", + "text": f"Hello {BLOCKED_PHRASE}", + } + ], + }, + } + ) + await input_queue.put(blocked_item) + # Give guardrail time to process before the follow-up response.create + await asyncio.sleep(0.3) + await input_queue.put(json.dumps({"type": "response.create"})) + + # Allow time for guardrail round-trip + await asyncio.sleep(3.0) + + finally: + backend_task.cancel() + client_task.cancel() + await asyncio.gather(backend_task, client_task, return_exceptions=True) + + # --- Assertions --- + event_types = [e.get("type") for e in client_events] + + # 1. Must have received guardrail error + error_events = [e for e in client_events if e.get("type") == "error"] + assert len(error_events) >= 1, ( + f"Expected at least one error event but got: {event_types}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation", ( + f"Wrong error type: {error_events[0]}" + ) + + # 2. Must have the guardrail message surfaced as an AI transcript delta + transcript_deltas = [ + e + for e in client_events + if e.get("type") == "response.audio_transcript.delta" + ] + assert len(transcript_deltas) >= 1, ( + f"Expected guardrail message in transcript delta, got: {event_types}" + ) + + # 3. No real AI response should have been generated - response.done would only + # appear if we sent a response.create and OpenAI replied. We allow it in the + # synthetic form (empty output=[]) but NOT with actual AI content. + done_events = [e for e in client_events if e.get("type") == "response.done"] + for done in done_events: + output = done.get("response", {}).get("output", []) + ai_texts = [ + c.get("text", "") or c.get("transcript", "") + for item in output + for c in item.get("content", []) + ] + real_ai_text = " ".join(ai_texts).strip() + assert real_ai_text == "", ( + f"AI responded with real content even though message was blocked: {real_ai_text!r}" + ) + + finally: + litellm.callbacks = [] + + +@pytest.mark.asyncio +async def test_voice_transcript_blocked_by_guardrail(): + """ + Simulate a backend-side voice transcription event containing the blocked phrase. + Guardrail must block it - no response.create sent to OpenAI. + """ + from websockets.exceptions import ConnectionClosed + + guardrail = _make_guardrail(GuardrailEventHooks.realtime_input_transcription) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + # Build the transcript event that would come from the OpenAI backend + transcript_event = json.dumps( + { + "type": "conversation.item.input_audio_transcription.completed", + "transcript": f"This is {BLOCKED_PHRASE} in my voice message", + "item_id": "item_integ_test", + } + ).encode() + + # Mock backend that delivers the transcript then closes + backend_ws = MagicMock() + backend_ws.recv = AsyncMock( + side_effect=[ + transcript_event, + ConnectionClosed(None, None), + ] + ) + backend_ws.send = AsyncMock() + + try: + streaming, _ = await _build_streaming(client_events, backend_ws) + await streaming.backend_to_client_send_messages() + + event_types = [e.get("type") for e in client_events] + + # 1. Error event must be sent to client + error_events = [e for e in client_events if e.get("type") == "error"] + assert len(error_events) >= 1, ( + f"Expected guardrail error event, got: {event_types}" + ) + assert error_events[0]["error"]["type"] == "guardrail_violation" + + # 2. response.create must NOT have been sent to backend + sent_to_backend = [ + json.loads(c.args[0]) + for c in backend_ws.send.call_args_list + if c.args and isinstance(c.args[0], str) + ] + response_creates = [ + e for e in sent_to_backend if e.get("type") == "response.create" + ] + assert len(response_creates) == 0, ( + f"Guardrail should have stopped response.create, got: {sent_to_backend}" + ) + + # 3. Guardrail message surfaced as AI transcript delta + transcript_deltas = [ + e + for e in client_events + if e.get("type") == "response.audio_transcript.delta" + ] + assert len(transcript_deltas) >= 1, ( + f"Expected guardrail message in transcript delta, got: {event_types}" + ) + + finally: + litellm.callbacks = [] + + +@pytest.mark.asyncio +async def test_clean_text_message_passes_through_to_openai(): + """ + A clean message (no blocked phrase) must pass the guardrail and result in a real + AI response from OpenAI (response.done with non-empty output). + """ + import websockets + + guardrail = _make_guardrail(GuardrailEventHooks.pre_call) + litellm.callbacks = [guardrail] + + client_events: List[dict] = [] + + try: + async with websockets.connect( + OPENAI_REALTIME_URL, + additional_headers={ + "Authorization": f"Bearer {OPENAI_API_KEY}", + "OpenAI-Beta": "realtime=v1", + }, + ) as backend_ws: + streaming, input_queue = await _build_streaming(client_events, backend_ws) + + backend_task = asyncio.create_task( + streaming.backend_to_client_send_messages() + ) + client_task = asyncio.create_task(streaming.client_ack_messages()) + + try: + await _wait_for_event(client_events, "session.created", timeout=15) + + # Send a clean message + clean_item = json.dumps( + { + "type": "conversation.item.create", + "item": { + "role": "user", + "content": [ + {"type": "input_text", "text": "Reply with just: OK"} + ], + }, + } + ) + await input_queue.put(clean_item) + await asyncio.sleep(0.1) + await input_queue.put(json.dumps({"type": "response.create"})) + + # Wait for OpenAI to respond + await _wait_for_event(client_events, "response.done", timeout=30) + + finally: + backend_task.cancel() + client_task.cancel() + await asyncio.gather(backend_task, client_task, return_exceptions=True) + + # No guardrail error should have been sent + error_events = [e for e in client_events if e.get("type") == "error"] + guardrail_errors = [ + e for e in error_events if e.get("error", {}).get("type") == "guardrail_violation" + ] + assert len(guardrail_errors) == 0, ( + f"Clean message should not trigger guardrail, got: {guardrail_errors}" + ) + + # AI response must be present + done_events = [e for e in client_events if e.get("type") == "response.done"] + assert len(done_events) >= 1, ( + f"Expected response.done from OpenAI, got: {[e.get('type') for e in client_events]}" + ) + + finally: + litellm.callbacks = [] diff --git a/tests/llm_translation/test_prompt_factory.py b/tests/llm_translation/test_prompt_factory.py index 88bca007740..9f902f2bd86 100644 --- a/tests/llm_translation/test_prompt_factory.py +++ b/tests/llm_translation/test_prompt_factory.py @@ -1214,6 +1214,181 @@ def test_anthropic_messages_pt_with_server_tool_use(): assert tool_use["id"] == "toolu_01XYZ789" +def test_convert_to_anthropic_tool_invoke_with_tool_results(): + """ + Test that non-web-search *_tool_result blocks (e.g. bash_code_execution_tool_result) + stored in provider_specific_fields["tool_results"] are paired with their server_tool_use + block when reconstructing assistant history. + + Regression for: server tool result blocks dropped on multi-turn replay + (bash_code_execution_tool_result, text_editor_code_execution_tool_result, etc.) + """ + tool_calls = [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(2)\\""}', + }, + } + ] + + tool_results = [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + + result = convert_to_anthropic_tool_invoke(tool_calls, tool_results=tool_results) + + assert len(result) == 2 + # First: server_tool_use + assert result[0]["type"] == "server_tool_use" + assert result[0]["id"] == "srvtoolu_01BASH" + assert result[0]["name"] == "bash_code_execution" + # Second: bash_code_execution_tool_result paired correctly + assert result[1]["type"] == "bash_code_execution_tool_result" + assert result[1]["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_raw_bash_tool_result_passthrough(): + """ + Test that raw assistant content lists containing bash_code_execution_tool_result + blocks are passed through intact to Anthropic. + + Regression: the raw-block passthrough only handled tool_search_tool_result; + bash_code_execution_tool_result and other *_tool_result types were silently dropped. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": [ + { + "type": "server_tool_use", + "id": "srvtoolu_01BASH", + "name": "bash_code_execution", + "input": {"command": "python3 -c \"print(1+1)\""}, + }, + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + }, + {"type": "text", "text": "The answer is 2."}, + ], + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be preserved" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result block must not be dropped" + assert "text" in types + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + +def test_anthropic_messages_pt_with_bash_tool_result_in_provider_specific_fields(): + """ + Test that anthropic_messages_pt correctly reconstructs bash_code_execution_tool_result + from provider_specific_fields["tool_results"] when replaying LiteLLM response objects. + + Regression: only web_search_results were read from provider_specific_fields; + tool_results (bash_code_execution_tool_result, etc.) were silently lost. + """ + messages = [ + {"role": "user", "content": "What is 1+1?"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "srvtoolu_01BASH", + "type": "function", + "function": { + "name": "bash_code_execution", + "arguments": '{"command": "python3 -c \\"print(1+1)\\""}', + }, + } + ], + "provider_specific_fields": { + "tool_results": [ + { + "type": "bash_code_execution_tool_result", + "tool_use_id": "srvtoolu_01BASH", + "content": { + "type": "bash_code_execution_result", + "stdout": "2\n", + "stderr": "", + "return_code": 0, + "content": [], + }, + } + ] + }, + }, + {"role": "user", "content": "Thanks!"}, + ] + + result = anthropic_messages_pt( + messages, model="claude-sonnet-4-5", llm_provider="anthropic" + ) + + assistant_msg = next(m for m in result if m["role"] == "assistant") + content = assistant_msg["content"] + types = [c.get("type") for c in content] + + assert "server_tool_use" in types, "server_tool_use block must be reconstructed" + assert ( + "bash_code_execution_tool_result" in types + ), "bash_code_execution_tool_result must be paired from provider_specific_fields['tool_results']" + + # Result must immediately follow its server_tool_use + srv_idx = types.index("server_tool_use") + result_idx = types.index("bash_code_execution_tool_result") + assert result_idx == srv_idx + 1 + + srv = next(c for c in content if c.get("type") == "server_tool_use") + assert srv["id"] == "srvtoolu_01BASH" + bash_result = next( + c for c in content if c.get("type") == "bash_code_execution_tool_result" + ) + assert bash_result["tool_use_id"] == "srvtoolu_01BASH" + + # ============ parse_tool_call_arguments Tests ============ # Tests for the shared utility that parses tool call JSON arguments diff --git a/tests/mcp_tests/test_mcp_server.py b/tests/mcp_tests/test_mcp_server.py index 930b0a03042..dc1e2068365 100644 --- a/tests/mcp_tests/test_mcp_server.py +++ b/tests/mcp_tests/test_mcp_server.py @@ -427,7 +427,8 @@ async def test_streamable_http_mcp_handler_mock(): # Call the handler await handle_streamable_http_mcp(mock_scope, mock_receive, mock_send) - # Verify session manager handle_request was called + # Verify session manager handle_request was called with correct args + # send is passed directly (no wrapper) mock_session_manager.handle_request.assert_called_once_with( mock_scope, mock_receive, mock_send ) diff --git a/tests/test_litellm/caching/test_redis_connection_pool.py b/tests/test_litellm/caching/test_redis_connection_pool.py index b8922846e82..3d808438850 100644 --- a/tests/test_litellm/caching/test_redis_connection_pool.py +++ b/tests/test_litellm/caching/test_redis_connection_pool.py @@ -1,12 +1,8 @@ -""" -Regression tests for Redis connection pool leak fixes (RC1-RC5). +"""Redis connection pool and LLMClientCache eviction tests.""" -Tests are pure unit tests — no Redis server required. -""" - -import asyncio from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import redis.asyncio as async_redis @@ -131,37 +127,33 @@ async def test_disconnect_idempotent(): await cache.disconnect() # should not raise +# Regression: cache eviction must not close shared httpx clients (PR #22247) + @pytest.mark.asyncio -async def test_eviction_calls_aclose(): - """When an async client is evicted from LLMClientCache, its aclose() - should be scheduled via create_task.""" - cache = LLMClientCache(max_size_in_memory=2, default_ttl=600) +async def test_httpx_client_survives_capacity_eviction(): + """Evicting an httpx client from LLMClientCache must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=1, default_ttl=600) + client = httpx.AsyncClient() - client = AsyncMock() - client.aclose = AsyncMock() + cache.set_cache("client_1", client) + # Exceed capacity — client_1 gets evicted + cache.set_cache("client_2", "other") - cache.set_cache(key="client-0", value=client) - cache.set_cache(key="filler", value="x") - # Third insert triggers eviction of client-0 - cache.set_cache(key="trigger", value="y") - - # Let the scheduled task run - await asyncio.sleep(0.05) - - assert client.aclose.await_count > 0 + assert not client.is_closed + await client.aclose() @pytest.mark.asyncio -async def test_eviction_non_closeable_safe(): - """Evicting plain values (strings, dicts, ints) should not crash.""" - cache = LLMClientCache(max_size_in_memory=2, default_ttl=600) +async def test_httpx_client_survives_ttl_eviction(): + """Evicting an httpx client via TTL expiry must NOT close it.""" + cache = LLMClientCache(max_size_in_memory=200, default_ttl=600) + client = httpx.AsyncClient() - cache.set_cache(key="str-val", value="hello") - cache.set_cache(key="dict-val", value={"foo": "bar"}) - # This evicts "str-val" — should not raise - cache.set_cache(key="int-val", value=42) + # TTL=0 so it expires immediately + cache.set_cache("client_1", client, ttl=0) + cache.evict_cache() + + assert not client.is_closed + await client.aclose() - await asyncio.sleep(0.05) - # If we got here without exception, the test passes - assert cache.get_cache(key="int-val") == 42 diff --git a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py index a840e2fe162..cd76ba1e863 100644 --- a/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py +++ b/tests/test_litellm/integrations/test_prometheus_user_team_metrics.py @@ -1,7 +1,8 @@ """ Unit tests for Prometheus user and team count metrics """ -from unittest.mock import MagicMock +from datetime import datetime, timezone +from unittest.mock import MagicMock, patch import pytest from prometheus_client import REGISTRY @@ -258,3 +259,267 @@ class TestPrometheusUserTeamCountMetrics: assert True except Exception as e: pytest.fail(f"Metrics should handle large values: {e}") + + +# --------------------------------------------------------------------------- +# Regression tests: team budget showing +Inf when user_api_key_team_max_budget +# is None in request metadata but the team has a real budget in the DB. +# --------------------------------------------------------------------------- + + +async def test_assemble_team_object_uses_db_max_budget_when_metadata_is_none( + prometheus_logger, +): + """ + When max_budget is None in request metadata (e.g. stale key cache), + _assemble_team_object must fall back to the value returned by get_team_object + so that _safe_get_remaining_budget does not return +Inf. + """ + db_team = MagicMock() + db_team.max_budget = 3000.0 + db_team.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="c5c33858-4379-4c90-8733-d9c58c312c10", + team_alias="ai-ml-local_dev", + spend=1617.02, + max_budget=None, # simulates None coming from request metadata + response_cost=0.5, + ) + + assert team_object.max_budget == 3000.0, ( + "max_budget should be populated from DB when metadata value is None" + ) + assert team_object.budget_reset_at == datetime(2026, 3, 1, tzinfo=timezone.utc) + + +async def test_assemble_team_object_does_not_override_metadata_max_budget( + prometheus_logger, +): + """ + When max_budget IS present in request metadata, it must not be overridden + by the DB value. + """ + db_team = MagicMock() + db_team.max_budget = 9999.0 + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + team_object = await prometheus_logger._assemble_team_object( + team_id="team-1", + team_alias="my-team", + spend=50.0, + max_budget=100.0, # metadata has a real value + response_cost=1.0, + ) + + assert team_object.max_budget == 100.0, ( + "max_budget from metadata must not be replaced by the DB value" + ) + + +async def test_set_team_budget_metrics_after_api_request_no_inf_when_metadata_budget_none( + prometheus_logger, +): + """ + End-to-end: when user_api_key_team_max_budget is None in request metadata + but the team has a real budget in the DB, the metric must NOT be set to +Inf. + """ + prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() + prometheus_logger.litellm_team_max_budget_metric = MagicMock() + prometheus_logger.litellm_team_budget_remaining_hours_metric = MagicMock() + + db_team = MagicMock() + db_team.max_budget = 3000.0 + db_team.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + await prometheus_logger._set_team_budget_metrics_after_api_request( + user_api_team="c5c33858-4379-4c90-8733-d9c58c312c10", + user_api_team_alias="ai-ml-local_dev", + team_spend=1617.02, + team_max_budget=None, # simulates stale key cache + response_cost=0.5, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_team_budget_metric.labels().set.call_args + ) + assert set_call_args is not None, "remaining_team_budget_metric.labels().set was not called" + actual_value = set_call_args[0][0] + assert actual_value != float("inf"), ( + f"remaining_team_budget_metric must not be +Inf when team has a real budget; got {actual_value}" + ) + expected = 3000.0 - 1617.02 - 0.5 + assert abs(actual_value - expected) < 0.01, ( + f"Expected remaining budget ~{expected}, got {actual_value}" + ) + + +async def test_set_team_budget_metrics_after_api_request_inf_when_genuinely_no_budget( + prometheus_logger, +): + """ + When the team genuinely has no budget (max_budget=None in both metadata and + DB), +Inf is the correct value and must be preserved. + """ + prometheus_logger.litellm_remaining_team_budget_metric = MagicMock() + prometheus_logger.litellm_team_max_budget_metric = MagicMock() + prometheus_logger.litellm_team_budget_remaining_hours_metric = MagicMock() + + db_team = MagicMock() + db_team.max_budget = None + db_team.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_team_object") as mock_get_team: + mock_get_team.return_value = db_team + await prometheus_logger._set_team_budget_metrics_after_api_request( + user_api_team="team-no-budget", + user_api_team_alias="no-budget-team", + team_spend=10.0, + team_max_budget=None, + response_cost=1.0, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_team_budget_metric.labels().set.call_args + ) + assert set_call_args is not None + actual_value = set_call_args[0][0] + assert actual_value == float("inf"), ( + "remaining_team_budget_metric should be +Inf when team truly has no budget" + ) + + +# --------------------------------------------------------------------------- +# Regression tests: user budget showing +Inf when user_api_key_user_max_budget +# is None in request metadata but the user has a real budget in the DB. +# --------------------------------------------------------------------------- + + +async def test_assemble_user_object_uses_db_max_budget_when_metadata_is_none( + prometheus_logger, +): + """ + When max_budget is None in request metadata (e.g. stale key cache), + _assemble_user_object must fall back to the value returned by get_user_object + so that _safe_get_remaining_budget does not return +Inf. + """ + db_user = MagicMock() + db_user.max_budget = 500.0 + db_user.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + user_object = await prometheus_logger._assemble_user_object( + user_id="user-abc-123", + spend=120.0, + max_budget=None, # simulates None coming from request metadata + response_cost=0.5, + ) + + assert user_object.max_budget == 500.0, ( + "max_budget should be populated from DB when metadata value is None" + ) + assert user_object.budget_reset_at == datetime(2026, 3, 1, tzinfo=timezone.utc) + + +async def test_assemble_user_object_does_not_override_metadata_max_budget( + prometheus_logger, +): + """ + When max_budget IS present in request metadata, it must not be overridden + by the DB value. + """ + db_user = MagicMock() + db_user.max_budget = 9999.0 + db_user.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + user_object = await prometheus_logger._assemble_user_object( + user_id="user-abc-123", + spend=50.0, + max_budget=100.0, # metadata has a real value + response_cost=1.0, + ) + + assert user_object.max_budget == 100.0, ( + "max_budget from metadata must not be replaced by the DB value" + ) + + +async def test_set_user_budget_metrics_after_api_request_no_inf_when_metadata_budget_none( + prometheus_logger, +): + """ + End-to-end: when user_max_budget is None in request metadata but the user + has a real budget in the DB, the metric must NOT be set to +Inf. + """ + prometheus_logger.litellm_remaining_user_budget_metric = MagicMock() + prometheus_logger.litellm_user_max_budget_metric = MagicMock() + prometheus_logger.litellm_user_budget_remaining_hours_metric = MagicMock() + + db_user = MagicMock() + db_user.max_budget = 500.0 + db_user.budget_reset_at = datetime(2026, 3, 1, tzinfo=timezone.utc) + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + await prometheus_logger._set_user_budget_metrics_after_api_request( + user_id="user-abc-123", + user_spend=120.0, + user_max_budget=None, # simulates stale key cache + response_cost=0.5, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_user_budget_metric.labels().set.call_args + ) + assert set_call_args is not None, "remaining_user_budget_metric.labels().set was not called" + actual_value = set_call_args[0][0] + assert actual_value != float("inf"), ( + f"remaining_user_budget_metric must not be +Inf when user has a real budget; got {actual_value}" + ) + expected = 500.0 - 120.0 - 0.5 + assert abs(actual_value - expected) < 0.01, ( + f"Expected remaining budget ~{expected}, got {actual_value}" + ) + + +async def test_set_user_budget_metrics_after_api_request_inf_when_genuinely_no_budget( + prometheus_logger, +): + """ + When the user genuinely has no budget (max_budget=None in both metadata and + DB), +Inf is the correct value and must be preserved. + """ + prometheus_logger.litellm_remaining_user_budget_metric = MagicMock() + prometheus_logger.litellm_user_max_budget_metric = MagicMock() + prometheus_logger.litellm_user_budget_remaining_hours_metric = MagicMock() + + db_user = MagicMock() + db_user.max_budget = None + db_user.budget_reset_at = None + + with patch("litellm.proxy.auth.auth_checks.get_user_object") as mock_get_user: + mock_get_user.return_value = db_user + await prometheus_logger._set_user_budget_metrics_after_api_request( + user_id="user-no-budget", + user_spend=10.0, + user_max_budget=None, + response_cost=1.0, + ) + + set_call_args = ( + prometheus_logger.litellm_remaining_user_budget_metric.labels().set.call_args + ) + assert set_call_args is not None + actual_value = set_call_args[0][0] + assert actual_value == float("inf"), ( + "remaining_user_budget_metric should be +Inf when user truly has no budget" + ) diff --git a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py index b45cbbd99c0..7e8848be301 100644 --- a/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py +++ b/tests/test_litellm/litellm_core_utils/llm_cost_calc/test_llm_cost_calc_utils.py @@ -9,9 +9,19 @@ import litellm from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import ( StandardBuiltInToolCostTracking, ) +from litellm.llms.gemini.image_generation.cost_calculator import ( + cost_calculator as gemini_image_generation_cost_calculator, +) +from litellm.llms.vertex_ai.image_generation.cost_calculator import ( + cost_calculator as vertex_image_generation_cost_calculator, +) from litellm.types.llms.openai import FileSearchTool, WebSearchOptions from litellm.types.utils import ( CompletionTokensDetailsWrapper, + ImageObject, + ImageResponse, + ImageUsage, + ImageUsageInputTokensDetails, ModelInfo, ModelResponse, PromptTokensDetailsWrapper, @@ -766,7 +776,14 @@ def test_service_tier_fallback_pricing(): assert abs(std_cost[1] - expected_standard_completion) < 1e-10, f"Standard completion cost mismatch: {std_cost[1]} vs {expected_standard_completion}" -def test_gemini_image_generation_cost_with_zero_text_tokens(): +@pytest.mark.parametrize( + "model", + [ + "gemini-3-pro-image-preview", + "gemini-3.1-flash-image-preview", + ], +) +def test_gemini_image_generation_cost_with_zero_text_tokens(model: str): """ Test that image_tokens are correctly costed when text_tokens=0. @@ -779,7 +796,6 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" litellm.model_cost = litellm.get_model_cost_map(url="") - model = "gemini-3-pro-image-preview" custom_llm_provider = "vertex_ai" # Usage from the issue: text_tokens=0, image_tokens=1120, reasoning_tokens=225 @@ -809,9 +825,9 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): # Expected costs: # - text_tokens: 0 * output_cost_per_token = 0 - # - image_tokens: 1120 * output_cost_per_image_token = 1120 * 1.2e-04 = 0.1344 - # - reasoning_tokens: 225 * output_cost_per_token = 225 * 1.2e-05 = 0.0027 - # Total completion: ~0.1371 + # - image_tokens: 1120 * output_cost_per_image_token + # - reasoning_tokens: 225 * output_cost_per_token + # Total completion should include both image + reasoning costs. output_cost_per_image_token = model_cost_map.get("output_cost_per_image_token", 0) output_cost_per_token = model_cost_map.get("output_cost_per_token", 0) @@ -820,18 +836,151 @@ def test_gemini_image_generation_cost_with_zero_text_tokens(): expected_reasoning_cost = 225 * output_cost_per_token # reasoning uses base token cost expected_completion_cost = expected_image_cost + expected_reasoning_cost - # The bug was: all 1345 tokens were treated as text = 1345 * 1.2e-05 = 0.01614 - # Fixed: image_tokens use image pricing = ~0.137 - - assert completion_cost > 0.10, ( - f"Completion cost should be > $0.10 (image tokens are expensive), got ${completion_cost:.6f}. " - f"Bug: tokens may be incorrectly treated as text tokens." + # The bug was: all completion tokens were treated as text tokens only. + bugged_text_only_cost = 1345 * output_cost_per_token + assert completion_cost > bugged_text_only_cost * 2, ( + f"Completion cost should be significantly larger than text-only bugged path. " + f"Expected > {bugged_text_only_cost * 2:.6f}, got {completion_cost:.6f}" ) assert round(completion_cost, 4) == round(expected_completion_cost, 4), ( f"Expected completion cost ${expected_completion_cost:.6f}, got ${completion_cost:.6f}" ) +def test_vertex_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Vertex image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + input_text_tokens = 50 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_vertex_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Vertex image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini-3.1-flash-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="vertex_ai") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = vertex_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + +def test_gemini_image_generation_cost_prefers_token_usage_metadata(): + """ + When usage metadata exists on image responses, Gemini image generation cost + should be calculated from token pricing, not flat output_cost_per_image. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + input_text_tokens = 20 + input_image_tokens = 1120 + output_image_tokens = 1120 + prompt_tokens = input_text_tokens + input_image_tokens + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")], + usage=ImageUsage( + input_tokens=prompt_tokens, + input_tokens_details=ImageUsageInputTokensDetails( + text_tokens=input_text_tokens, + image_tokens=input_image_tokens, + ), + output_tokens=output_image_tokens, + total_tokens=prompt_tokens + output_image_tokens, + ), + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_prompt_cost = prompt_tokens * model_info["input_cost_per_token"] + expected_completion_cost = output_image_tokens * model_info["output_cost_per_image_token"] + expected_total_cost = expected_prompt_cost + expected_completion_cost + + assert round(cost, 10) == round(expected_total_cost, 10) + # Ensure this is not falling back to flat per-image pricing. + assert cost != len(image_response.data) * model_info["output_cost_per_image"] + + +def test_gemini_image_generation_cost_falls_back_to_flat_image_pricing(): + """ + Without usage metadata, Gemini image generation cost should fall back to + output_cost_per_image * number_of_images. + """ + os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" + litellm.model_cost = litellm.get_model_cost_map(url="") + + model = "gemini/gemini-3-pro-image-preview" + model_info = litellm.get_model_info(model=model, custom_llm_provider="gemini") + + image_response = ImageResponse( + data=[ImageObject(b64_json="img1"), ImageObject(b64_json="img2")] + ) + + cost = gemini_image_generation_cost_calculator( + model=model, + image_response=image_response, + ) + + expected_cost = len(image_response.data) * model_info["output_cost_per_image"] + assert round(cost, 10) == round(expected_cost, 10) + + def test_bedrock_anthropic_prompt_caching(): """Test Bedrock Anthropic models with prompt caching return correct costs.""" model = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" diff --git a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py index bcda3c7bfac..11d6bb028d8 100644 --- a/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py +++ b/tests/test_litellm/litellm_core_utils/test_realtime_streaming.py @@ -637,9 +637,10 @@ async def test_realtime_text_input_guardrail_uses_pre_call_mode(): assert streaming._has_realtime_guardrails() is True, ( "pre_call guardrail should be recognized as a realtime guardrail" ) - # pre_call guardrail should NOT trigger the audio/VAD session.update injection - assert streaming._has_audio_transcription_guardrails() is False, ( - "pre_call guardrail should not trigger audio transcription guardrail path" + # pre_call guardrail SHOULD trigger the audio/VAD session.update injection so + # that the LLM does not auto-respond before the guardrail can check the transcript. + assert streaming._has_audio_transcription_guardrails() is True, ( + "pre_call guardrail should trigger audio transcription guardrail path" ) litellm.callbacks = [] # cleanup @@ -711,10 +712,11 @@ async def test_realtime_session_created_injects_session_update_for_audio_guardra @pytest.mark.asyncio -async def test_realtime_session_created_no_injection_for_pre_call_only(): +async def test_realtime_session_created_injects_session_update_for_pre_call_guardrail(): """ - Test that when only a pre_call guardrail is configured (no audio transcription), - session.created does NOT trigger the session.update injection. + Test that when a pre_call guardrail is configured, session.created triggers the + session.update injection (create_response: false) so the LLM does not auto-respond + before the guardrail can check the voice transcript. """ import litellm from litellm.integrations.custom_guardrail import CustomGuardrail @@ -751,14 +753,15 @@ async def test_realtime_session_created_no_injection_for_pre_call_only(): streaming = RealTimeStreaming(client_ws, backend_ws, logging_obj) await streaming.backend_to_client_send_messages() - # No session.update should be injected + # session.update SHOULD be injected so the LLM waits for guardrail approval sent_to_backend = [ json.loads(c.args[0]) for c in backend_ws.send.call_args_list if c.args ] session_updates = [e for e in sent_to_backend if e.get("type") == "session.update"] - assert len(session_updates) == 0, ( - f"pre_call guardrail should NOT inject session.update, got: {sent_to_backend}" + assert len(session_updates) == 1, ( + f"pre_call guardrail should inject session.update to gate audio responses, got: {sent_to_backend}" ) + assert session_updates[0]["session"]["turn_detection"]["create_response"] is False litellm.callbacks = [] # cleanup diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 73ecaa20a2d..153ca5d5aab 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -1187,6 +1187,93 @@ def test_is_chunk_non_empty_with_valid_tool_calls( ) +def test_usage_chunk_after_finish_reason_updates_hidden_params(logging_obj): + """ + Test that provider-reported usage from a post-finish_reason chunk + is surfaced in _hidden_params even when stream_options is NOT set. + + Reproduces issue #20760: OpenRouter sends a final chunk with usage data + after the finish_reason chunk. The hidden_params["usage"] on the last + user-visible chunk was being calculated before this usage chunk arrived, + resulting in zeros. The fix recalculates it in the StopIteration handler + after stream_chunk_builder processes all chunks. + """ + # Simulate OpenRouter's actual streaming pattern: + # 1) content chunk + # 2) finish_reason chunk (content="") + # 3) usage chunk (content="", finish_reason=None, usage={...}) + chunks = [ + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content="Hello"), + finish_reason=None, + ) + ], + ), + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(content=""), + finish_reason="stop", + ) + ], + ), + ModelResponseStream( + id="gen-abc", + object="chat.completion.chunk", + created=1000000, + model="openrouter/openai/gpt-4o-mini", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", content=""), + finish_reason=None, + ) + ], + usage=Usage( + prompt_tokens=20, + completion_tokens=135, + total_tokens=155, + ), + ), + ] + + # Create a CustomStreamWrapper with NO stream_options + wrapper = CustomStreamWrapper( + completion_stream=ModelResponseListIterator(model_responses=chunks), + model="openrouter/openai/gpt-4o-mini", + logging_obj=logging_obj, + custom_llm_provider="openrouter", + stream_options=None, + ) + + # Consume the stream + collected = [] + for chunk in wrapper: + collected.append(chunk) + + # The last user-visible chunk's _hidden_params["usage"] should + # contain the provider-reported values, not zeros. + last_chunk = collected[-1] + hidden_usage = last_chunk._hidden_params.get("usage") + assert hidden_usage is not None, "Expected usage in _hidden_params" + assert hidden_usage.prompt_tokens == 20, ( + f"Expected prompt_tokens=20 from provider, got {hidden_usage.prompt_tokens}" + ) + assert hidden_usage.completion_tokens == 135, ( + f"Expected completion_tokens=135 from provider, got {hidden_usage.completion_tokens}" + ) @pytest.mark.asyncio async def test_custom_stream_wrapper_aclose(): """Test that aclose() delegates to the underlying completion_stream's aclose()""" diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py index 77c74a7847e..c671d9b37b8 100644 --- a/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py @@ -16,13 +16,14 @@ from litellm.types.utils import Delta, ModelResponse, StreamingChoices def test_anthropic_experimental_pass_through_messages_handler(): """ - Test that api key is passed to litellm.completion + Test that api key is passed to litellm.responses for OpenAI models. + OpenAI and Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -32,19 +33,20 @@ def test_anthropic_experimental_pass_through_messages_handler(): ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_and_api_base_and_custom_values(): """ - Test that api key is passed to litellm.completion + Test that api key, api base, and extra kwargs are forwarded to litellm.responses for Azure models. + Azure models are routed directly to the Responses API. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=100, @@ -56,10 +58,10 @@ def test_anthropic_experimental_pass_through_messages_handler_dynamic_api_key_an ) except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - assert mock_completion.call_args.kwargs["api_key"] == "test-api-key" - assert mock_completion.call_args.kwargs["api_base"] == "test-api-base" - assert mock_completion.call_args.kwargs["custom_key"] == "custom_value" + mock_responses.assert_called_once() + assert mock_responses.call_args.kwargs["api_key"] == "test-api-key" + assert mock_responses.call_args.kwargs["api_base"] == "test-api-base" + assert mock_responses.call_args.kwargs["custom_key"] == "custom_value" def test_anthropic_experimental_pass_through_messages_handler_custom_llm_provider(): @@ -143,19 +145,19 @@ async def test_bedrock_converse_budget_tokens_preserved(): assert thinking_param.get("budget_tokens") == 1024, f"thinking.budget_tokens should be 1024, but got {thinking_param.get('budget_tokens')}" -def test_openai_model_with_thinking_converts_to_reasoning_effort(): +def test_openai_model_with_thinking_converts_to_reasoning(): """ - Test that when using a non-Anthropic model (like OpenAI gpt-5.2) with thinking parameter, - the thinking is converted to reasoning_effort and NOT passed as thinking. - - This ensures we don't regress on issue #16052 where non-Anthropic models would fail - with UnsupportedParamsError when thinking was passed directly. + Test that when using an OpenAI model with thinking parameter, the thinking is + converted to a Responses API `reasoning` param (NOT passed as thinking). + + OpenAI models are routed directly to the Responses API, so we verify that + litellm.responses() is called with `reasoning` properly set. """ from litellm.llms.anthropic.experimental_pass_through.messages.handler import ( anthropic_messages_handler, ) - with patch("litellm.completion", return_value="test-response") as mock_completion: + with patch("litellm.responses", return_value="test-response") as mock_responses: try: anthropic_messages_handler( max_tokens=1024, @@ -170,20 +172,22 @@ def test_openai_model_with_thinking_converts_to_reasoning_effort(): except Exception as e: print(f"Error: {e}") - mock_completion.assert_called_once() - - call_kwargs = mock_completion.call_args.kwargs - - # Verify reasoning_effort is set (converted from thinking) - assert "reasoning_effort" in call_kwargs, "reasoning_effort should be passed to completion" + mock_responses.assert_called_once() - # reasoning_effort is transformed into a dict with effort and summary fields - expected_reasoning_effort = {"effort": "minimal", "summary": "detailed"} - assert call_kwargs["reasoning_effort"] == expected_reasoning_effort, \ - f"reasoning_effort should be {expected_reasoning_effort} for budget_tokens=1024, got {call_kwargs.get('reasoning_effort')}" + call_kwargs = mock_responses.call_args.kwargs - # Verify thinking is NOT passed (non-Claude model) - assert "thinking" not in call_kwargs, "thinking should NOT be passed for non-Claude models" + # Verify reasoning is set (converted from thinking) + assert "reasoning" in call_kwargs, "reasoning should be passed to litellm.responses" + + # budget_tokens=1024 -> effort="minimal" (< 2000 threshold) + expected_reasoning = {"effort": "minimal", "summary": "detailed"} + assert call_kwargs["reasoning"] == expected_reasoning, ( + f"reasoning should be {expected_reasoning} for budget_tokens=1024, " + f"got {call_kwargs.get('reasoning')}" + ) + + # Verify thinking is NOT passed directly to the Responses API + assert "thinking" not in call_kwargs, "thinking should NOT be passed directly to litellm.responses" class TestThinkingParameterTransformation: diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py new file mode 100644 index 00000000000..252ba230ff7 --- /dev/null +++ b/tests/test_litellm/llms/anthropic/experimental_pass_through/responses_adapters/test_responses_adapters_transformation.py @@ -0,0 +1,987 @@ +""" +Tests for LiteLLMAnthropicToResponsesAPIAdapter +(litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py) +""" + +import json +import os +import sys +from typing import Any, Dict, List +from unittest.mock import MagicMock + +sys.path.insert(0, os.path.abspath("../../../../../../..")) + +from litellm.llms.anthropic.experimental_pass_through.responses_adapters.transformation import ( + LiteLLMAnthropicToResponsesAPIAdapter, +) +from litellm.types.llms.anthropic import AnthropicMessagesRequest + + +def _make_request(**overrides) -> AnthropicMessagesRequest: + base: dict = { + "model": "openai.gpt-5.1-codex", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 1024, + } + base.update(overrides) + return AnthropicMessagesRequest(**base) + + +_ADAPTER = LiteLLMAnthropicToResponsesAPIAdapter() + + +# --------------------------------------------------------------------------- +# context_management conversion +# --------------------------------------------------------------------------- + + +class TestContextManagementConversion: + """Anthropic dict -> OpenAI array conversion for context_management.""" + + def test_compact_edit_converted_to_array(self): + """compact_20260112 with trigger maps to OpenAI compaction entry.""" + cm = { + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + } + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 150000}] + + def test_compact_edit_without_trigger(self): + """compact_20260112 without a trigger still maps to a compaction entry.""" + cm = {"edits": [{"type": "compact_20260112"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction"}] + + def test_unknown_edit_type_is_dropped(self): + """Anthropic-only edit types (e.g. clear_thinking) are silently dropped.""" + cm = {"edits": [{"type": "clear_thinking_20251015", "keep": "all"}]} + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result is None + + def test_mixed_edits_only_known_types_kept(self): + """Only compact_20260112 is converted; unknown types are dropped.""" + cm = { + "edits": [ + {"type": "clear_thinking_20251015", "keep": "all"}, + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 200000}, + }, + ] + } + result = _ADAPTER.translate_context_management_to_responses_api(cm) + assert result == [{"type": "compaction", "compact_threshold": 200000}] + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_context_management_to_responses_api([]) # type: ignore + assert result is None + + def test_translate_request_includes_context_management(self): + """translate_request converts context_management and sets it on kwargs.""" + req = _make_request( + context_management={ + "edits": [ + { + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 100000}, + } + ] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["context_management"] == [ + {"type": "compaction", "compact_threshold": 100000} + ] + + def test_translate_request_drops_anthropic_only_context_management(self): + """context_management with only unknown edit types is omitted from kwargs.""" + req = _make_request( + context_management={ + "edits": [{"type": "clear_thinking_20251015", "keep": "all"}] + } + ) + kwargs = _ADAPTER.translate_request(req) + assert "context_management" not in kwargs + + +# --------------------------------------------------------------------------- +# structured output via output_config +# --------------------------------------------------------------------------- + + +class TestOutputConfigStructuredOutput: + """output_config.format.json_schema -> OpenAI text.format conversion.""" + + _SCHEMA = { + "type": "object", + "properties": { + "name": {"type": "string"}, + "email": {"type": "string"}, + }, + "required": ["name", "email"], + "additionalProperties": False, + } + + def test_output_config_format_json_schema_converted(self): + """output_config.format.json_schema is converted to OpenAI text.format.""" + req = _make_request( + output_config={"format": {"type": "json_schema", "schema": self._SCHEMA}} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + fmt = kwargs["text"]["format"] + assert fmt["type"] == "json_schema" + assert fmt["schema"] == self._SCHEMA + assert fmt["strict"] is True + assert fmt["name"] == "structured_output" + + def test_output_config_without_format_does_not_set_text(self): + """output_config with only non-format keys doesn't produce text.format.""" + req = _make_request(output_config={"effort": "high"}) + kwargs = _ADAPTER.translate_request(req) + assert "text" not in kwargs + + def test_output_format_still_works(self): + """The original output_format field still takes precedence when present.""" + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA} + ) + kwargs = _ADAPTER.translate_request(req) + assert "text" in kwargs + assert kwargs["text"]["format"]["type"] == "json_schema" + + def test_output_format_takes_precedence_over_output_config(self): + """output_format takes precedence over output_config.format.""" + other_schema = {"type": "object", "properties": {"id": {"type": "integer"}}} + req = _make_request( + output_format={"type": "json_schema", "schema": self._SCHEMA}, + output_config={"format": {"type": "json_schema", "schema": other_schema}}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["text"]["format"]["schema"] == self._SCHEMA + + +# --------------------------------------------------------------------------- +# translate_messages_to_responses_input +# --------------------------------------------------------------------------- + +# Helper: cast plain dicts to the expected type so call sites stay clean. +def _translate_messages(messages: List[Any]) -> List[Dict[str, Any]]: + return _ADAPTER.translate_messages_to_responses_input(messages) # type: ignore[arg-type] + + +class TestTranslateMessagesToResponsesInput: + """Anthropic messages list -> OpenAI Responses API input items.""" + + def test_user_string_content(self): + """Plain string user message becomes a message with input_text.""" + messages = [{"role": "user", "content": "Hello world"}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "Hello world"}], + } + ] + + def test_user_list_text_block(self): + """User message with text content block maps to input_text.""" + messages = [ + { + "role": "user", + "content": [{"type": "text", "text": "What is 2+2?"}], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "user", + "content": [{"type": "input_text", "text": "What is 2+2?"}], + } + ] + + def test_user_multiple_text_blocks(self): + """Multiple text blocks in a user message are all converted.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "First part."}, + {"type": "text", "text": "Second part."}, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_text", "text": "First part."}, + {"type": "input_text", "text": "Second part."}, + ] + + def test_user_base64_image(self): + """User message with base64 image source becomes input_image with data URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": "image/png", + "data": "abc123", + }, + } + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "data:image/png;base64,abc123"} + ] + + def test_user_url_image(self): + """User message with URL image source becomes input_image with the URL.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/img.jpg"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "input_image", "image_url": "https://example.com/img.jpg"} + ] + + def test_user_base64_image_empty_data_skipped(self): + """Base64 image with empty data is skipped (no URL can be formed).""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/jpeg", "data": ""}, + } + ], + } + ] + result = _translate_messages(messages) + # No user_parts -> no message item appended + assert result == [] + + def test_user_tool_result_string_content(self): + """tool_result with string content becomes function_call_output.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_abc", + "content": "42 degrees", + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call_output", + "call_id": "call_abc", + "output": "42 degrees", + } + ] + + def test_user_tool_result_list_content(self): + """tool_result with list of text blocks is joined into a single string.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "call_xyz", + "content": [ + {"type": "text", "text": "Line 1"}, + {"type": "text", "text": "Line 2"}, + ], + } + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "Line 1\nLine 2" + + def test_user_tool_result_null_content(self): + """tool_result with null content becomes empty string output.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": "call_null", "content": None} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["output"] == "" + + def test_assistant_string_content(self): + """Plain string assistant message becomes a message with output_text.""" + messages = [{"role": "assistant", "content": "I can help with that."}] + result = _translate_messages(messages) + assert result == [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "I can help with that."}], + } + ] + + def test_assistant_text_block(self): + """Assistant message with text block maps to output_text.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "text", "text": "Here is the answer."}], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Here is the answer."} + ] + + def test_assistant_tool_use_becomes_function_call(self): + """Assistant tool_use block becomes a top-level function_call item.""" + messages = [ + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_01", + "name": "get_weather", + "input": {"location": "Boston"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [ + { + "type": "function_call", + "call_id": "toolu_01", + "name": "get_weather", + "arguments": json.dumps({"location": "Boston"}), + } + ] + + def test_assistant_thinking_block_becomes_output_text(self): + """Assistant thinking block text is included as output_text.""" + messages = [ + { + "role": "assistant", + "content": [ + {"type": "thinking", "thinking": "Let me reason step by step."} + ], + } + ] + result = _translate_messages(messages) + assert result[0]["content"] == [ + {"type": "output_text", "text": "Let me reason step by step."} + ] + + def test_assistant_empty_thinking_block_skipped(self): + """Assistant thinking block with empty thinking text is skipped.""" + messages = [ + { + "role": "assistant", + "content": [{"type": "thinking", "thinking": ""}], + } + ] + result = _translate_messages(messages) + assert result == [] + + def test_mixed_messages_ordering(self): + """Full multi-turn conversation is converted in order.""" + messages = [ + {"role": "user", "content": "What's the weather?"}, + { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_02", + "name": "get_weather", + "input": {"city": "NYC"}, + } + ], + }, + { + "role": "user", + "content": [ + { + "type": "tool_result", + "tool_use_id": "toolu_02", + "content": "Sunny, 72F", + } + ], + }, + {"role": "assistant", "content": "It's sunny and 72°F in NYC."}, + ] + result = _translate_messages(messages) + types = [item["type"] for item in result] + assert types == ["message", "function_call", "function_call_output", "message"] + + def test_user_text_and_image_mixed(self): + """User message with both text and image produces both parts.""" + messages = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Describe this image:"}, + { + "type": "image", + "source": {"type": "url", "url": "https://example.com/cat.jpg"}, + }, + ], + } + ] + result = _translate_messages(messages) + assert len(result) == 1 + assert result[0]["content"][0] == {"type": "input_text", "text": "Describe this image:"} + assert result[0]["content"][1] == { + "type": "input_image", + "image_url": "https://example.com/cat.jpg", + } + + def test_unknown_image_source_type_skipped(self): + """Image block with unknown source type is silently skipped.""" + messages = [ + { + "role": "user", + "content": [ + { + "type": "image", + "source": {"type": "file_path", "path": "/tmp/img.png"}, + } + ], + } + ] + result = _translate_messages(messages) + assert result == [] + + +# --------------------------------------------------------------------------- +# translate_tools_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolsToResponsesAPI: + """Anthropic tool definitions -> Responses API function tools.""" + + def test_regular_tool_with_description_and_schema(self): + """Standard tool with description and input_schema is converted to function.""" + tools = [ + { + "name": "get_weather", + "description": "Get current weather for a city.", + "input_schema": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [ + { + "type": "function", + "name": "get_weather", + "description": "Get current weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + } + ] + + def test_tool_without_description(self): + """Tool without a description omits the description key.""" + tools = [{"name": "ping", "input_schema": {"type": "object", "properties": {}}}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert result[0]["name"] == "ping" + assert "description" not in result[0] + + def test_tool_without_input_schema(self): + """Tool without input_schema omits the parameters key.""" + tools = [{"name": "no_schema_tool", "description": "Does something."}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result[0]["type"] == "function" + assert "parameters" not in result[0] + + def test_web_search_tool_by_name(self): + """Tool named 'web_search' maps to web_search_preview.""" + tools = [{"name": "web_search", "type": "custom"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_web_search_tool_by_type_prefix(self): + """Tool with type starting with 'web_search' maps to web_search_preview.""" + tools = [{"name": "search", "type": "web_search_20250305"}] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert result == [{"type": "web_search_preview"}] + + def test_multiple_tools_order_preserved(self): + """Multiple tools are converted in order.""" + tools = [ + {"name": "tool_a", "description": "A"}, + {"name": "web_search", "type": "custom"}, + {"name": "tool_b", "description": "B"}, + ] + result = _ADAPTER.translate_tools_to_responses_api(tools) # type: ignore[arg-type] + assert len(result) == 3 + assert result[0]["name"] == "tool_a" + assert result[1] == {"type": "web_search_preview"} + assert result[2]["name"] == "tool_b" + + def test_empty_tools_list(self): + """Empty tools list returns empty list.""" + assert _ADAPTER.translate_tools_to_responses_api([]) == [] + + +# --------------------------------------------------------------------------- +# translate_tool_choice_to_responses_api +# --------------------------------------------------------------------------- + + +class TestTranslateToolChoiceToResponsesAPI: + """Anthropic tool_choice -> Responses API tool_choice.""" + + def test_auto_maps_to_auto(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "auto"}) == { + "type": "auto" + } + + def test_any_maps_to_required(self): + assert _ADAPTER.translate_tool_choice_to_responses_api({"type": "any"}) == { + "type": "required" + } + + def test_specific_tool_maps_to_function(self): + result = _ADAPTER.translate_tool_choice_to_responses_api( + {"type": "tool", "name": "get_weather"} + ) + assert result == {"type": "function", "name": "get_weather"} + + def test_unknown_type_defaults_to_auto(self): + result = _ADAPTER.translate_tool_choice_to_responses_api({"type": "none"}) + assert result == {"type": "auto"} + + +# --------------------------------------------------------------------------- +# translate_thinking_to_reasoning +# --------------------------------------------------------------------------- + + +class TestTranslateThinkingToReasoning: + """Anthropic thinking param -> Responses API reasoning param.""" + + def test_budget_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 10000} + ) + assert result == {"effort": "high", "summary": "detailed"} + + def test_budget_above_threshold_high_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 50000} + ) + assert result is not None + assert result["effort"] == "high" + + def test_budget_medium_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 7500} + ) + assert result == {"effort": "medium", "summary": "detailed"} + + def test_budget_low_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 3000} + ) + assert result == {"effort": "low", "summary": "detailed"} + + def test_budget_minimal_effort(self): + result = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 500} + ) + assert result == {"effort": "minimal", "summary": "detailed"} + + def test_budget_at_exact_thresholds(self): + result_medium = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 5000} + ) + assert result_medium is not None + assert result_medium["effort"] == "medium" + result_low = _ADAPTER.translate_thinking_to_reasoning( + {"type": "enabled", "budget_tokens": 2000} + ) + assert result_low is not None + assert result_low["effort"] == "low" + + def test_disabled_type_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning({"type": "disabled"}) + assert result is None + + def test_non_dict_returns_none(self): + result = _ADAPTER.translate_thinking_to_reasoning("enabled") # type: ignore + assert result is None + + def test_missing_budget_defaults_to_minimal(self): + """Missing budget_tokens defaults to 0, which is < 2000 -> minimal.""" + result = _ADAPTER.translate_thinking_to_reasoning({"type": "enabled"}) + assert result == {"effort": "minimal", "summary": "detailed"} + + +# --------------------------------------------------------------------------- +# translate_request – broader coverage +# --------------------------------------------------------------------------- + + +class TestTranslateRequestBroaderCoverage: + """Full translate_request call: field-by-field mapping verification.""" + + def test_model_and_input_always_present(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + assert "model" in kwargs + assert "input" in kwargs + + def test_system_string_becomes_instructions(self): + req = _make_request(system="You are a helpful assistant.") + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "You are a helpful assistant." + + def test_system_list_of_text_blocks_joined(self): + req = _make_request( + system=[ + {"type": "text", "text": "Be concise."}, + {"type": "text", "text": "Be helpful."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Be concise.\nBe helpful." + + def test_system_list_skips_non_text_blocks(self): + req = _make_request( + system=[ + {"type": "image", "source": {}}, + {"type": "text", "text": "Only text matters."}, + ] + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["instructions"] == "Only text matters." + + def test_max_tokens_mapped_to_max_output_tokens(self): + req = _make_request(max_tokens=512) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["max_output_tokens"] == 512 + + def test_temperature_passed_through(self): + req = _make_request(temperature=0.7) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["temperature"] == 0.7 + + def test_top_p_passed_through(self): + req = _make_request(top_p=0.9) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["top_p"] == 0.9 + + def test_tools_translated(self): + req = _make_request( + tools=[{"name": "calculator", "description": "Does math.", "input_schema": {}}] + ) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["tools"]) == 1 + assert kwargs["tools"][0]["name"] == "calculator" + + def test_tool_choice_translated(self): + req = _make_request( + tools=[{"name": "do_thing"}], + tool_choice={"type": "tool", "name": "do_thing"}, + ) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["tool_choice"] == {"type": "function", "name": "do_thing"} + + def test_thinking_translated_to_reasoning(self): + req = _make_request(thinking={"type": "enabled", "budget_tokens": 12000}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["reasoning"] == {"effort": "high", "summary": "detailed"} + + def test_disabled_thinking_not_included_in_kwargs(self): + req = _make_request(thinking={"type": "disabled"}) + kwargs = _ADAPTER.translate_request(req) + assert "reasoning" not in kwargs + + def test_metadata_user_id_mapped_to_user(self): + req = _make_request(metadata={"user_id": "user-42"}) + kwargs = _ADAPTER.translate_request(req) + assert kwargs["user"] == "user-42" + + def test_metadata_user_id_truncated_to_64_chars(self): + long_id = "x" * 100 + req = _make_request(metadata={"user_id": long_id}) + kwargs = _ADAPTER.translate_request(req) + assert len(kwargs["user"]) == 64 + + def test_no_optional_fields_does_not_add_spurious_keys(self): + req = _make_request() + kwargs = _ADAPTER.translate_request(req) + for key in ("instructions", "temperature", "top_p", "tools", "tool_choice", + "reasoning", "text", "context_management", "user"): + assert key not in kwargs, f"unexpected key: {key}" + + +# --------------------------------------------------------------------------- +# translate_response +# --------------------------------------------------------------------------- + + +def _make_mock_response( + output: list, + status: str = "completed", + response_id: str = "resp_001", + model: str = "gpt-4o", + input_tokens: int = 100, + output_tokens: int = 50, +) -> MagicMock: + """Build a minimal mock ResponsesAPIResponse.""" + usage = MagicMock() + usage.input_tokens = input_tokens + usage.output_tokens = output_tokens + + resp = MagicMock() + resp.id = response_id + resp.model = model + resp.status = status + resp.output = output + resp.usage = usage + return resp + + +def _make_output_message(texts: List[str]) -> MagicMock: + """Build a mock ResponseOutputMessage with output_text parts.""" + from openai.types.responses import ResponseOutputMessage # type: ignore[import] + + parts = [] + for t in texts: + part = MagicMock() + part.type = "output_text" + part.text = t + parts.append(part) + + msg = MagicMock(spec=ResponseOutputMessage) + msg.content = parts + return msg + + +def _make_function_call_item( + call_id: str, name: str, arguments: str +) -> MagicMock: + """Build a mock ResponseFunctionToolCall.""" + from openai.types.responses import ResponseFunctionToolCall # type: ignore[import] + + item = MagicMock(spec=ResponseFunctionToolCall) + item.call_id = call_id + item.id = call_id + item.name = name + item.arguments = arguments + return item + + +def _make_reasoning_item(summaries: List[str]) -> MagicMock: + """Build a mock ResponseReasoningItem.""" + from openai.types.responses import ResponseReasoningItem # type: ignore[import] + + summary_mocks = [] + for text in summaries: + s = MagicMock() + s.text = text + summary_mocks.append(s) + + item = MagicMock(spec=ResponseReasoningItem) + item.summary = summary_mocks + return item + + +class TestTranslateResponse: + """Responses API -> AnthropicMessagesResponse conversion.""" + + def test_output_text_message_becomes_text_block(self): + """ResponseOutputMessage with output_text parts -> Anthropic text content.""" + response = _make_mock_response(output=[_make_output_message(["Hello!"])]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Hello!" + + def test_multiple_text_parts(self): + """Multiple output_text parts become multiple text content blocks.""" + response = _make_mock_response( + output=[_make_output_message(["Part 1", "Part 2"])] + ) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 2 + assert result["content"][0]["text"] == "Part 1" + assert result["content"][1]["text"] == "Part 2" + + def test_function_call_becomes_tool_use(self): + """ResponseFunctionToolCall -> Anthropic tool_use content block.""" + fc = _make_function_call_item("call_99", "get_weather", '{"city": "NYC"}') + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + block = result["content"][0] + assert block["type"] == "tool_use" + assert block["id"] == "call_99" + assert block["name"] == "get_weather" + assert block["input"] == {"city": "NYC"} + + def test_function_call_sets_stop_reason_tool_use(self): + """Presence of a function_call sets stop_reason to 'tool_use'.""" + fc = _make_function_call_item("call_1", "tool_a", "{}") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "tool_use" + + def test_text_only_stop_reason_end_turn(self): + """Text-only response has stop_reason 'end_turn'.""" + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "end_turn" + + def test_incomplete_status_sets_max_tokens(self): + """status='incomplete' overrides stop_reason to 'max_tokens'.""" + response = _make_mock_response( + output=[_make_output_message(["Truncated..."])], + status="incomplete", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["stop_reason"] == "max_tokens" + + def test_reasoning_item_becomes_thinking_block(self): + """ResponseReasoningItem summaries -> Anthropic thinking content blocks.""" + reasoning = _make_reasoning_item(["Step 1: analyze. Step 2: conclude."]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert len(result["content"]) == 1 + assert result["content"][0]["type"] == "thinking" + assert "Step 1" in result["content"][0]["thinking"] + + def test_empty_reasoning_summary_skipped(self): + """Reasoning item with empty text summary is not added to content.""" + reasoning = _make_reasoning_item([""]) + response = _make_mock_response(output=[reasoning]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + + def test_usage_mapped_correctly(self): + """Input/output tokens from ResponseAPIUsage are mapped to AnthropicUsage.""" + response = _make_mock_response( + output=[_make_output_message(["OK"])], + input_tokens=200, + output_tokens=75, + ) + result: Any = _ADAPTER.translate_response(response) + assert result["usage"]["input_tokens"] == 200 + assert result["usage"]["output_tokens"] == 75 + + def test_model_and_id_preserved(self): + """Model and response ID from the Responses API are forwarded.""" + response = _make_mock_response( + output=[_make_output_message(["Hi"])], + response_id="resp_xyz", + model="gpt-4-turbo", + ) + result: Any = _ADAPTER.translate_response(response) + assert result["id"] == "resp_xyz" + assert result["model"] == "gpt-4-turbo" + + def test_role_is_always_assistant(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["role"] == "assistant" + + def test_type_is_always_message(self): + response = _make_mock_response(output=[_make_output_message(["Hi"])]) + result: Any = _ADAPTER.translate_response(response) + assert result["type"] == "message" + + def test_empty_output_list(self): + """Empty output list produces empty content with 'end_turn' stop reason.""" + response = _make_mock_response(output=[]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"] == [] + assert result["stop_reason"] == "end_turn" + + def test_function_call_with_invalid_json_arguments(self): + """Invalid JSON in function_call arguments falls back to empty dict.""" + fc = _make_function_call_item("call_bad", "broken_tool", "not-valid-json") + response = _make_mock_response(output=[fc]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["input"] == {} + + def test_dict_output_message_item(self): + """Dict-shaped output message (type=message) is also handled.""" + output_item = { + "type": "message", + "content": [{"type": "output_text", "text": "Dict-based response"}], + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "text" + assert result["content"][0]["text"] == "Dict-based response" + + def test_dict_function_call_item(self): + """Dict-shaped function_call item is converted to tool_use block.""" + output_item = { + "type": "function_call", + "call_id": "call_dict_1", + "name": "search", + "arguments": '{"query": "cats"}', + } + response = _make_mock_response(output=[output_item]) + result: Any = _ADAPTER.translate_response(response) + assert result["content"][0]["type"] == "tool_use" + assert result["content"][0]["name"] == "search" + assert result["content"][0]["input"] == {"query": "cats"} + assert result["stop_reason"] == "tool_use" + + def test_mixed_reasoning_text_and_tool_use(self): + """Reasoning + text + tool_use in one response all convert correctly.""" + reasoning = _make_reasoning_item(["Thinking..."]) + text_msg = _make_output_message(["Here is my answer."]) + fc = _make_function_call_item("call_mix", "lookup", '{"id": 1}') + response = _make_mock_response(output=[reasoning, text_msg, fc]) + result: Any = _ADAPTER.translate_response(response) + types = [b["type"] for b in result["content"]] + assert "thinking" in types + assert "text" in types + assert "tool_use" in types + assert result["stop_reason"] == "tool_use" diff --git a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py index 26395597166..f6d3d3c12f7 100644 --- a/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py +++ b/tests/test_litellm/llms/bedrock/chat/test_converse_transformation.py @@ -2616,11 +2616,11 @@ def test_empty_assistant_message_handling(): empty or whitespace-only content with a placeholder to prevent AWS Bedrock Converse API 400 Bad Request errors. """ - # Import the litellm module that factory.py uses to ensure we patch the correct reference - import litellm.litellm_core_utils.prompt_templates.factory as factory_module from litellm.litellm_core_utils.prompt_templates.factory import ( _bedrock_converse_messages_pt, ) + # Import the litellm module that factory.py uses to ensure we patch the correct reference + import litellm.litellm_core_utils.prompt_templates.factory as factory_module # Test case 1: Empty string content - test with modify_params=True to prevent merging messages = [ @@ -3135,12 +3135,7 @@ def test_native_structured_output_no_fake_stream(): def test_transform_request_with_output_config(): """Test that outputConfig flows through _transform_request_helper into the final request.""" - from litellm.types.llms.bedrock import ( - JsonSchemaDefinition, - OutputConfigBlock, - OutputFormat, - OutputFormatStructure, - ) + from litellm.types.llms.bedrock import OutputConfigBlock, OutputFormat, OutputFormatStructure, JsonSchemaDefinition config = AmazonConverseConfig() @@ -3382,59 +3377,78 @@ def test_output_config_applies_additional_properties(): -def test_parallel_tool_calls_in_request_transformation(): - """Test that parallel_tool_calls is correctly placed in additionalModelRequestFields after full transformation""" - config = AmazonConverseConfig() - - messages = [ - {"role": "user", "content": "What's the weather in SF and NYC?"} - ] - - non_default_params = { - "parallel_tool_calls": False, - "tools": [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The location to get weather for" - } - }, - "required": ["location"] +_TOOL_PARAM = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Get the weather", + "parameters": { + "type": "object", + "properties": { + "location": { + "type": "string", + "description": "The location to get weather for", } - } - } - ], - "max_tokens": 100, + }, + "required": ["location"], + }, + }, } - +] + + +def test_parallel_tool_calls_newer_model_adds_disable_flag(): + """Newer Claude models (4.5+) should get disable_parallel_tool_use in additionalModelRequestFields.""" + config = AmazonConverseConfig() + model = "anthropic.claude-sonnet-4-5-20250929-v1:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + optional_params = config.map_openai_params( - non_default_params=non_default_params, + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, optional_params={}, - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, drop_params=False, ) - - # Transform the request + request_data = config.transform_request( - model="anthropic.claude-sonnet-4-5-v2:0", + model=model, messages=messages, optional_params=optional_params, litellm_params={}, headers={}, ) - - # Verify the structure + assert "additionalModelRequestFields" in request_data assert "tool_choice" in request_data["additionalModelRequestFields"] - assert "disable_parallel_tool_use" in request_data["additionalModelRequestFields"]["tool_choice"] assert request_data["additionalModelRequestFields"]["tool_choice"]["disable_parallel_tool_use"] is True + assert "parallel_tool_calls" not in request_data["additionalModelRequestFields"] + + +def test_parallel_tool_calls_older_model_drops_disable_flag(): + """Older Claude models (pre-4.5) must NOT receive disable_parallel_tool_use — Bedrock rejects it.""" + config = AmazonConverseConfig() + model = "anthropic.claude-3-5-sonnet-20241022-v2:0" + messages = [{"role": "user", "content": "What's the weather in SF and NYC?"}] + + optional_params = config.map_openai_params( + non_default_params={"parallel_tool_calls": False, "tools": _TOOL_PARAM}, + optional_params={}, + model=model, + drop_params=False, + ) + + request_data = config.transform_request( + model=model, + messages=messages, + optional_params=optional_params, + litellm_params={}, + headers={}, + ) + + additional = request_data.get("additionalModelRequestFields", {}) + assert "tool_choice" not in additional + assert "parallel_tool_calls" not in additional class TestBedrockMinThinkingBudgetTokens: diff --git a/tests/test_litellm/ocr/__init__.py b/tests/test_litellm/ocr/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/ocr/test_ocr_file_input.py b/tests/test_litellm/ocr/test_ocr_file_input.py new file mode 100644 index 00000000000..492253e2f11 --- /dev/null +++ b/tests/test_litellm/ocr/test_ocr_file_input.py @@ -0,0 +1,464 @@ +""" +Tests for OCR file input support. + +Tests that: +1. The SDK document parameter with type="file" correctly converts file paths, + file objects, and raw bytes to base64 data URIs before sending to providers. +2. The proxy _build_document_from_upload helper correctly handles uploaded file bytes. +3. The proxy rejects type="file" documents received via JSON (security guard). +4. The proxy returns user-friendly errors for invalid JSON bodies. +""" +import base64 +import os +import tempfile +from io import BytesIO +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import orjson +import pytest + +from litellm.ocr.main import convert_file_document_to_url_document, get_mime_type + + +class TestGetMimeType: + def test_should_detect_pdf_mime_type(self): + assert get_mime_type("document.pdf") == "application/pdf" + + def test_should_detect_png_mime_type(self): + assert get_mime_type("image.png") == "image/png" + + def test_should_detect_jpg_mime_type(self): + assert get_mime_type("photo.jpg") == "image/jpeg" + + def test_should_detect_jpeg_mime_type(self): + assert get_mime_type("photo.jpeg") == "image/jpeg" + + def test_should_detect_gif_mime_type(self): + assert get_mime_type("animation.gif") == "image/gif" + + def test_should_detect_webp_mime_type(self): + assert get_mime_type("image.webp") == "image/webp" + + def test_should_detect_tiff_mime_type(self): + assert get_mime_type("scan.tiff") == "image/tiff" + + def test_should_detect_tif_mime_type(self): + assert get_mime_type("scan.tif") == "image/tiff" + + def test_should_detect_bmp_mime_type(self): + assert get_mime_type("bitmap.bmp") == "image/bmp" + + def test_should_be_case_insensitive(self): + assert get_mime_type("DOCUMENT.PDF") == "application/pdf" + assert get_mime_type("IMAGE.PNG") == "image/png" + + def test_should_fallback_for_unknown_extension(self): + result = get_mime_type("file.xyz123") + assert isinstance(result, str) + + +class TestConvertFileDocumentToUrlDocument: + def test_should_convert_pdf_file_path_to_document_url(self): + """File path to a PDF should produce type=document_url with base64 data URI.""" + pdf_content = b"%PDF-1.4 test content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(pdf_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == pdf_content + finally: + os.unlink(tmp_path) + + def test_should_convert_image_file_path_to_image_url(self): + """File path to a PNG image should produce type=image_url with base64 data URI.""" + png_content = b"\x89PNG\r\n\x1a\n fake png content" + + with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as f: + f.write(png_content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + b64_data = result["image_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == png_content + finally: + os.unlink(tmp_path) + + def test_should_convert_pathlib_path(self): + """pathlib.Path objects should work the same as string paths.""" + content = b"test pdf content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = Path(f.name) + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + finally: + os.unlink(str(tmp_path)) + + def test_should_convert_raw_bytes(self): + """Raw bytes should be converted using a fallback MIME type.""" + content = b"raw bytes content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_convert_raw_bytes_with_explicit_mime_type(self): + """Raw bytes with explicit mime_type should use the specified MIME type.""" + content = b"raw pdf content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "application/pdf"} + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_convert_raw_bytes_with_image_mime_type(self): + """Raw bytes with an image MIME type should produce type=image_url.""" + content = b"raw image content" + + result = convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "image/jpeg"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_convert_file_like_object(self): + """BytesIO and other file-like objects should be supported.""" + content = b"file-like content" + file_obj = BytesIO(content) + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "document_url" + assert "base64," in result["document_url"] + + def test_should_convert_file_like_object_with_name(self): + """File-like objects with a .name attribute should detect MIME from the name.""" + content = b"file-like png content" + file_obj = BytesIO(content) + file_obj.name = "test_image.png" + + result = convert_file_document_to_url_document( + {"type": "file", "file": file_obj} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_raise_error_for_missing_file_field(self): + """Missing 'file' field should raise ValueError.""" + with pytest.raises(ValueError, match="must include a 'file' field"): + convert_file_document_to_url_document({"type": "file"}) + + def test_should_raise_error_for_nonexistent_file_path(self): + """Non-existent file path should raise FileNotFoundError.""" + with pytest.raises(FileNotFoundError, match="File not found"): + convert_file_document_to_url_document( + {"type": "file", "file": "/nonexistent/path/to/file.pdf"} + ) + + def test_should_raise_error_for_empty_file(self): + """Empty file should raise ValueError.""" + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + tmp_path = f.name + + try: + with pytest.raises(ValueError, match="File is empty"): + convert_file_document_to_url_document( + {"type": "file", "file": tmp_path} + ) + finally: + os.unlink(tmp_path) + + def test_should_raise_error_for_unsupported_type(self): + """Unsupported file input types should raise ValueError.""" + with pytest.raises(ValueError, match="Unsupported file input type"): + convert_file_document_to_url_document({"type": "file", "file": 12345}) + + def test_should_raise_error_for_invalid_mime_type(self): + """MIME types with special characters should be rejected.""" + content = b"some content" + with pytest.raises(ValueError, match="Invalid MIME type"): + convert_file_document_to_url_document( + {"type": "file", "file": content, "mime_type": "text/html; charset=utf-8\nX-Injected: true"} + ) + + def test_should_override_mime_type_for_file_path(self): + """Explicit mime_type should override auto-detection from extension.""" + content = b"some content" + + with tempfile.NamedTemporaryFile(suffix=".pdf", delete=False) as f: + f.write(content) + f.flush() + tmp_path = f.name + + try: + result = convert_file_document_to_url_document( + {"type": "file", "file": tmp_path, "mime_type": "image/png"} + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + finally: + os.unlink(tmp_path) + + +class TestBuildDocumentFromUpload: + """Test the proxy endpoint's file upload to document conversion helper.""" + + @pytest.fixture(autouse=True) + def _import_helper(self): + """Import the proxy helper, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _build_document_from_upload, + ) + + self._build = _build_document_from_upload + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + def test_should_build_document_url_for_pdf(self): + content = b"%PDF-1.4 test content" + + result = self._build( + file_content=content, + filename="document.pdf", + content_type="application/pdf", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_build_image_url_for_png(self): + content = b"\x89PNG fake png" + + result = self._build( + file_content=content, + filename="screenshot.png", + content_type="image/png", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_build_image_url_for_jpeg(self): + content = b"\xff\xd8\xff fake jpeg" + + result = self._build( + file_content=content, + filename="photo.jpg", + content_type="image/jpeg", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/jpeg;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_octet_stream(self): + content = b"pdf content" + + result = self._build( + file_content=content, + filename="report.pdf", + content_type="application/octet-stream", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_detect_mime_from_filename_when_content_type_is_none(self): + content = b"png content" + + result = self._build( + file_content=content, + filename="image.png", + content_type=None, + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + def test_should_fallback_to_octet_stream_for_unknown(self): + content = b"unknown content" + + result = self._build( + file_content=content, + filename=None, + content_type=None, + ) + + assert result["type"] == "document_url" + assert "application/octet-stream" in result["document_url"] + + def test_should_preserve_base64_content_correctly(self): + content = b"Hello, World! \x00\x01\x02\xff" + + result = self._build( + file_content=content, + filename="test.pdf", + content_type="application/pdf", + ) + + b64_data = result["document_url"].split(";base64,")[1] + assert base64.b64decode(b64_data) == content + + def test_should_strip_mime_parameters_from_content_type(self): + """Content-Type with parameters (e.g. charset) should be stripped to the base MIME type.""" + content = b"%PDF-1.4 test" + + result = self._build( + file_content=content, + filename="doc.pdf", + content_type="application/pdf; charset=utf-8", + ) + + assert result["type"] == "document_url" + assert result["document_url"].startswith("data:application/pdf;base64,") + + def test_should_strip_mime_parameters_with_multiple_params(self): + """Content-Type with multiple parameters should still be stripped correctly.""" + content = b"image data" + + result = self._build( + file_content=content, + filename="img.png", + content_type="image/png; charset=utf-8; boundary=something", + ) + + assert result["type"] == "image_url" + assert result["image_url"].startswith("data:image/png;base64,") + + +class TestProxySecurityGuard: + """Test that the proxy rejects type='file' documents in JSON requests + and that multipart form fields cannot override the constructed document.""" + + @pytest.fixture(autouse=True) + def _import_helpers(self): + """Import the proxy helpers, skip if proxy deps aren't installed.""" + try: + from litellm.proxy.ocr_endpoints.endpoints import ( + _parse_multipart_form, + _parse_ocr_request, + ) + + self._parse = _parse_ocr_request + self._parse_multipart = _parse_multipart_form + except ImportError: + pytest.skip("Proxy dependencies (fastapi/orjson) not installed") + + @pytest.mark.asyncio + async def test_should_reject_file_type_document_in_json_body(self): + """type='file' in a JSON body must be rejected to prevent server-side file reads.""" + body = orjson.dumps( + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "file", "file": "/etc/passwd"}, + } + ) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + with pytest.raises(ValueError, match="not supported through the JSON API"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_accept_document_url_type_in_json_body(self): + """type='document_url' in a JSON body should pass through normally.""" + expected = { + "model": "mistral/mistral-ocr-latest", + "document": { + "type": "document_url", + "document_url": "https://example.com/doc.pdf", + }, + } + body = orjson.dumps(expected) + + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=body) + mock_request._form = None + + result = await self._parse(mock_request) + assert result["document"]["type"] == "document_url" + + @pytest.mark.asyncio + async def test_should_raise_on_invalid_json_body(self): + """Invalid JSON should produce a user-friendly ValueError.""" + mock_request = MagicMock() + mock_request.headers = {"content-type": "application/json"} + mock_request.body = AsyncMock(return_value=b"not valid json{{{") + mock_request._form = None + + with pytest.raises(ValueError, match="Invalid JSON in request body"): + await self._parse(mock_request) + + @pytest.mark.asyncio + async def test_should_ignore_document_form_field_injection(self): + """A 'document' form field must not override the document built from the uploaded file.""" + from starlette.datastructures import UploadFile + + file_content = b"%PDF-1.4 legit content" + upload = UploadFile(filename="legit.pdf", file=BytesIO(file_content)) + + injected = '{"type": "file", "file": "/etc/passwd"}' + + mock_form = { + "file": upload, + "model": "mistral/mistral-ocr-latest", + "document": injected, + } + + mock_request = MagicMock() + mock_request.headers = {"content-type": "multipart/form-data; boundary=---"} + mock_request.form = AsyncMock(return_value=mock_form) + + result = await self._parse_multipart(mock_request) + + assert result["document"]["type"] == "document_url" + assert result["document"]["document_url"].startswith("data:application/pdf;base64,") + assert result["model"] == "mistral/mistral-ocr-latest" diff --git a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py index 9b6e0631762..32fd0750de8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_access_group_endpoints.py @@ -99,10 +99,12 @@ def client_and_mocks(monkeypatch): mock_team_table = MagicMock() mock_team_table.find_many = AsyncMock(return_value=[]) + mock_team_table.find_unique = AsyncMock(return_value=None) mock_team_table.update = AsyncMock(return_value=None) mock_key_table = MagicMock() mock_key_table.find_many = AsyncMock(return_value=[]) + mock_key_table.find_unique = AsyncMock(return_value=None) mock_key_table.update = AsyncMock(return_value=None) @asynccontextmanager @@ -570,11 +572,13 @@ def test_delete_access_group_cleans_up_teams_and_keys(client_and_mocks): team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "key-token-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) resp = client.delete("/v1/access_group/ag-to-delete") assert resp.status_code == 204 @@ -669,11 +673,13 @@ def test_delete_access_group_patches_cached_team_and_key( team_with_group.team_id = "team-1" team_with_group.access_group_ids = ["ag-to-delete", "ag-keep"] mock_team_table.find_many = AsyncMock(return_value=[team_with_group]) + mock_team_table.find_unique = AsyncMock(return_value=team_with_group) key_with_group = MagicMock() key_with_group.token = "hashed-key-1" key_with_group.access_group_ids = ["ag-to-delete"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # Build cached team object (returned from proxy_logging dual cache) if team_cache_group_ids is not None: @@ -762,6 +768,7 @@ def test_delete_access_group_patches_key_cached_as_dict(client_and_mocks): key_with_group.token = "hashed-key-dict" key_with_group.access_group_ids = ["ag-to-delete", "ag-other"] mock_key_table.find_many = AsyncMock(return_value=[key_with_group]) + mock_key_table.find_unique = AsyncMock(return_value=key_with_group) # No team in cache mock_proxy_logging.internal_usage_cache.dual_cache.async_get_cache = AsyncMock( @@ -882,3 +889,304 @@ def test_record_to_access_group_table(): assert result.access_group_name == "unit-test-group" assert result.access_model_names == ["gpt-4", "claude-3"] assert result.access_agent_ids == ["agent-1"] + + +# --------------------------------------------------------------------------- +# Sync tests: CREATE +# --------------------------------------------------------------------------- + + +def test_create_access_group_syncs_assigned_teams(client_and_mocks): + """Create adds access_group_id to each assigned team's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-1"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-1"} + # The newly created access group id ("ag-new") should be in the updated list + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_syncs_assigned_keys(client_and_mocks): + """Create adds access_group_id to each assigned key's access_group_ids in DB.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + key_record = MagicMock() + key_record.token = "hashed-token-1" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_key_ids": ["hashed-token-1"]}, + ) + assert resp.status_code == 201 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "hashed-token-1"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "hashed-token-1"} + assert "ag-new" in call_kwargs["data"]["access_group_ids"] + + +def test_create_access_group_skips_sync_for_nonexistent_team(client_and_mocks): + """Create skips updating a team that doesn't exist in DB.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + mock_team_table.find_unique = AsyncMock(return_value=None) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["nonexistent-team"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +def test_create_access_group_idempotent_team_sync(client_and_mocks): + """Create skips updating a team that already has the access_group_id.""" + client, mock_prisma, _, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + team_record = MagicMock() + team_record.team_id = "team-1" + team_record.access_group_ids = ["ag-new"] # already synced + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.post( + "/v1/access_group", + json={"access_group_name": "new-group", "assigned_team_ids": ["team-1"]}, + ) + assert resp.status_code == 201 + mock_team_table.update.assert_not_awaited() + + +# --------------------------------------------------------------------------- +# Sync tests: UPDATE +# --------------------------------------------------------------------------- + + +def test_update_access_group_syncs_added_teams(client_and_mocks): + """Update adds access_group_id to newly assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-existing"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_record = MagicMock() + team_record.team_id = "team-new" + team_record.access_group_ids = [] + mock_team_table.find_unique = AsyncMock(return_value=team_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-existing", "team-new"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-new"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-new"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_teams(client_and_mocks): + """Update removes access_group_id from de-assigned teams.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-keep", "team-remove"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + team_to_remove = MagicMock() + team_to_remove.team_id = "team-remove" + team_to_remove.access_group_ids = ["ag-update"] + mock_team_table.find_unique = AsyncMock(return_value=team_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": ["team-keep"]}, + ) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-remove"}) + mock_team_table.update.assert_awaited_once() + call_kwargs = mock_team_table.update.call_args.kwargs + assert call_kwargs["where"] == {"team_id": "team-remove"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_no_team_sync_when_ids_not_in_payload(client_and_mocks): + """Update does not sync teams when assigned_team_ids is absent from the payload.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_team_ids=["team-1"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + resp = client.put("/v1/access_group/ag-update", json={"description": "new desc"}) + assert resp.status_code == 200 + + mock_team_table.find_unique.assert_not_awaited() + mock_team_table.update.assert_not_awaited() + + +def test_update_access_group_syncs_added_keys(client_and_mocks): + """Update adds access_group_id to newly assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["old-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_record = MagicMock() + key_record.token = "new-token" + key_record.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=key_record) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["old-token", "new-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "new-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "new-token"} + assert "ag-update" in call_kwargs["data"]["access_group_ids"] + + +def test_update_access_group_syncs_removed_keys(client_and_mocks): + """Update removes access_group_id from de-assigned keys.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-update", assigned_key_ids=["keep-token", "remove-token"] + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + key_to_remove = MagicMock() + key_to_remove.token = "remove-token" + key_to_remove.access_group_ids = ["ag-update"] + mock_key_table.find_unique = AsyncMock(return_value=key_to_remove) + + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_key_ids": ["keep-token"]}, + ) + assert resp.status_code == 200 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "remove-token"}) + mock_key_table.update.assert_awaited_once() + call_kwargs = mock_key_table.update.call_args.kwargs + assert call_kwargs["where"] == {"token": "remove-token"} + assert "ag-update" not in call_kwargs["data"]["access_group_ids"] + + +# --------------------------------------------------------------------------- +# Sync tests: DELETE (out-of-sync data handling) +# --------------------------------------------------------------------------- + + +def test_delete_access_group_handles_out_of_sync_assigned_teams(client_and_mocks): + """Delete includes teams from assigned_team_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_team_table = mock_prisma.db.litellm_teamtable + + # Access group has assigned_team_ids but the team's access_group_ids is not synced + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_team_ids=["team-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + # hasSome query finds nothing (team's own access_group_ids is out of sync) + mock_team_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_team = MagicMock() + out_of_sync_team.team_id = "team-out-of-sync" + out_of_sync_team.access_group_ids = [] # already clean, no update needed + mock_team_table.find_unique = AsyncMock(return_value=out_of_sync_team) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + # find_unique is called for the out-of-sync team (included via union with assigned_team_ids) + mock_team_table.find_unique.assert_awaited_once_with(where={"team_id": "team-out-of-sync"}) + # No update needed since team's access_group_ids doesn't contain "ag-to-delete" + mock_team_table.update.assert_not_awaited() + + +def test_delete_access_group_handles_out_of_sync_assigned_keys(client_and_mocks): + """Delete includes keys from assigned_key_ids even when not found by hasSome query.""" + client, mock_prisma, mock_access_group_table, mock_cache, mock_proxy_logging = client_and_mocks + mock_key_table = mock_prisma.db.litellm_verificationtoken + + existing = _make_access_group_record( + access_group_id="ag-to-delete", + assigned_key_ids=["token-out-of-sync"], + ) + mock_access_group_table.find_unique = AsyncMock(return_value=existing) + + mock_key_table.find_many = AsyncMock(return_value=[]) + + out_of_sync_key = MagicMock() + out_of_sync_key.token = "token-out-of-sync" + out_of_sync_key.access_group_ids = [] + mock_key_table.find_unique = AsyncMock(return_value=out_of_sync_key) + + resp = client.delete("/v1/access_group/ag-to-delete") + assert resp.status_code == 204 + + mock_key_table.find_unique.assert_awaited_once_with(where={"token": "token-out-of-sync"}) + mock_key_table.update.assert_not_awaited() + + +def test_update_access_group_null_assigned_ids_treated_as_empty(client_and_mocks): + """Update with explicit null for assigned_*_ids clears the list and writes [] to DB.""" + client, _, mock_table, *_ = client_and_mocks + + existing = _make_access_group_record( + access_group_id="ag-update", + assigned_team_ids=["team-1"], + assigned_key_ids=["key-1"], + ) + mock_table.find_unique = AsyncMock(return_value=existing) + + # Sending null for assigned_team_ids and assigned_key_ids + resp = client.put( + "/v1/access_group/ag-update", + json={"assigned_team_ids": None, "assigned_key_ids": None}, + ) + assert resp.status_code == 200 + + # Verify the DB update was called with [] (not null) for list fields + update_call_kwargs = mock_table.update.call_args.kwargs + assert update_call_kwargs["data"]["assigned_team_ids"] == [] + assert update_call_kwargs["data"]["assigned_key_ids"] == [] diff --git a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py index 0b21bc2636c..53c98c8c400 100644 --- a/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/test_litellm/proxy/public_endpoints/test_public_endpoints.py @@ -87,54 +87,6 @@ def test_get_litellm_model_cost_map_returns_cost_map(): assert "input_cost_per_token" in sample_model_data or "output_cost_per_token" in sample_model_data -def test_get_provider_supported_endpoints(): - """Test /public/supported_endpoints returns correct structure with endpoints and providers.""" - app = FastAPI() - app.include_router(router) - client = TestClient(app) - - response = client.get("/public/supported_endpoints") - - assert response.status_code == 200 - data = response.json() - - # Check top-level structure - assert "endpoints" in data - assert "providers" in data - assert isinstance(data["endpoints"], list) - assert isinstance(data["providers"], list) - - # Verify endpoints structure - assert len(data["endpoints"]) > 0 - for endpoint in data["endpoints"]: - assert "key" in endpoint - assert "display_name" in endpoint - assert "endpoint" in endpoint - assert isinstance(endpoint["key"], str) - assert isinstance(endpoint["display_name"], str) - assert endpoint["endpoint"].startswith("/") - - # Verify providers structure - assert len(data["providers"]) > 0 - for provider in data["providers"]: - assert "slug" in provider - assert "display_name" in provider - assert "supported" in provider - assert isinstance(provider["slug"], str) - assert isinstance(provider["display_name"], str) - assert isinstance(provider["supported"], list) - - # Verify some expected endpoints exist - endpoint_keys = {e["key"] for e in data["endpoints"]} - assert "chat_completions" in endpoint_keys - assert "embeddings" in endpoint_keys - assert "responses" in endpoint_keys - - # Verify some expected providers exist - provider_slugs = {p["slug"] for p in data["providers"]} - assert "openai" in provider_slugs - - def test_watsonx_provider_fields(): """Test that Watsonx provider has all required credential fields including multiple auth options.""" app = FastAPI() @@ -405,3 +357,164 @@ def test_public_model_hub_mixed_health_statuses(): assert claude["health_checked_at"] is None app.dependency_overrides.clear() + +# --------------------------------------------------------------------------- +# /public/endpoints +# --------------------------------------------------------------------------- + +import litellm.proxy.public_endpoints.public_endpoints as _pe_module +from litellm.proxy.public_endpoints.public_endpoints import _build_endpoints, _clean_display_name + + +@pytest.fixture(autouse=False) +def reset_endpoints_cache(): + """Reset the module-level cache before and after each cache-related test.""" + original = _pe_module._cached_endpoints + _pe_module._cached_endpoints = None + yield + _pe_module._cached_endpoints = original + + +def _make_client(): + app = FastAPI() + app.include_router(router) + return TestClient(app) + + +def test_get_supported_endpoints_returns_200(reset_endpoints_cache): + response = _make_client().get("/public/endpoints") + assert response.status_code == 200 + + +def test_get_supported_endpoints_response_shape(reset_endpoints_cache): + data = _make_client().get("/public/endpoints").json() + assert "endpoints" in data + assert isinstance(data["endpoints"], list) + assert len(data["endpoints"]) > 0 + + +def test_get_supported_endpoints_item_fields(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + assert "key" in item + assert "label" in item + assert "endpoint" in item + assert "providers" in item + assert isinstance(item["providers"], list) + + +def test_get_supported_endpoints_provider_fields(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + for provider in item["providers"]: + assert "slug" in provider + assert "display_name" in provider + + +def test_get_supported_endpoints_paths_start_with_slash(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + assert item["endpoint"].startswith("/"), f"Expected path starting with /, got: {item['endpoint']}" + + +def test_get_supported_endpoints_chat_completions_present(reset_endpoints_cache): + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + keys = [item["key"] for item in endpoints] + assert "chat_completions" in keys + + chat = next(item for item in endpoints if item["key"] == "chat_completions") + assert chat["endpoint"] == "/chat/completions" + assert chat["label"] == "Chat Completions" + assert len(chat["providers"]) > 0 + + +def test_get_supported_endpoints_display_names_have_no_slug_suffix(reset_endpoints_cache): + """Provider display_names must not contain the raw `` (`slug`) `` suffix.""" + import re + suffix_re = re.compile(r"\(`[^`]+`\)") + endpoints = _make_client().get("/public/endpoints").json()["endpoints"] + for item in endpoints: + for provider in item["providers"]: + assert not suffix_re.search(provider["display_name"]), ( + f"display_name still contains slug suffix: {provider['display_name']!r}" + ) + + +def test_get_supported_endpoints_is_cached(reset_endpoints_cache): + """`_load_endpoints` is called only once; subsequent requests use the cache.""" + client = _make_client() + with patch( + "litellm.proxy.public_endpoints.public_endpoints._load_endpoints", + wraps=_pe_module._load_endpoints, + ) as mock_load: + client.get("/public/endpoints") + client.get("/public/endpoints") + client.get("/public/endpoints") + + mock_load.assert_called_once() + + +# --------------------------------------------------------------------------- +# _build_endpoints unit tests (transformation logic) +# --------------------------------------------------------------------------- + +_MINIMAL_RAW = { + "providers": { + "openai": { + "display_name": "OpenAI (`openai`)", + "url": "https://example.com", + "endpoints": {"chat_completions": True, "embeddings": True, "images": False}, + }, + "anthropic": { + "display_name": "Anthropic (`anthropic`)", + "url": "https://example.com", + "endpoints": {"chat_completions": True, "embeddings": False, "images": False}, + }, + } +} + + +def test_build_endpoints_known_key_uses_metadata(): + result = _build_endpoints(_MINIMAL_RAW) + chat = next(e for e in result if e["key"] == "chat_completions") + assert chat["label"] == "Chat Completions" + assert chat["endpoint"] == "/chat/completions" + + +def test_build_endpoints_only_includes_supporting_providers(): + result = _build_endpoints(_MINIMAL_RAW) + embeddings = next(e for e in result if e["key"] == "embeddings") + slugs = [p["slug"] for p in embeddings["providers"]] + assert slugs == ["openai"] + + +def test_build_endpoints_unknown_key_derives_label_and_path(): + raw = { + "providers": { + "someprovider": { + "display_name": "Some Provider (`someprovider`)", + "endpoints": {"my_custom_endpoint": True}, + } + } + } + result = _build_endpoints(raw) + item = result[0] + assert item["key"] == "my_custom_endpoint" + assert item["label"] == "My Custom Endpoint" + assert item["endpoint"].startswith("/") + + +def test_build_endpoints_empty_providers_returns_empty(): + result = _build_endpoints({"providers": {}}) + assert result == [] + + +def test_clean_display_name_strips_suffix(): + assert _clean_display_name("OpenAI (`openai`)") == "OpenAI" + assert _clean_display_name("AI/ML API (`aiml`)") == "AI/ML API" + assert _clean_display_name("A2A (Agent-to-Agent) (`a2a`)") == "A2A (Agent-to-Agent)" + + +def test_clean_display_name_passthrough_when_no_suffix(): + assert _clean_display_name("OpenAI") == "OpenAI" + assert _clean_display_name("") == "" diff --git a/tests/test_litellm/proxy/test_spend_log_cleanup.py b/tests/test_litellm/proxy/test_spend_log_cleanup.py index 1ffbb83caef..c1fa3ad0c43 100644 --- a/tests/test_litellm/proxy/test_spend_log_cleanup.py +++ b/tests/test_litellm/proxy/test_spend_log_cleanup.py @@ -151,28 +151,16 @@ async def test_should_delete_spend_logs(): @pytest.mark.asyncio async def test_cleanup_old_spend_logs_batch_deletion(): - from types import SimpleNamespace - from unittest.mock import AsyncMock, MagicMock, patch + from unittest.mock import AsyncMock, MagicMock # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - # Mock spendlogs table - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock() - mock_spendlogs.delete_many = AsyncMock() - - # Create 1500 mocked logs with .request_id - mock_logs = [SimpleNamespace(request_id=f"req_{i}") for i in range(1500)] - mock_spendlogs.find_many.side_effect = [ - mock_logs[:1000], # Batch 1 - mock_logs[1000:], # Batch 2 - [], # Done - ] + # Mock execute_raw to return deleted counts + mock_db.execute_raw = AsyncMock(side_effect=[1000, 500, 0]) # Wire up mocks - mock_db.litellm_spendlogs = mock_spendlogs mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -189,15 +177,13 @@ async def test_cleanup_old_spend_logs_batch_deletion(): assert cleaner._should_delete_spend_logs() is True await cleaner.cleanup_old_spend_logs(mock_prisma_client) - # Validate batching and deletion - assert mock_spendlogs.find_many.call_count == 3 - assert mock_spendlogs.delete_many.call_count == 2 - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000)]}} - ) - mock_spendlogs.delete_many.assert_any_call( - where={"request_id": {"in": [f"req_{i}" for i in range(1000, 1500)]}} - ) + # Validate batching and deletion via raw SQL + assert mock_db.execute_raw.call_count == 3 + + # Check the first call argument + call_args_sql = mock_db.execute_raw.call_args_list[0][0][0] + assert 'DELETE FROM "LiteLLM_SpendLogs"' in call_args_sql + assert 'WHERE "request_id" IN' in call_args_sql @pytest.mark.asyncio @@ -208,10 +194,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): # Setup Prisma client mock_prisma_client = MagicMock() mock_db = MagicMock() - mock_spendlogs = MagicMock() - mock_spendlogs.find_many = AsyncMock(return_value=[]) - mock_spendlogs.delete_many = AsyncMock() - mock_db.litellm_spendlogs = mock_spendlogs + mock_db.execute_raw = AsyncMock(return_value=0) mock_prisma_client.db = mock_db # Mock Redis cache and pod_lock_manager @@ -229,7 +212,7 @@ async def test_cleanup_old_spend_logs_retention_period_cutoff(): await cleaner.cleanup_old_spend_logs(mock_prisma_client) # Verify the cutoff date is correct - cutoff_date = mock_spendlogs.find_many.call_args[1]["where"]["startTime"]["lt"] + cutoff_date = mock_db.execute_raw.call_args[0][1] expected_cutoff = datetime.now(timezone.utc) - timedelta(seconds=86400) assert ( abs((cutoff_date - expected_cutoff).total_seconds()) < 1 @@ -242,14 +225,12 @@ async def test_cleanup_old_spend_logs_no_retention_period(): Test that no logs are deleted when no retention period is set """ mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_spendlogs.find_many = AsyncMock() - mock_prisma_client.db.litellm_spendlogs.delete = AsyncMock() + mock_prisma_client.db.execute_raw = AsyncMock() cleaner = SpendLogCleanup(general_settings={}) # no retention await cleaner.cleanup_old_spend_logs(mock_prisma_client) - mock_prisma_client.db.litellm_spendlogs.find_many.assert_not_called() - mock_prisma_client.db.litellm_spendlogs.delete.assert_not_called() + mock_prisma_client.db.execute_raw.assert_not_called() def test_cleanup_batch_size_env_var(monkeypatch): diff --git a/tests/test_litellm/test_exception_header_preservation.py b/tests/test_litellm/test_exception_header_preservation.py index d3e33fa13b3..ec52d9fb746 100644 --- a/tests/test_litellm/test_exception_header_preservation.py +++ b/tests/test_litellm/test_exception_header_preservation.py @@ -16,6 +16,8 @@ from litellm.exceptions import ( ContentPolicyViolationError, ContextWindowExceededError, ImageFetchError, + MidStreamFallbackError, + RateLimitError, ) @@ -210,6 +212,46 @@ class TestExceptionAttributes: assert error.num_retries == 1 assert error.status_code == 400 + def test_midstream_fallback_error_status_code_propagation(self): + """ + MidStreamFallbackError should preserve the original status code and keep + message/request/response fields consistent after super().__init__(). + """ + original_req = httpx.Request("POST", "https://api.openai.com/v1/chat/completions") + original_resp = httpx.Response(status_code=429, request=original_req) + + rate_limit_error = RateLimitError( + message="Rate limit exceeded", + llm_provider="openai", + model="gpt-4o-mini", + response=original_resp, + ) + + midstream_error = MidStreamFallbackError( + message="stream broke", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=rate_limit_error, + ) + + assert midstream_error.status_code == 429 + assert midstream_error.response.status_code == 429 + assert str(midstream_error.response.request.url) == "https://openai.com/v1/" + assert midstream_error.message == "litellm.MidStreamFallbackError: stream broke" + assert midstream_error.args == ("litellm.MidStreamFallbackError: stream broke",) + + # With no original exception, should default to 503. + midstream_fallback = MidStreamFallbackError( + message="stream broke without original", + model="gpt-4o-mini", + llm_provider="openai", + original_exception=None, + ) + + assert midstream_fallback.status_code == 503 + assert midstream_fallback.response.status_code == 503 + assert str(midstream_fallback.response.request.url) == "https://openai.com/v1/" + class TestProxyHeaderExtraction: """Test that proxy correctly extracts headers from exceptions.""" diff --git a/tests/test_litellm/test_project_tags_pydantic.py b/tests/test_litellm/test_project_tags_pydantic.py new file mode 100644 index 00000000000..b3f58df2325 --- /dev/null +++ b/tests/test_litellm/test_project_tags_pydantic.py @@ -0,0 +1,31 @@ +import pytest +from litellm.proxy._types import NewProjectRequest, UpdateProjectRequest + + +def test_new_project_request_tags(): + # Test tags correctly stay top level initially + req = NewProjectRequest( + project_id="test_proj", team_id="team_1", tags=["tag1", "tag2"] + ) + + # tags should be top level initially + assert req.tags == ["tag1", "tag2"] + + +def test_update_project_request_tags(): + # Test tags correctly stay top level initially + req = UpdateProjectRequest(project_id="test_proj", tags=["new_tag"]) + + assert req.tags == ["new_tag"] + + +def test_new_project_request_invalid_tags_type(): + # tags must be a list — a string should raise a ValidationError + with pytest.raises(Exception): + NewProjectRequest(project_id="test_proj", team_id="team_1", tags="not-a-list") + + +def test_update_project_request_invalid_tags_type(): + # tags must be a list — a string should raise a ValidationError + with pytest.raises(Exception): + UpdateProjectRequest(project_id="test_proj", tags="not-a-list") diff --git a/tests/test_litellm/types/llms/test_types_llms_openai.py b/tests/test_litellm/types/llms/test_types_llms_openai.py index 87cc9586665..054fe505764 100644 --- a/tests/test_litellm/types/llms/test_types_llms_openai.py +++ b/tests/test_litellm/types/llms/test_types_llms_openai.py @@ -169,3 +169,97 @@ class TestResponsesAPIResponseOutputText: ) assert response.output_text == "" + + +class TestAssistantMessageImageUrlContent: + """ + Regression tests for image_url blocks in assistant message content. + + Bug: ChatCompletionAssistantMessage.content did not include + ChatCompletionImageObject in its union, so Pydantic v2 silently dropped + image_url blocks (content → []) when serialising via AllMessageValues. + This affects users who store conversation history as JSON (e.g. in a DB) + and read it back typed as list[AllMessageValues]. + """ + + ASSISTANT_MESSAGE_WITH_IMAGE = { + "role": "assistant", + "content": [ + {"type": "text", "text": "Here is the image you requested:"}, + { + "type": "image_url", + "image_url": { + "url": ( + "data:image/png;base64," + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAA" + "DUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" + ) + }, + }, + ], + } + + def test_assistant_message_image_url_preserved_single(self): + """ + TypeAdapter(ChatCompletionAssistantMessage): image_url block must survive + validate_python → dump_python without being dropped or raising an error. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import ChatCompletionAssistantMessage + + adapter = TypeAdapter(ChatCompletionAssistantMessage) + validated = adapter.validate_python(self.ASSISTANT_MESSAGE_WITH_IMAGE) + dumped = adapter.dump_python(validated) + + raw_content = dumped.get("content") + # Pydantic may return a lazy SerializationIterator for Iterable fields; + # convert to list to consume it — this must not raise ValidationError. + content_blocks = list(raw_content) if raw_content is not None else [] + + assert len(content_blocks) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content_blocks)}: {content_blocks}" + ) + types = [b.get("type") for b in content_blocks if isinstance(b, dict)] + assert "image_url" in types, f"image_url block was silently dropped; blocks: {content_blocks}" + + def test_assistant_message_image_url_preserved_in_all_message_values(self): + """ + TypeAdapter(List[AllMessageValues]) DB round-trip: image_url blocks in an + assistant message must not be silently dropped during dump_python(mode='json'). + + This is the primary failing path: conversation history stored as JSON in a + database and read back typed as list[AllMessageValues]. + """ + from typing import List + + from pydantic import TypeAdapter + + from litellm.types.llms.openai import AllMessageValues + + conversation = [ + { + "role": "user", + "content": "Generate an image of a banana wearing a LiteLLM costume", + }, + self.ASSISTANT_MESSAGE_WITH_IMAGE, + ] + + adapter = TypeAdapter(List[AllMessageValues]) + validated = adapter.validate_python(conversation) + dumped = adapter.dump_python(validated, mode="json") + + assistant = next((m for m in dumped if m.get("role") == "assistant"), None) + assert assistant is not None, "Assistant message missing after serialisation" + + content = assistant.get("content", []) + assert isinstance(content, list), f"content should be a list, got {type(content)}" + assert len(content) == 2, ( + f"Expected 2 content blocks (text + image_url), got {len(content)}: {content}" + ) + types = [b.get("type") for b in content if isinstance(b, dict)] + assert "image_url" in types, ( + f"image_url block was silently dropped during AllMessageValues serialisation; blocks: {content}" + ) diff --git a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx index 8fa5ffd56a2..1d8c980c5ae 100644 --- a/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx +++ b/ui/litellm-dashboard/src/components/add_model/handle_add_model_submit.tsx @@ -38,10 +38,11 @@ export const prepareModelAddRequest = async (formValues: Record, ac litellmParamsObj["model"] = mapping.litellm_model; // Handle pricing conversion before processing other fields - if (formValues.input_cost_per_token) { + // Use explicit checks to allow 0 (zero cost models for budget bypass) + if (formValues.input_cost_per_token !== undefined && formValues.input_cost_per_token !== null && formValues.input_cost_per_token !== "") { formValues.input_cost_per_token = Number(formValues.input_cost_per_token) / 1000000; } - if (formValues.output_cost_per_token) { + if (formValues.output_cost_per_token !== undefined && formValues.output_cost_per_token !== null && formValues.output_cost_per_token !== "") { formValues.output_cost_per_token = Number(formValues.output_cost_per_token) / 1000000; } // Keep input_cost_per_second as is, no conversion needed @@ -116,7 +117,7 @@ export const prepareModelAddRequest = async (formValues: Record, ac // Handle the pricing fields else if (key === "input_cost_per_token" || key === "output_cost_per_token" || key === "input_cost_per_second") { - if (value) { + if (value !== undefined && value !== null && value !== "") { litellmParamsObj[key] = Number(value); } continue;