Merge pull request #27037 from BerriAI/litellm_internal_staging

merge main

.github/workflows/create-release.yml

@@ -4,7 +4,7 @@ on:
   workflow_dispatch:
     inputs:
       tag:
-        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
+        description: "Release tag (e.g. 1.84.0, 1.84.0rc1, 1.84.0.dev42, 1.84.0-dev.2, 1.84.0.post1; legacy v1.83.10-stable still accepted)"
         required: true
         type: string
       commit_hash:
@@ -46,9 +46,11 @@ jobs:
             const commitHash = process.env.COMMIT_HASH;
 
             // Mark RC / dev / nightly / alpha / beta tags as GitHub pre-releases.
+            // Accept both PEP 440 (`.dev`) and SemVer (`-dev`) separators so tags
+            // like `1.84.0.dev2` and `1.84.0-dev.2` are both detected.
             // PEP 440 post-releases (e.g. `1.84.0.post1`) and legacy `-stable[.patch.N]`
             // are stable maintenance releases, not pre-releases.
-            const isPrerelease = /(?:rc|nightly|alpha|beta|\.dev)/i.test(tag);
+            const isPrerelease = /(?:rc|nightly|alpha|beta|[-.]dev)/i.test(tag);
 
             const cosignSection = [
               `## Verify Docker Image Signature`,

.gitignore

@@ -90,7 +90,6 @@ test.py
 litellm_config.yaml
 !.github/observatory/litellm_config.yaml
 .cursor
-.vscode/launch.json
 litellm/proxy/to_delete_loadtest_work/*
 update_model_cost_map.py
 tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@@ -100,4 +99,5 @@ STABILIZATION_TODO.md
 **/test-results
 **/playwright-report
 **/*.storageState.json
-**/coverage
+**/coverage
+test-config

Makefile

@@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
 	$(UV_RUN) pytest tests/llm_translation/$(FILE) \
 		--junitxml=test-results/junit.xml \
 		-v --tb=short --maxfail=100 --timeout=300
+
+test-llm-translation-flush-vcr-cache:
+	$(UV_RUN) python tests/_flush_vcr_cache.py

README.md

@@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
     <td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
     <td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
     <td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
-    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
+    <td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
     <td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
     <td><h2>Netflix</h2></td>
     <td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>

docs/my-website/docs/providers/crusoe.md

@@ -0,0 +1,196 @@
+import Tabs from '@theme/Tabs';
+import TabItem from '@theme/TabItem';
+
+# Crusoe
+
+## Overview
+
+| Property | Details |
+|-------|-------|
+| Description | Crusoe Cloud provides GPU-accelerated inference for open-source large language models, optimized for performance and cost efficiency. |
+| Provider Route on LiteLLM | `crusoe/` |
+| Link to Provider Doc | [Crusoe Managed Inference Documentation ↗](https://docs.crusoecloud.com/managed-inference/overview/index.html) |
+| Base URL | `https://managed-inference-api-proxy.crusoecloud.com/v1` |
+| Supported Operations | [`/chat/completions`](#sample-usage) |
+
+<br />
+<br />
+
+**We support ALL Crusoe models, just set `crusoe/` as a prefix when sending completion requests**
+
+## Available Models
+
+| Model | Description | Context Window |
+|-------|-------------|----------------|
+| `crusoe/deepseek-ai/DeepSeek-R1-0528` | DeepSeek R1 reasoning model (May 2025) | 163,840 tokens |
+| `crusoe/deepseek-ai/DeepSeek-V3-0324` | DeepSeek V3 chat model (March 2025) | 163,840 tokens |
+| `crusoe/google/gemma-3-12b-it` | Google Gemma 3 12B instruction-tuned | 131,072 tokens |
+| `crusoe/meta-llama/Llama-3.3-70B-Instruct` | Llama 3.3 70B instruction-tuned | 131,072 tokens |
+| `crusoe/moonshotai/Kimi-K2-Thinking` | Kimi K2 extended thinking model | 262,144 tokens |
+| `crusoe/openai/gpt-oss-120b` | OpenAI 120B open-source model | 131,072 tokens |
+| `crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507` | Qwen3 235B MoE instruction-tuned | 262,144 tokens |
+
+## Required Variables
+
+```python showLineNumbers title="Environment Variables"
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+```
+
+## Usage - LiteLLM Python SDK
+
+### Non-streaming
+
+```python showLineNumbers title="Crusoe Non-streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Hello, how are you?", "role": "user"}]
+
+# Crusoe call
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages
+)
+
+print(response)
+```
+
+### Streaming
+
+```python showLineNumbers title="Crusoe Streaming Completion"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+messages = [{"content": "Write a short story about AI", "role": "user"}]
+
+# Crusoe call with streaming
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    stream=True
+)
+
+for chunk in response:
+    print(chunk)
+```
+
+### Function Calling
+
+```python showLineNumbers title="Crusoe Function Calling"
+import os
+import litellm
+from litellm import completion
+
+os.environ["CRUSOE_API_KEY"] = ""  # your Crusoe API key
+
+tools = [{
+    "type": "function",
+    "function": {
+        "name": "get_weather",
+        "description": "Get the current weather in a location",
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "location": {
+                    "type": "string",
+                    "description": "The city and state, e.g. San Francisco, CA"
+                }
+            },
+            "required": ["location"]
+        }
+    }
+}]
+
+messages = [{"role": "user", "content": "What's the weather in Boston?"}]
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=messages,
+    tools=tools,
+    tool_choice="auto"
+)
+
+print(response)
+```
+
+## Usage - LiteLLM Proxy Server
+
+```yaml showLineNumbers title="config.yaml"
+model_list:
+  - model_name: llama-3.3-70b
+    litellm_params:
+      model: crusoe/meta-llama/Llama-3.3-70B-Instruct
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-r1
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-R1-0528
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: deepseek-v3
+    litellm_params:
+      model: crusoe/deepseek-ai/DeepSeek-V3-0324
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: qwen3-235b
+    litellm_params:
+      model: crusoe/Qwen/Qwen3-235B-A22B-Instruct-2507
+      api_key: os.environ/CRUSOE_API_KEY
+  - model_name: kimi-k2
+    litellm_params:
+      model: crusoe/moonshotai/Kimi-K2-Thinking
+      api_key: os.environ/CRUSOE_API_KEY
+```
+
+## Custom API Base
+
+**Option 1: Environment variable**
+
+```python showLineNumbers title="Custom API Base via env var"
+import os
+from litellm import completion
+
+os.environ["CRUSOE_API_BASE"] = "https://custom.crusoecloud.com/v1"
+os.environ["CRUSOE_API_KEY"] = ""  # your API key
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+)
+```
+
+**Option 2: Pass directly**
+
+```python showLineNumbers title="Custom API Base via parameter"
+from litellm import completion
+
+response = completion(
+    model="crusoe/meta-llama/Llama-3.3-70B-Instruct",
+    messages=[{"content": "Hello!", "role": "user"}],
+    api_base="https://custom.crusoecloud.com/v1",
+    api_key="your-api-key",
+)
+```
+
+## Supported OpenAI Parameters
+
+- `temperature`
+- `max_tokens`
+- `max_completion_tokens`
+- `top_p`
+- `frequency_penalty`
+- `presence_penalty`
+- `stop`
+- `n`
+- `stream`
+- `tools`
+- `tool_choice`
+- `response_format`
+- `seed`
+- `user`
+- `logit_bias`
+- `logprobs`
+- `top_logprobs`

enterprise/litellm_enterprise/proxy/auth/custom_sso_handler.py

@@ -10,28 +10,21 @@ has already authenticated the user) and you need to extract user information fro
 custom headers or other request attributes.
 """
 
-from typing import TYPE_CHECKING, Dict, Optional, Union, cast
+from typing import cast
 
 from fastapi import Request
 from fastapi.responses import RedirectResponse
 
-if TYPE_CHECKING:
-    from fastapi_sso.sso.base import OpenID
-else:
-    from typing import Any as OpenID
-
-from litellm.proxy.management_endpoints.types import CustomOpenID
-
 
 class EnterpriseCustomSSOHandler:
     """
     Enterprise Custom SSO Handler for LiteLLM Proxy
-    
+
     This class provides methods for handling custom SSO authentication flows
     where users can implement their own authentication logic by processing
     request headers and returning user information in OpenID format.
     """
-    
+
     @staticmethod
     async def handle_custom_ui_sso_sign_in(
         request: Request,
@@ -40,16 +33,16 @@ class EnterpriseCustomSSOHandler:
         Allow a user to execute their custom code to parse incoming request headers and return a OpenID object
 
         Use this when you have an OAuth proxy in front of LiteLLM (where the OAuth proxy has already authenticated the user)
-        
+
         Args:
             request: The FastAPI request object containing headers and other request data
-            
+
         Returns:
             RedirectResponse: Redirect response that sends the user to the LiteLLM UI with authentication token
-            
+
         Raises:
             ValueError: If custom_ui_sso_sign_in_handler is not configured
-            
+
         Example:
             This method is typically called when a user has already been authenticated by an
             external OAuth proxy and the proxy has added custom headers containing user information.
@@ -60,27 +53,44 @@ class EnterpriseCustomSSOHandler:
         from litellm.integrations.custom_sso_handler import CustomSSOLoginHandler
         from litellm.proxy.proxy_server import (
             CommonProxyErrors,
+            general_settings,
             premium_user,
             user_custom_ui_sso_sign_in_handler,
         )
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
+
         if premium_user is not True:
             raise ValueError(CommonProxyErrors.not_premium_user.value)
-        
+
         if user_custom_ui_sso_sign_in_handler is None:
-            raise ValueError("custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings.")
-        
-        custom_sso_login_handler = cast(CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler)
-        openid_response: OpenID = await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
+            raise ValueError(
+                "custom_ui_sso_sign_in_handler is not configured. Please set it in general_settings."
+            )
+
+        require_trusted_proxy_request(
             request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
         )
-        
+
+        custom_sso_login_handler = cast(
+            CustomSSOLoginHandler, user_custom_ui_sso_sign_in_handler
+        )
+        openid_response: OpenID = (
+            await custom_sso_login_handler.handle_custom_ui_sso_sign_in(
+                request=request,
+            )
+        )
+
         # Import here to avoid circular imports
         from litellm.proxy.management_endpoints.ui_sso import SSOAuthenticationHandler
-        
+
         return await SSOAuthenticationHandler.get_redirect_response_from_openid(
             result=openid_response,
             request=request,
             received_response=None,
             generic_client_id=None,
             ui_access_mode=None,
-        ) 
+        )

enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py

@@ -588,24 +588,21 @@ async def update_project(  # noqa: PLR0915
                 param="project_id",
             )
 
-        # Validate team exists and get team object for limit + permission checks
-        team_id_to_check = data.team_id or existing_project.team_id
-        team_obj_for_checks = None
-        if team_id_to_check is not None:
-            team_obj_for_checks = await _validate_team_exists(
-                team_id=team_id_to_check, prisma_client=prisma_client
+        # Permission to *edit* the project must be evaluated against the
+        # project's CURRENT team. Sourcing the team from `data.team_id`
+        # would let an admin of any team pass the check by supplying their
+        # own team_id, hijacking the project (VERIA-55).
+        target_team_id = data.team_id or existing_project.team_id
+        target_team_obj = None
+        if target_team_id is not None:
+            target_team_obj = await _validate_team_exists(
+                team_id=target_team_id, prisma_client=prisma_client
             )
 
-        # Check if user has permission to update this project
         has_permission = await _check_user_permission_for_project(
             user_api_key_dict=user_api_key_dict,
             team_id=existing_project.team_id,
             prisma_client=prisma_client,
-            team_object=(
-                LiteLLM_TeamTable(**team_obj_for_checks.model_dump())
-                if team_obj_for_checks
-                else None
-            ),
         )
 
         if not has_permission:
@@ -614,10 +611,32 @@ async def update_project(  # noqa: PLR0915
                 detail={"error": "Only admins or team admins can update projects"},
             )
 
+        # Reassigning to a different team also requires admin rights on the
+        # destination team — otherwise a team admin could shed projects into
+        # an unsuspecting team's namespace.
+        if data.team_id is not None and data.team_id != existing_project.team_id:
+            can_assign_to_target = await _check_user_permission_for_project(
+                user_api_key_dict=user_api_key_dict,
+                team_id=data.team_id,
+                prisma_client=prisma_client,
+                team_object=(
+                    LiteLLM_TeamTable(**target_team_obj.model_dump())
+                    if target_team_obj
+                    else None
+                ),
+            )
+            if not can_assign_to_target:
+                raise HTTPException(
+                    status_code=403,
+                    detail={
+                        "error": "Cannot reassign project to a team you are not an admin of"
+                    },
+                )
+
         # Validate project limits against team limits
-        if team_obj_for_checks is not None:
+        if target_team_obj is not None:
             _check_team_project_limits(
-                team_object=LiteLLM_TeamTable(**team_obj_for_checks.model_dump()),
+                team_object=LiteLLM_TeamTable(**target_team_obj.model_dump()),
                 data=data,
             )
 
@@ -857,10 +876,16 @@ async def project_info(
                 where={"team_id": project.team_id}
             )
             if team:
-                is_team_member = (
-                    user_api_key_dict.user_id in team.admins
-                    or user_api_key_dict.user_id in team.members
-                )
+                caller_user_id = user_api_key_dict.user_id
+                for m in team.members_with_roles or []:
+                    m_user_id = (
+                        m.get("user_id")
+                        if isinstance(m, dict)
+                        else getattr(m, "user_id", None)
+                    )
+                    if m_user_id == caller_user_id:
+                        is_team_member = True
+                        break
 
         if not (is_admin or is_team_member):
             raise HTTPException(
@@ -911,20 +936,20 @@ async def list_projects(
                 include={"litellm_budget_table": True, "object_permission": True}
             )
         else:
-            # Get projects for teams the user belongs to
-            user_teams = await prisma_client.db.litellm_teamtable.find_many(
-                where={
-                    "OR": [
-                        {"members": {"has": user_api_key_dict.user_id}},
-                        {"admins": {"has": user_api_key_dict.user_id}},
-                    ]
-                }
+            # Look up the user's team memberships via the reverse-index on
+            # LiteLLM_UserTable.teams (maintained by team_member_add alongside
+            # members_with_roles). This avoids a full scan of all team rows.
+            user_record = await prisma_client.db.litellm_usertable.find_unique(
+                where={"user_id": user_api_key_dict.user_id},
+            )
+            user_team_ids = (
+                user_record.teams
+                if user_record is not None and user_record.teams
+                else []
             )
 
-            team_ids = [team.team_id for team in user_teams]
-
             projects = await prisma_client.db.litellm_projecttable.find_many(
-                where={"team_id": {"in": team_ids}},
+                where={"team_id": {"in": user_team_ids}},
                 include={"litellm_budget_table": True, "object_permission": True},
             )
 

litellm-proxy-extras/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "litellm-proxy-extras"
-version = "0.4.69"
+version = "0.4.70"
 description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -26,7 +26,7 @@ required-version = ">=0.10.9"
 module-root = ""
 
 [tool.commitizen]
-version = "0.4.69"
+version = "0.4.70"
 version_files = [
     "pyproject.toml:^version",
     "../pyproject.toml:litellm-proxy-extras==",

litellm/__init__.py

@@ -330,6 +330,9 @@ enable_model_config_credential_overrides: bool = False
 enable_key_alias_format_validation: bool = (
     False  # opt-in validation of key_alias format on /key/generate and /key/update
 )
+enable_gemini_default_thinking_level_low: bool = (
+    False  # opt-in: force thinkingLevel low/minimal for Gemini 3 thinking param mapping
+)
 ####################
 logging: bool = True
 enable_loadbalancing_on_batch_endpoints: Optional[bool] = None

litellm/_logging.py

@@ -1,12 +1,12 @@
 import ast
 import logging
 import os
-import re
 import sys
 from datetime import datetime
 from logging import Formatter
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, Optional
 
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
 from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
 
@@ -21,74 +21,11 @@ _ENABLE_SECRET_REDACTION = (
     os.getenv("LITELLM_DISABLE_REDACT_SECRETS", "").lower() != "true"
 )
 
-_REDACTED = "REDACTED"
-
-
-def _build_secret_patterns() -> re.Pattern:
-    patterns: List[str] = [
-        # ── PEM private key / certificate blocks ──
-        r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
-        # ── GCP OAuth2 access tokens (ya29.*) ──
-        r"\bya29\.[A-Za-z0-9_.~+/-]+",
-        # ── Credential %s formatting (space separator, no key= prefix) ──
-        r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
-        # AWS access key IDs
-        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
-        # AWS secrets / session tokens / access key IDs (key=value)
-        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
-        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
-        # Bearer tokens (OAuth, JWT, etc.)
-        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
-        # Basic auth headers
-        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
-        # OpenAI / Anthropic sk- prefixed keys
-        r"sk-[A-Za-z0-9\-_]{20,}",
-        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
-        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
-        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
-        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Anthropic internal header keys
-        r"x-ak-[A-Za-z0-9\-_]{20,}",
-        # Google API keys
-        r"AIza[0-9A-Za-z\-_]{35}",
-        # Password / secret params (handles key=value and 'key': 'value')
-        # Word boundary prevents O(n^2) backtracking on long word-char runs.
-        r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
-        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
-        # Database connection string credentials (scheme://user:pass@host)
-        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
-        # Databricks personal access tokens
-        r"dapi[0-9a-f]{32}",
-        # ── Key-name-based redaction ──
-        # Catches secrets inside dicts/config dumps by matching on the KEY name
-        # regardless of what the value looks like.
-        # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
-        # private_key with PEM-aware value capture
-        r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
-        r"(?:master_key|database_url|db_url|connection_string|"
-        r"signing_key|encryption_key|"
-        r"auth_token|access_token|refresh_token|"
-        r"slack_webhook_url|webhook_url|"
-        r"database_connection_string|"
-        r"huggingface_token|jwt_secret)"
-        r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
-        # ── Raw JWTs (without Bearer prefix) ──
-        r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
-        # ── Azure SAS tokens in URLs ──
-        r"[?&]sig=[A-Za-z0-9%+/=]+",
-        # ── Full JSON service-account blobs (single-line and multi-line) ──
-        r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
-    ]
-    return re.compile("|".join(patterns), re.IGNORECASE)
-
-
-_SECRET_RE = _build_secret_patterns()
-
 
 def _redact_string(value: str) -> str:
     if not _ENABLE_SECRET_REDACTION:
         return value
-    return _SECRET_RE.sub(_REDACTED, value)
+    return redact_string(value)
 
 
 def redact_secrets(value: str) -> str:

litellm/batches/batch_utils.py

@@ -387,6 +387,27 @@ def _get_batch_job_total_usage_from_file_content(
     )
 
 
+def _get_models_from_batch_input_file_content(
+    file_content_dictionary: List[dict],
+) -> List[str]:
+    """Extract the distinct ``body.model`` values from a batch *input* file.
+
+    Used by the proxy's batch pre-call hook to enforce that the caller is
+    authorized for every model named inside the JSONL — not just the one
+    on the outer request — so the proxy's per-key model allowlist isn't
+    bypassed by smuggling expensive models into the batch file.
+    """
+    models: List[str] = []
+    seen: set = set()
+    for _item in file_content_dictionary:
+        body = _item.get("body") or {}
+        model = body.get("model")
+        if model and model not in seen:
+            seen.add(model)
+            models.append(model)
+    return models
+
+
 def _get_batch_job_input_file_usage(
     file_content_dictionary: List[dict],
     custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai",
@@ -403,11 +424,25 @@ def _get_batch_job_input_file_usage(
     for _item in file_content_dictionary:
         body = _item.get("body", {})
         model = body.get("model", model_name or "")
-        messages = body.get("messages", [])
 
+        # Chat completion payloads.
+        messages = body.get("messages")
         if messages:
-            item_tokens = token_counter(model=model, messages=messages)
-            prompt_tokens += item_tokens
+            prompt_tokens += token_counter(model=model, messages=messages)
+            continue
+
+        # Text completion payloads (`prompt`).
+        prompt = body.get("prompt")
+        if prompt:
+            prompt_tokens += _count_prompt_or_input_tokens(model=model, value=prompt)
+            continue
+
+        # Embedding payloads (`input`).
+        input_data = body.get("input")
+        if input_data:
+            prompt_tokens += _count_prompt_or_input_tokens(
+                model=model, value=input_data
+            )
 
     return Usage(
         total_tokens=prompt_tokens + completion_tokens,
@@ -416,6 +451,43 @@ def _get_batch_job_input_file_usage(
     )
 
 
+def _count_prompt_or_input_tokens(model: str, value: Any) -> int:
+    """Token-count a ``prompt`` / ``input`` field that the OpenAI batch
+    schema allows in four shapes:
+
+    - ``str``: a single text prompt.
+    - ``list[str]``: multiple text prompts.
+    - ``list[int]``: a pre-tokenized prompt (each int counts as 1 token).
+    - ``list[list[int]]``: multiple pre-tokenized prompts.
+
+    Pre-fix only the string shapes were counted, so a caller could send
+    a large ``list[list[int]]`` payload and slip past TPM rate limits
+    with a recorded cost of zero tokens.
+    """
+    if isinstance(value, str):
+        return token_counter(model=model, text=value)
+    if isinstance(value, list):
+        total = 0
+        for chunk in value:
+            if isinstance(chunk, str):
+                total += token_counter(model=model, text=chunk)
+            elif isinstance(chunk, int):
+                # Single pre-tokenized prompt at the top level: each
+                # int counts as one token.
+                total += 1
+            elif isinstance(chunk, list):
+                # Nested pre-tokenized prompt: every int contributes a
+                # token. Mixed string/int items still count.
+                total += sum(1 if isinstance(t, int) else 0 for t in chunk)
+                total += sum(
+                    token_counter(model=model, text=t)
+                    for t in chunk
+                    if isinstance(t, str)
+                )
+        return total
+    return 0
+
+
 def _get_batch_job_usage_from_response_body(response_body: dict) -> Usage:
     """
     Get the tokens of a batch job from the response body

litellm/batches/main.py

@@ -543,15 +543,17 @@ def _handle_retrieve_batch_providers_without_provider_config(
         )
     else:
         raise litellm.exceptions.BadRequestError(
-            message="LiteLLM doesn't support {} for 'create_batch'. Only 'openai' is supported.".format(
-                custom_llm_provider
-            ),
+            message=(
+                "LiteLLM doesn't support custom_llm_provider={} for 'retrieve_batch' without a `model` kwarg. "
+                "Supported via this path: 'openai', 'azure', 'vertex_ai', 'anthropic'. "
+                "'bedrock' is supported but requires `model` to be passed so the provider config can be loaded."
+            ).format(custom_llm_provider),
             model="n/a",
             llm_provider=custom_llm_provider,
             response=httpx.Response(
                 status_code=400,
                 content="Unsupported provider",
-                request=httpx.Request(method="create_thread", url="https://github.com/BerriAI/litellm"),  # type: ignore
+                request=httpx.Request(method="retrieve_batch", url="https://github.com/BerriAI/litellm"),  # type: ignore
             ),
         )
     return response

litellm/caching/caching.py

@@ -432,9 +432,10 @@ class Cache:
             str: The final hashed cache key with the redis namespace.
         """
         dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
+        metadata = kwargs.get("metadata") or {}
         namespace = (
             dynamic_cache_control.get("namespace")
-            or kwargs.get("metadata", {}).get("redis_namespace")
+            or metadata.get("redis_namespace")
             or self.namespace
         )
         if namespace:

litellm/caching/caching_handler.py

@@ -87,6 +87,18 @@ class CachingHandlerResponse(BaseModel):
 in_memory_cache_obj = InMemoryCache()
 
 
+def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
+    """
+    When stream=True, do not run success callbacks at cache-hit time.
+
+    Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
+    replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
+    handlers when the stream finishes; firing them here too would double-count
+    spend and callback records.
+    """
+    return kwargs.get("stream", False) is True
+
+
 class LLMCachingHandler:
     def __init__(
         self,
@@ -99,6 +111,7 @@ class LLMCachingHandler:
         self.async_streaming_chunks: List[ModelResponse] = []
         self.sync_streaming_chunks: List[ModelResponse] = []
         self.request_kwargs = request_kwargs
+        self.preset_cache_key: Optional[str] = None
         self.original_function = original_function
         self.start_time = start_time
         if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
@@ -206,7 +219,7 @@ class LLMCachingHandler:
                         custom_llm_provider=kwargs.get("custom_llm_provider", None),
                         args=args,
                     )
-                    if kwargs.get("stream", False) is False:
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
                         # LOG SUCCESS
                         self._async_log_cache_hit_on_callbacks(
                             logging_obj=logging_obj,
@@ -215,11 +228,12 @@ class LLMCachingHandler:
                             end_time=end_time,
                             cache_hit=cache_hit,
                         )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
+                    )
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
                 elif (
@@ -265,8 +279,6 @@ class LLMCachingHandler:
         kwargs: Dict[str, Any],
         args: Optional[Tuple[Any, ...]] = None,
     ) -> CachingHandlerResponse:
-        from litellm.utils import CustomStreamWrapper
-
         cached_result: Optional[Any] = None
 
         # Check if caching should be performed BEFORE doing expensive kwargs copy
@@ -282,6 +294,11 @@ class LLMCachingHandler:
                     args,
                 )
             )
+            if new_kwargs.get("metadata") is None:
+                new_kwargs.pop("metadata", None)
+            if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+                new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+            self.request_kwargs = new_kwargs
             print_verbose("Checking Sync Cache")
             cached_result = litellm.cache.get_cache(**new_kwargs)
             if cached_result is not None:
@@ -322,17 +339,19 @@ class LLMCachingHandler:
                         is_async=False,
                     )
 
-                    logging_obj.handle_sync_success_callbacks_for_async_calls(
-                        result=cached_result,
-                        start_time=start_time,
-                        end_time=end_time,
-                        cache_hit=cache_hit,
+                    if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
+                        logging_obj.handle_sync_success_callbacks_for_async_calls(
+                            result=cached_result,
+                            start_time=start_time,
+                            end_time=end_time,
+                            cache_hit=cache_hit,
+                        )
+                    cache_key = (
+                        self.preset_cache_key
+                        or self.request_kwargs.get("cache_key")
+                        or litellm.cache.get_cache_key(**self.request_kwargs)
                     )
-                    cache_key = litellm.cache.get_cache_key(**kwargs)
-                    if (
-                        isinstance(cached_result, BaseModel)
-                        or isinstance(cached_result, CustomStreamWrapper)
-                    ) and hasattr(cached_result, "_hidden_params"):
+                    if hasattr(cached_result, "_hidden_params"):
                         cached_result._hidden_params["cache_key"] = cache_key  # type: ignore
                     return CachingHandlerResponse(cached_result=cached_result)
         return CachingHandlerResponse(cached_result=cached_result)
@@ -686,6 +705,11 @@ class LLMCachingHandler:
                 args,
             )
         )
+        if new_kwargs.get("metadata") is None:
+            new_kwargs.pop("metadata", None)
+        if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
+            new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
+        self.request_kwargs = new_kwargs
         cached_result: Optional[Any] = None
         if call_type == CallTypes.aembedding.value:
             if isinstance(new_kwargs["input"], str):
@@ -710,14 +734,26 @@ class LLMCachingHandler:
                 if all(result is None for result in cached_result):
                     cached_result = None
         else:
+            request_kwargs = new_kwargs.copy()
+            request_cache_key = request_kwargs.pop("cache_key", None)
             if litellm.cache._supports_async() is True:
                 ## check if dual cache is supported ##
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = await litellm.cache.async_get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
             else:  # fallback for caches that don't support async
+                self.preset_cache_key = (
+                    request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
+                )
                 cached_result = litellm.cache.get_cache(
-                    dynamic_cache_object=self.dual_cache, **new_kwargs
+                    dynamic_cache_object=self.dual_cache,
+                    cache_key=self.preset_cache_key,
+                    **request_kwargs,
                 )
         return cached_result
 
@@ -825,8 +861,27 @@ class LLMCachingHandler:
         elif (call_type == "aresponses" or call_type == "responses") and isinstance(
             cached_result, dict
         ):
-            # Convert cached dict back to ResponsesAPIResponse object
-            cached_result = ResponsesAPIResponse(**cached_result)
+            from litellm.responses.streaming_iterator import (
+                CachedResponsesAPIStreamingIterator,
+            )
+
+            response_obj = ResponsesAPIResponse(**cached_result)
+            if (
+                hasattr(response_obj, "_hidden_params")
+                and response_obj._hidden_params is not None
+                and isinstance(response_obj._hidden_params, dict)
+            ):
+                response_obj._hidden_params["cache_hit"] = True
+
+            if kwargs.get("stream", False) is True:
+                cached_result = CachedResponsesAPIStreamingIterator(
+                    response=response_obj,
+                    logging_obj=logging_obj,
+                    request_data=kwargs,
+                    call_type=call_type,
+                )
+            else:
+                cached_result = response_obj
 
         if (
             hasattr(cached_result, "_hidden_params")

litellm/caching/dual_cache.py

@@ -92,6 +92,25 @@ class DualCache(BaseCache):
         if default_redis_ttl is not None:
             self.default_redis_ttl = default_redis_ttl
 
+    def attach_redis_cache(
+        self,
+        redis_cache: Optional[RedisCache] = None,
+        *,
+        default_redis_ttl: Optional[float] = None,
+    ) -> None:
+        """
+        Attach a Redis backend if this DualCache does not already have one.
+
+        No-op when ``redis_cache`` is None or when Redis was already set (constructor
+        or a prior attach). Use this for lazy wiring after a shared Redis client exists.
+        Does not backfill in-memory-only keys to Redis.
+        """
+        if redis_cache is None or self.redis_cache is not None:
+            return
+        self.redis_cache = redis_cache
+        if default_redis_ttl is not None:
+            self.default_redis_ttl = default_redis_ttl
+
     def set_cache(self, key, value, local_only: bool = False, **kwargs):
         # Update both Redis and in-memory cache
         try:
@@ -392,6 +411,7 @@ class DualCache(BaseCache):
         value: float,
         parent_otel_span: Optional[Span] = None,
         local_only: bool = False,
+        refresh_ttl: bool = False,
         **kwargs,
     ) -> Optional[float]:
         """
@@ -399,6 +419,9 @@ class DualCache(BaseCache):
 
         Value - float - the value you want to increment by
 
+        Refresh_ttl - bool - if True, resets the Redis TTL on every write.
+        Default False preserves window-style semantics.
+
         Returns - the incremented value, or None if no cache backend is
         available (in_memory_cache is None and Redis failed/is absent).
         """
@@ -415,6 +438,7 @@ class DualCache(BaseCache):
                     value,
                     parent_otel_span=parent_otel_span,
                     ttl=kwargs.get("ttl", None),
+                    refresh_ttl=refresh_ttl,
                 )
 
             return result

litellm/caching/redis_cache.py

@@ -551,6 +551,13 @@ class RedisCache(BaseCache):
     async def async_set_cache(self, key, value, **kwargs):
         from redis.asyncio import Redis
 
+        if key is None:
+            verbose_logger.debug(
+                "LiteLLM Redis Caching: async set() skipped — key is None, value=%r",
+                value,
+            )
+            return None
+
         start_time = time.time()
         try:
             _redis_client: Redis = self.init_async_client()  # type: ignore
@@ -569,8 +576,9 @@ class RedisCache(BaseCache):
                 )
             )
             verbose_logger.error(
-                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
+                "LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
                 str(e),
+                key,
                 value,
             )
             raise e
@@ -824,6 +832,7 @@ class RedisCache(BaseCache):
         value: float,
         ttl: Optional[int] = None,
         parent_otel_span: Optional[Span] = None,
+        refresh_ttl: bool = False,
     ) -> float:
         from redis.asyncio import Redis
 
@@ -834,11 +843,12 @@ class RedisCache(BaseCache):
         try:
             result = await _redis_client.incrbyfloat(name=key, amount=value)
             if _used_ttl is not None:
-                # check if key already has ttl, if not -> set ttl
-                current_ttl = await _redis_client.ttl(key)
-                if current_ttl == -1:
-                    # Key has no expiration
+                if refresh_ttl:
                     await _redis_client.expire(key, _used_ttl)
+                else:
+                    current_ttl = await _redis_client.ttl(key)
+                    if current_ttl == -1:
+                        await _redis_client.expire(key, _used_ttl)
 
             ## LOGGING ##
             end_time = time.time()

litellm/constants.py

@@ -419,9 +419,6 @@ CACHED_STREAMING_CHUNK_DELAY = float(os.getenv("CACHED_STREAMING_CHUNK_DELAY", 0
 AUDIO_SPEECH_CHUNK_SIZE = int(
     os.getenv("AUDIO_SPEECH_CHUNK_SIZE", 8192)
 )  # chunk_size for audio speech streaming. Balance between latency and memory usage
-MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB = int(
-    os.getenv("MAX_SIZE_PER_ITEM_IN_MEMORY_CACHE_IN_KB", 512)
-)
 DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2000))
 #### Networking settings ####
 # Sentinel used when `REQUEST_TIMEOUT` is unset: `litellm.request_timeout` keeps this
@@ -1425,6 +1422,7 @@ LITELLM_PROXY_ADMIN_NAME = "default_user_id"
 LITELLM_CLI_SOURCE_IDENTIFIER = "litellm-cli"
 LITELLM_CLI_SESSION_TOKEN_PREFIX = "litellm-session-token"
 CLI_SSO_SESSION_CACHE_KEY_PREFIX = "cli_sso_session"
+CLI_SSO_SESSION_TTL_SECONDS = 600
 CLI_JWT_TOKEN_NAME = "cli-jwt-token"
 # Support both CLI_JWT_EXPIRATION_HOURS and LITELLM_CLI_JWT_EXPIRATION_HOURS for backwards compatibility
 CLI_JWT_EXPIRATION_HOURS = int(

litellm/cost_calculator.py

@@ -513,7 +513,10 @@ def cost_per_token(  # noqa: PLR0915
         return fireworks_ai_cost_per_token(model=model, usage=usage_block)
     elif custom_llm_provider == "azure":
         return azure_openai_cost_per_token(
-            model=model, usage=usage_block, response_time_ms=response_time_ms
+            model=model,
+            usage=usage_block,
+            response_time_ms=response_time_ms,
+            service_tier=service_tier,
         )
     elif custom_llm_provider == "gemini":
         return gemini_cost_per_token(
@@ -539,6 +542,7 @@ def cost_per_token(  # noqa: PLR0915
             usage=usage_block,
             response_time_ms=response_time_ms,
             request_model=request_model,
+            service_tier=service_tier,
         )
     else:
         model_info = _cached_get_model_info_helper(

litellm/integrations/arize/_utils.py

@@ -220,23 +220,57 @@ def _set_structured_outputs(span: "Span", response_obj, msg_attrs, span_attrs):
             safe_set_attribute(span, f"{prefix}.{msg_attrs.MESSAGE_ROLE}", message_role)
 
 
+def _safe_get(obj, key, default=None):
+    """Read ``key`` from a dict-like or Pydantic-model-like object.
+
+    The arize/langfuse_otel logger receives ``usage`` objects from many sources:
+    plain dicts, litellm ``Usage`` (which exposes ``.get``), and raw OpenAI
+    Pydantic models (e.g. ``openai.types.completion_usage.CompletionUsage`` and
+    nested ``CompletionTokensDetails`` / ``OutputTokensDetails``) which do NOT
+    expose ``.get``. Calling ``.get`` on the latter raised ``AttributeError`` —
+    see https://github.com/BerriAI/litellm/issues/13672.
+    """
+    if obj is None:
+        return default
+    getter = getattr(obj, "get", None)
+    if callable(getter):
+        try:
+            return getter(key, default)
+        except TypeError:
+            # Some objects expose `.get` with a different signature
+            pass
+    return getattr(obj, key, default)
+
+
 def _set_usage_outputs(span: "Span", response_obj, span_attrs):
     usage = response_obj and response_obj.get("usage")
     if not usage:
         return
 
     safe_set_attribute(
-        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, usage.get("total_tokens")
+        span, span_attrs.LLM_TOKEN_COUNT_TOTAL, _safe_get(usage, "total_tokens")
+    )
+    completion_tokens = _safe_get(usage, "completion_tokens") or _safe_get(
+        usage, "output_tokens"
     )
-    completion_tokens = usage.get("completion_tokens") or usage.get("output_tokens")
     if completion_tokens:
         safe_set_attribute(
             span, span_attrs.LLM_TOKEN_COUNT_COMPLETION, completion_tokens
         )
-    prompt_tokens = usage.get("prompt_tokens") or usage.get("input_tokens")
+    prompt_tokens = _safe_get(usage, "prompt_tokens") or _safe_get(
+        usage, "input_tokens"
+    )
     if prompt_tokens:
         safe_set_attribute(span, span_attrs.LLM_TOKEN_COUNT_PROMPT, prompt_tokens)
-    reasoning_tokens = usage.get("output_tokens_details", {}).get("reasoning_tokens")
+
+    # Reasoning tokens live in `completion_tokens_details` for Chat Completions
+    # API (Usage) and in `output_tokens_details` for Responses API
+    # (ResponseAPIUsage). Both nested objects may be plain Pydantic models
+    # without `.get`.
+    token_details = _safe_get(usage, "completion_tokens_details") or _safe_get(
+        usage, "output_tokens_details"
+    )
+    reasoning_tokens = _safe_get(token_details, "reasoning_tokens")
     if reasoning_tokens:
         safe_set_attribute(
             span,

litellm/integrations/arize/arize_phoenix_client.py

@@ -2,11 +2,23 @@
 Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
 """
 
+import urllib.parse
 from typing import Any, Dict, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_id(identifier: str) -> str:
+    """Reject path traversal characters and URL-encode the identifier."""
+    if any(c in identifier for c in ("/", "\\", "#", "?")):
+        raise ValueError(
+            f"Invalid identifier {identifier!r}: contains disallowed characters"
+        )
+    if ".." in identifier:
+        raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
+    return urllib.parse.quote(identifier, safe="")
+
+
 class ArizePhoenixClient:
     """
     Client for interacting with Arize Phoenix API to fetch prompt versions.
@@ -53,7 +65,8 @@ class ArizePhoenixClient:
         Returns:
             Dictionary containing prompt version data, or None if not found
         """
-        url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
+        safe_id = _sanitize_id(prompt_version_id)
+        url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
 
         try:
             # Use the underlying httpx client directly to avoid query param extraction

litellm/integrations/bitbucket/bitbucket_client.py

@@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
 """
 
 import base64
+import urllib.parse
 from typing import Any, Dict, List, Optional
 
 from litellm.llms.custom_httpx.http_handler import HTTPHandler
 
 
+def _sanitize_file_path(file_path: str) -> str:
+    """Reject path traversal and URL-encode each path segment."""
+    if "#" in file_path or "?" in file_path:
+        raise ValueError(
+            f"Invalid file path {file_path!r}: contains URL special characters"
+        )
+    parts = file_path.split("/")
+    for part in parts:
+        if part == "..":
+            raise ValueError(
+                f"Invalid file path {file_path!r}: path traversal detected"
+            )
+    return "/".join(urllib.parse.quote(part, safe="") for part in parts)
+
+
 class BitBucketClient:
     """
     Client for interacting with BitBucket API to fetch .prompt files.
@@ -72,7 +88,8 @@ class BitBucketClient:
         Returns:
             File content as string, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -119,7 +136,8 @@ class BitBucketClient:
         Returns:
             List of file paths
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
+        safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
 
         try:
             response = self.http_handler.get(url, headers=self.headers)
@@ -211,7 +229,8 @@ class BitBucketClient:
         Returns:
             Dictionary containing file metadata, or None if file not found
         """
-        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
+        safe_path = _sanitize_file_path(file_path)
+        url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
 
         try:
             # Use GET with Range header to get just the headers (HEAD equivalent)

litellm/integrations/custom_sso_handler.py

@@ -18,6 +18,17 @@ class CustomSSOLoginHandler(CustomLogger):
         self,
         request: Request,
     ) -> OpenID:
+        from litellm.proxy.auth.trusted_proxy_utils import (
+            require_trusted_proxy_request,
+        )
+        from litellm.proxy.proxy_server import general_settings
+
+        require_trusted_proxy_request(
+            request=request,
+            general_settings=general_settings,
+            feature_name="Custom UI SSO",
+        )
+
         request_headers_dict = dict(request.headers)
         return OpenID(
             id=request_headers_dict.get("x-litellm-user-id"),

litellm/integrations/langfuse/langfuse.py

@@ -90,6 +90,29 @@ def _extract_cache_read_input_tokens(usage_obj) -> int:
     return cache_read_input_tokens
 
 
+def resolve_langfuse_credentials(
+    langfuse_public_key=None,
+    langfuse_secret=None,
+    langfuse_secret_key=None,
+    langfuse_host=None,
+    allow_env_credentials: bool = True,
+):
+    if allow_env_credentials is False and langfuse_host is not None:
+        secret_key = langfuse_secret or langfuse_secret_key
+        public_key = langfuse_public_key
+    else:
+        secret_key = (
+            langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
+        )
+        public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
+
+    resolved_host = langfuse_host or os.getenv(
+        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    )
+
+    return public_key, secret_key, resolved_host
+
+
 class LangFuseLogger:
     # Class variables or attributes
     def __init__(
@@ -98,6 +121,7 @@ class LangFuseLogger:
         langfuse_secret=None,
         langfuse_host=None,
         flush_interval=1,
+        allow_env_credentials: bool = True,
     ):
         try:
             import langfuse
@@ -106,11 +130,13 @@ class LangFuseLogger:
             raise Exception(
                 f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n{traceback.format_exc()}\033[0m"
             )
-        # Instance variables
-        self.secret_key = langfuse_secret or os.getenv("LANGFUSE_SECRET_KEY")
-        self.public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-        self.langfuse_host = langfuse_host or os.getenv(
-            "LANGFUSE_HOST", "https://cloud.langfuse.com"
+        self.public_key, self.secret_key, self.langfuse_host = (
+            resolve_langfuse_credentials(
+                langfuse_public_key=langfuse_public_key,
+                langfuse_secret=langfuse_secret,
+                langfuse_host=langfuse_host,
+                allow_env_credentials=allow_env_credentials,
+            )
         )
         if not (
             self.langfuse_host.startswith("http://")
@@ -160,9 +186,10 @@ class LangFuseLogger:
                 project_id = None
 
         if os.getenv("UPSTREAM_LANGFUSE_SECRET_KEY") is not None:
+            upstream_langfuse_debug_env = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
             upstream_langfuse_debug = (
-                str_to_bool(self.upstream_langfuse_debug)
-                if self.upstream_langfuse_debug is not None
+                str_to_bool(upstream_langfuse_debug_env)
+                if upstream_langfuse_debug_env is not None
                 else None
             )
             self.upstream_langfuse_secret_key = os.getenv(
@@ -173,7 +200,7 @@ class LangFuseLogger:
             )
             self.upstream_langfuse_host = os.getenv("UPSTREAM_LANGFUSE_HOST")
             self.upstream_langfuse_release = os.getenv("UPSTREAM_LANGFUSE_RELEASE")
-            self.upstream_langfuse_debug = os.getenv("UPSTREAM_LANGFUSE_DEBUG")
+            self.upstream_langfuse_debug = upstream_langfuse_debug_env
             self.upstream_langfuse = Langfuse(
                 public_key=self.upstream_langfuse_public_key,
                 secret_key=self.upstream_langfuse_secret_key,

litellm/integrations/langfuse/langfuse_handler.py

@@ -115,8 +115,10 @@ class LangFuseHandler:
 
         langfuse_logger = LangFuseLogger(
             langfuse_public_key=credentials.get("langfuse_public_key"),
-            langfuse_secret=credentials.get("langfuse_secret"),
+            langfuse_secret=credentials.get("langfuse_secret")
+            or credentials.get("langfuse_secret_key"),
             langfuse_host=credentials.get("langfuse_host"),
+            allow_env_credentials=credentials.get("langfuse_host") is None,
         )
         in_memory_dynamic_logger_cache.set_cache(
             credentials=credentials,

litellm/integrations/langfuse/langfuse_prompt_management.py

@@ -20,7 +20,7 @@ from ...litellm_core_utils.specialty_caches.dynamic_logging_cache import (
     DynamicLoggingCache,
 )
 from ..prompt_management_base import PromptManagementBase
-from .langfuse import LangFuseLogger
+from .langfuse import LangFuseLogger, resolve_langfuse_credentials
 from .langfuse_handler import LangFuseHandler
 
 if TYPE_CHECKING:
@@ -46,6 +46,7 @@ def langfuse_client_init(
     langfuse_secret_key=None,
     langfuse_host=None,
     flush_interval=1,
+    allow_env_credentials: bool = True,
 ) -> LangfuseClass:
     """
     Initialize Langfuse client with caching to prevent multiple initializations.
@@ -70,14 +71,12 @@ def langfuse_client_init(
             f"\033[91mLangfuse not installed, try running 'pip install langfuse' to fix this error: {e}\n\033[0m"
         )
 
-    # Instance variables
-
-    secret_key = (
-        langfuse_secret or langfuse_secret_key or os.getenv("LANGFUSE_SECRET_KEY")
-    )
-    public_key = langfuse_public_key or os.getenv("LANGFUSE_PUBLIC_KEY")
-    langfuse_host = langfuse_host or os.getenv(
-        "LANGFUSE_HOST", "https://cloud.langfuse.com"
+    public_key, secret_key, langfuse_host = resolve_langfuse_credentials(
+        langfuse_public_key=langfuse_public_key,
+        langfuse_secret=langfuse_secret,
+        langfuse_secret_key=langfuse_secret_key,
+        langfuse_host=langfuse_host,
+        allow_env_credentials=allow_env_credentials,
     )
 
     if not (
@@ -222,6 +221,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,
@@ -246,6 +246,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge
             langfuse_secret=dynamic_callback_params.get("langfuse_secret"),
             langfuse_secret_key=dynamic_callback_params.get("langfuse_secret_key"),
             langfuse_host=dynamic_callback_params.get("langfuse_host"),
+            allow_env_credentials=dynamic_callback_params.get("langfuse_host") is None,
         )
         langfuse_prompt_client = self._get_prompt_from_id(
             langfuse_prompt_id=prompt_id,

litellm/integrations/langsmith.py

@@ -112,17 +112,28 @@ class LangsmithLogger(CustomBatchLogger):
         langsmith_project: Optional[str] = None,
         langsmith_base_url: Optional[str] = None,
         langsmith_tenant_id: Optional[str] = None,
+        allow_env_credentials: bool = True,
     ) -> LangsmithCredentialsObject:
-        _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
-        _credentials_project = (
-            langsmith_project or os.getenv("LANGSMITH_PROJECT") or "litellm-completion"
-        )
-        _credentials_base_url = (
-            langsmith_base_url
-            or os.getenv("LANGSMITH_BASE_URL")
-            or "https://api.smith.langchain.com"
-        )
-        _credentials_tenant_id = langsmith_tenant_id or os.getenv("LANGSMITH_TENANT_ID")
+        if allow_env_credentials is False and langsmith_base_url is not None:
+            _credentials_api_key = langsmith_api_key
+            _credentials_project = langsmith_project or "litellm-completion"
+            _credentials_base_url = langsmith_base_url
+            _credentials_tenant_id = langsmith_tenant_id
+        else:
+            _credentials_api_key = langsmith_api_key or os.getenv("LANGSMITH_API_KEY")
+            _credentials_project = (
+                langsmith_project
+                or os.getenv("LANGSMITH_PROJECT")
+                or "litellm-completion"
+            )
+            _credentials_base_url = (
+                langsmith_base_url
+                or os.getenv("LANGSMITH_BASE_URL")
+                or "https://api.smith.langchain.com"
+            )
+            _credentials_tenant_id = langsmith_tenant_id or os.getenv(
+                "LANGSMITH_TENANT_ID"
+            )
 
         return LangsmithCredentialsObject(
             LANGSMITH_API_KEY=_credentials_api_key,
@@ -540,6 +551,10 @@ class LangsmithLogger(CustomBatchLogger):
                 langsmith_tenant_id=standard_callback_dynamic_params.get(
                     "langsmith_tenant_id", None
                 ),
+                allow_env_credentials=standard_callback_dynamic_params.get(
+                    "langsmith_base_url", None
+                )
+                is None,
             )
         else:
             credentials = self.default_credentials

litellm/integrations/opentelemetry.py

@@ -69,6 +69,8 @@ class OpenTelemetryConfig:
     deployment_environment: Optional[str] = None
     model_id: Optional[str] = None
     ignore_context_propagation: Optional[bool] = None
+    # When True, create a private TracerProvider instead of reusing or setting the global one.
+    skip_set_global: bool = False
 
     def __post_init__(self) -> None:
         # If endpoint is specified but exporter is still the default "console",
@@ -259,16 +261,21 @@ class OpenTelemetry(CustomLogger):
         try:
             existing_provider = get_existing_provider_fn()
 
-            # If a real SDK provider exists (set by another SDK like Langfuse), use it
-            # This uses a positive check for SDK providers instead of a negative check for proxy providers
             if isinstance(existing_provider, sdk_provider_class):
-                verbose_logger.debug(
-                    "OpenTelemetry: Using existing %s: %s",
-                    provider_name,
-                    type(existing_provider).__name__,
-                )
-                provider = existing_provider
-                # Don't call set_provider to preserve existing context
+                if skip_set_global:
+                    verbose_logger.debug(
+                        "OpenTelemetry: existing %s found but skip_set_global=True; creating private %s for isolation",
+                        provider_name,
+                        provider_name,
+                    )
+                    provider = create_new_provider_fn()
+                else:
+                    verbose_logger.debug(
+                        "OpenTelemetry: Using existing %s: %s",
+                        provider_name,
+                        type(existing_provider).__name__,
+                    )
+                    provider = existing_provider
             else:
                 # Default proxy provider or unknown type, create our own
                 verbose_logger.debug("OpenTelemetry: Creating new %s", provider_name)
@@ -293,6 +300,12 @@ class OpenTelemetry(CustomLogger):
 
         return provider
 
+    def _skip_set_global(self) -> bool:
+        # langfuse_otel relies on the Langfuse SDK's providers; don't overwrite them.
+        return self.config.skip_set_global or (
+            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
+        )
+
     def _init_tracing(self, tracer_provider):
         from opentelemetry import trace
         from opentelemetry.sdk.trace import TracerProvider
@@ -303,11 +316,6 @@ class OpenTelemetry(CustomLogger):
             provider.add_span_processor(self._get_span_processor())
             return provider
 
-        # CRITICAL FIX: For Langfuse OTEL, skip setting global provider to prevent interference
-        skip_global = (
-            hasattr(self, "callback_name") and self.callback_name == "langfuse_otel"
-        )
-
         tracer_provider = self._get_or_create_provider(
             provider=tracer_provider,
             provider_name="TracerProvider",
@@ -315,16 +323,18 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=TracerProvider,
             create_new_provider_fn=create_tracer_provider,
             set_provider_fn=trace.set_tracer_provider,
-            skip_set_global=skip_global,
+            skip_set_global=self._skip_set_global(),
         )
 
         # Grab our tracer from the TracerProvider (not from global context)
         # This ensures we use the provided TracerProvider (e.g., for testing)
         self.tracer = tracer_provider.get_tracer(LITELLM_TRACER_NAME)
+        self._tracer_provider = tracer_provider
         self.span_kind = SpanKind
 
     def _init_metrics(self, meter_provider):
         if not self.config.enable_metrics:
+            self._meter_provider = None
             self._operation_duration_histogram = None
             self._token_usage_histogram = None
             self._cost_histogram = None
@@ -350,7 +360,9 @@ class OpenTelemetry(CustomLogger):
             sdk_provider_class=MeterProvider,
             create_new_provider_fn=create_meter_provider,
             set_provider_fn=metrics.set_meter_provider,
+            skip_set_global=self._skip_set_global(),
         )
+        self._meter_provider = meter_provider
 
         meter = meter_provider.get_meter(__name__)
 
@@ -388,6 +400,7 @@ class OpenTelemetry(CustomLogger):
     def _init_logs(self, logger_provider):
         # nothing to do if events disabled
         if not self.config.enable_events:
+            self._logger_provider = None
             return
 
         from opentelemetry._logs import get_logger_provider, set_logger_provider
@@ -404,13 +417,14 @@ class OpenTelemetry(CustomLogger):
             )
             return provider
 
-        self._get_or_create_provider(
+        self._logger_provider = self._get_or_create_provider(
             provider=logger_provider,
             provider_name="LoggerProvider",
             get_existing_provider_fn=get_logger_provider,
             sdk_provider_class=OTLoggerProvider,
             create_new_provider_fn=create_logger_provider,
             set_provider_fn=set_logger_provider,
+            skip_set_global=self._skip_set_global(),
         )
 
     def log_success_event(self, kwargs, response_obj, start_time, end_time):
@@ -1073,7 +1087,7 @@ class OpenTelemetry(CustomLogger):
         # See: https://github.com/open-telemetry/opentelemetry-python/pull/4676
         # TODO: Refactor to use the proper OTEL Logs API instead of directly creating SDK LogRecords
 
-        from opentelemetry._logs import SeverityNumber, get_logger
+        from opentelemetry._logs import SeverityNumber
 
         try:
             from opentelemetry.sdk._logs import (  # type: ignore[attr-defined]  # OTEL < 1.39.0
@@ -1084,7 +1098,10 @@ class OpenTelemetry(CustomLogger):
                 LogRecord as SdkLogRecord,  # type: ignore[attr-defined]  # OTEL >= 1.39.0
             )
 
-        otel_logger = get_logger(LITELLM_LOGGER_NAME)
+        # Resolve through the handler's own LoggerProvider (which may be a
+        # private one when skip_set_global=True) rather than the module-level
+        # get_logger() which always goes through the global provider.
+        otel_logger = self._logger_provider.get_logger(LITELLM_LOGGER_NAME)
 
         parent_ctx = span.get_span_context()
         provider = (kwargs.get("litellm_params") or {}).get(

litellm/integrations/prometheus.py

@@ -265,6 +265,7 @@ class PrometheusLogger(CustomLogger):
             ########################################
             # LiteLLM Virtual API KEY metrics
             ########################################
+
             # Remaining MODEL RPM limit for API Key
             self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
                 "litellm_remaining_api_key_requests_for_model",

litellm/integrations/prometheus_helpers/prometheus_api.py

@@ -2,6 +2,7 @@
 Helper functions to query prometheus API
 """
 
+import json
 import time
 from datetime import datetime, timedelta
 from typing import Optional
@@ -81,6 +82,24 @@ def is_prometheus_connected() -> bool:
     return False
 
 
+def _quote_promql_string_literal(value: str) -> str:
+    """Render ``value`` as a PromQL double-quoted string literal.
+
+    PromQL string literals follow Go's escape rules
+    (https://prometheus.io/docs/prometheus/latest/querying/basics/): a
+    backslash begins an escape sequence and a bare ``"`` ends the literal.
+    Without escaping, callers that accept arbitrary user-supplied values
+    (like the ``api_key`` filter on ``/global/spend/logs``) can inject extra
+    label matchers or selectors and read cross-tenant metrics.
+
+    JSON's quoting rules are a strict subset of Go's, so ``json.dumps`` of
+    a Python string produces a literal Prometheus accepts: ``\\``, ``\\"``,
+    and the standard ``\\n`` / ``\\t`` / ``\\uNNNN`` control-character
+    escapes. The returned value already includes the surrounding quotes.
+    """
+    return json.dumps(value, ensure_ascii=False)
+
+
 async def get_daily_spend_from_prometheus(api_key: Optional[str]):
     """
     Expected Response Format:
@@ -109,8 +128,11 @@ async def get_daily_spend_from_prometheus(api_key: Optional[str]):
     if api_key is None:
         query = "sum(delta(litellm_spend_metric_total[1d]))"
     else:
+        quoted_api_key = _quote_promql_string_literal(api_key)
         query = (
-            f'sum(delta(litellm_spend_metric_total{{hashed_api_key="{api_key}"}}[1d]))'
+            "sum(delta(litellm_spend_metric_total{"
+            f"hashed_api_key={quoted_api_key}"
+            "}[1d]))"
         )
 
     params = {

litellm/litellm_core_utils/cli_token_utils.py

@@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]:
         return None
 
 
-def get_litellm_gateway_api_key() -> Optional[str]:
+def get_litellm_gateway_api_key(
+    expected_base_url: Optional[str] = None,
+) -> Optional[str]:
     """
     Get the stored CLI API key for use with LiteLLM SDK.
 
     This function reads the token file created by `litellm-proxy login`
     and returns the API key for use in Python scripts.
 
+    Args:
+        expected_base_url: When provided, the key is only returned if it was
+            originally issued for this URL. Pass the target server URL to
+            prevent credential leakage when the client is pointed at a
+            different (possibly malicious) server.
+
     Returns:
-        str: The API key if found, None otherwise
+        str: The API key if found (and origin matches), None otherwise
 
     Example:
         >>> import litellm
@@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]:
         >>>     )
     """
     token_data = load_cli_token()
-    if token_data and "key" in token_data:
-        return token_data["key"]
-    return None
+    if not token_data or "key" not in token_data:
+        return None
+    if expected_base_url is not None:
+        stored_url = token_data.get("base_url")
+        if stored_url != expected_base_url.rstrip("/"):
+            return None
+    return token_data["key"]

litellm/litellm_core_utils/exception_mapping_utils.py

@@ -6,7 +6,8 @@ from typing import Any, Optional
 import httpx
 
 import litellm
-from litellm._logging import _redact_string, verbose_logger
+from litellm._logging import _ENABLE_SECRET_REDACTION, _redact_string, verbose_logger
+from litellm.litellm_core_utils.secret_redaction import redact_string
 from litellm.types.utils import LlmProviders
 
 from ..exceptions import (
@@ -261,10 +262,18 @@ def exception_type(  # type: ignore  # noqa: PLR0915
         original_exception=original_exception
     )
     try:
-        error_str = str(original_exception)
+        error_str = (
+            redact_string(str(original_exception))
+            if _ENABLE_SECRET_REDACTION
+            else str(original_exception)
+        )
         if model:
             if hasattr(original_exception, "message"):
-                error_str = str(original_exception.message)
+                error_str = (
+                    redact_string(str(original_exception.message))
+                    if _ENABLE_SECRET_REDACTION
+                    else str(original_exception.message)
+                )
             if isinstance(original_exception, BaseException):
                 exception_type = type(original_exception).__name__
             else:
@@ -2431,7 +2440,8 @@ def exception_type(  # type: ignore  # noqa: PLR0915
             else:
                 raise APIConnectionError(
                     message="{}\n{}".format(
-                        str(original_exception), _redact_string(traceback.format_exc())
+                        str(original_exception),
+                        _redact_string(traceback.format_exc()),
                     ),
                     llm_provider=custom_llm_provider,
                     model=model,
@@ -2461,7 +2471,8 @@ def exception_type(  # type: ignore  # noqa: PLR0915
                     raise e  # it's already mapped
             raised_exc = APIConnectionError(
                 message="{}\n{}".format(
-                    original_exception, _redact_string(traceback.format_exc())
+                    original_exception,
+                    _redact_string(traceback.format_exc()),
                 ),
                 llm_provider="",
                 model="",

litellm/litellm_core_utils/initialize_dynamic_callback_params.py

@@ -23,6 +23,13 @@ def _raise_env_reference_error(param: str, *, source: str) -> None:
     )
 
 
+def validate_no_callback_env_reference(
+    param: str, value: object, *, source: str
+) -> None:
+    if _is_env_reference(value):
+        _raise_env_reference_error(param, source=source)
+
+
 # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict
 _supported_callback_params = [
     "langfuse_public_key",
@@ -66,8 +73,9 @@ def initialize_standard_callback_dynamic_params(
         for param in _supported_callback_params:
             if param in kwargs:
                 _param_value = kwargs.get(param)
-                if _is_env_reference(_param_value):
-                    _raise_env_reference_error(param, source="request body")
+                validate_no_callback_env_reference(
+                    param, _param_value, source="request body"
+                )
                 standard_callback_dynamic_params[param] = _param_value  # type: ignore
 
         # 2. Fallback: check "metadata" or "litellm_params" -> "metadata"
@@ -80,8 +88,9 @@ def initialize_standard_callback_dynamic_params(
             for param in _supported_callback_params:
                 if param not in standard_callback_dynamic_params and param in metadata:
                     _param_value = metadata.get(param)
-                    if _is_env_reference(_param_value):
-                        _raise_env_reference_error(param, source="metadata")
+                    validate_no_callback_env_reference(
+                        param, _param_value, source="metadata"
+                    )
                     standard_callback_dynamic_params[param] = _param_value  # type: ignore
 
     return standard_callback_dynamic_params

litellm/litellm_core_utils/litellm_logging.py

@@ -3242,10 +3242,15 @@ class Logging(LiteLLMLoggingBaseClass):
                     ),
                     langfuse_secret=self.standard_callback_dynamic_params.get(
                         "langfuse_secret"
-                    ),
+                    )
+                    or self.standard_callback_dynamic_params.get("langfuse_secret_key"),
                     langfuse_host=self.standard_callback_dynamic_params.get(
                         "langfuse_host"
                     ),
+                    allow_env_credentials=self.standard_callback_dynamic_params.get(
+                        "langfuse_host"
+                    )
+                    is None,
                 )
             return langFuseLogger
 
@@ -4720,7 +4725,7 @@ class StandardLoggingPayloadSetup:
         ):
             for key, value in litellm_params["metadata"].items():
                 # Skip non-serializable objects like UserAPIKeyAuth
-                if key == "user_api_key_auth":
+                if key in {"user_api_key_auth", "user_api_key_budget_reservation"}:
                     continue
                 merged_metadata[key] = value
 

litellm/litellm_core_utils/llm_request_utils.py

@@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
     if litellm_params is None:
         return {}
 
-    proxy_request_headers = (
-        litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
-    )
+    proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get(
+        "headers"
+    ) or {}
 
     return proxy_request_headers

litellm/litellm_core_utils/prompt_templates/factory.py

@@ -4582,6 +4582,11 @@ class BedrockConverseMessagesProcessor:
                                     message=cast(ChatCompletionFileObject, element)
                                 )
                                 _parts.append(_part)
+                            elif element["type"] == "document":
+                                _part = BedrockConverseMessagesProcessor._process_document_message(
+                                    element
+                                )
+                                _parts.append(_part)
                             _cache_point_block = (
                                 litellm.AmazonConverseConfig()._get_cache_point_block(
                                     message_block=cast(
@@ -4864,6 +4869,44 @@ class BedrockConverseMessagesProcessor:
             image_url=cast(str, file_id or file_data), format=format
         )
 
+    @staticmethod
+    def _process_document_message(element: dict) -> BedrockContentBlock:
+        """Convert a document content block to a Bedrock DocumentBlock.
+
+        Handles the Anthropic-style document format:
+        {"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "..."}}
+        """
+        source = element["source"]
+        source_type = source.get("type")
+        if source_type != "base64":
+            raise ValueError(
+                f"Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. "
+                "Please convert the document to base64 before sending to Bedrock."
+            )
+        media_type: str = source["media_type"]
+        data: str = source["data"]
+        doc_format = BedrockImageProcessor._validate_format(
+            mime_type=media_type, image_format=media_type.split("/")[1]
+        )
+
+        # Deterministic name using the same hashing pattern as _create_bedrock_block
+        HASH_SAMPLE_BYTES = 64 * 1024
+        normalized = "".join(data.split()).encode("utf-8")
+        sample = normalized[:HASH_SAMPLE_BYTES]
+        hasher = hashlib.sha256()
+        hasher.update(sample)
+        hasher.update(str(len(normalized)).encode("utf-8"))
+        content_hash = hasher.hexdigest()[:16]
+        document_name = f"Document_{content_hash}_{doc_format}"
+
+        return BedrockContentBlock(
+            document=BedrockDocumentBlock(
+                source=BedrockSourceBlock(bytes=data),
+                format=doc_format,
+                name=document_name,
+            )
+        )
+
     @staticmethod
     def add_thinking_blocks_to_assistant_content(
         thinking_blocks: List[BedrockContentBlock],
@@ -4961,6 +5004,11 @@ def _bedrock_converse_messages_pt(  # noqa: PLR0915
                                 )
                             )
                             _parts.append(_part)
+                        elif element["type"] == "document":
+                            _part = BedrockConverseMessagesProcessor._process_document_message(
+                                element
+                            )
+                            _parts.append(_part)
                         _cache_point_block = (
                             litellm.AmazonConverseConfig()._get_cache_point_block(
                                 message_block=cast(

litellm/litellm_core_utils/redact_messages.py

@@ -60,6 +60,9 @@ def _redact_choice_content(choice):
 def _redact_responses_api_output(output_items):
     """Helper to redact ResponsesAPIResponse output items."""
     for output_item in output_items:
+        if hasattr(output_item, "text"):
+            output_item.text = "redacted-by-litellm"
+
         if hasattr(output_item, "content") and isinstance(output_item.content, list):
             for content_part in output_item.content:
                 if hasattr(content_part, "text"):
@@ -75,6 +78,28 @@ def _redact_responses_api_output(output_items):
                         summary_item.text = "redacted-by-litellm"
 
 
+def _redact_responses_api_output_dict(output_items, redacted_str: str):
+    """Helper to redact ResponsesAPIResponse output items in dict form."""
+    for output_item in output_items:
+        if not isinstance(output_item, dict):
+            continue
+
+        if "text" in output_item:
+            output_item["text"] = redacted_str
+
+        if isinstance(output_item.get("content"), list):
+            for content_item in output_item["content"]:
+                if isinstance(content_item, dict) and "text" in content_item:
+                    content_item["text"] = redacted_str
+
+        if output_item.get("type") == "reasoning" and isinstance(
+            output_item.get("summary"), list
+        ):
+            for summary_item in output_item["summary"]:
+                if isinstance(summary_item, dict) and "text" in summary_item:
+                    summary_item["text"] = redacted_str
+
+
 def _redact_standard_logging_object(model_call_details: dict):
     """Redact messages and response inside standard_logging_object if present."""
     standard_logging_object = model_call_details.get("standard_logging_object")
@@ -93,28 +118,11 @@ def _redact_standard_logging_object(model_call_details: dict):
         if isinstance(response, dict) and "output" in response:
             # ResponsesAPIResponse format - redact content in output items
             if isinstance(response.get("output"), list):
-                for output_item in response["output"]:
-                    if isinstance(output_item, dict) and "content" in output_item:
-                        if isinstance(output_item["content"], list):
-                            for content_item in output_item["content"]:
-                                if (
-                                    isinstance(content_item, dict)
-                                    and "text" in content_item
-                                ):
-                                    content_item["text"] = redacted_str
+                _redact_responses_api_output_dict(response["output"], redacted_str)
         elif isinstance(response, dict) and "choices" in response:
             # ModelResponse dict format - redact content in choices
             if isinstance(response.get("choices"), list):
-                for choice in response["choices"]:
-                    if isinstance(choice, dict):
-                        if "message" in choice and isinstance(choice["message"], dict):
-                            choice["message"]["content"] = redacted_str
-                            if "audio" in choice["message"]:
-                                choice["message"]["audio"] = None
-                        elif "delta" in choice and isinstance(choice["delta"], dict):
-                            choice["delta"]["content"] = redacted_str
-                            if "audio" in choice["delta"]:
-                                choice["delta"]["audio"] = None
+                _redact_model_response_dict_choices(response["choices"], redacted_str)
         elif isinstance(response, str):
             standard_logging_object["response"] = redacted_str
         else:
@@ -122,6 +130,29 @@ def _redact_standard_logging_object(model_call_details: dict):
             standard_logging_object["response"] = {"text": redacted_str}
 
 
+def _redact_model_response_dict_choices(choices, redacted_str: str):
+    for choice in choices:
+        if isinstance(choice, dict):
+            if "message" in choice and isinstance(choice["message"], dict):
+                choice["message"]["content"] = redacted_str
+                if "reasoning_content" in choice["message"]:
+                    choice["message"]["reasoning_content"] = redacted_str
+                if "thinking_blocks" in choice["message"]:
+                    choice["message"]["thinking_blocks"] = None
+                if "audio" in choice["message"]:
+                    choice["message"]["audio"] = None
+            elif "delta" in choice and isinstance(choice["delta"], dict):
+                choice["delta"]["content"] = redacted_str
+                if "reasoning_content" in choice["delta"]:
+                    choice["delta"]["reasoning_content"] = redacted_str
+                if "thinking_blocks" in choice["delta"]:
+                    choice["delta"]["thinking_blocks"] = None
+                if "audio" in choice["delta"]:
+                    choice["delta"]["audio"] = None
+        else:
+            _redact_choice_content(choice)
+
+
 def perform_redaction(model_call_details: dict, result):
     """
     Performs the actual redaction on the logging object and result.
@@ -132,6 +163,7 @@ def perform_redaction(model_call_details: dict, result):
     ]
     model_call_details["prompt"] = ""
     model_call_details["input"] = ""
+    _redact_standard_logging_object(model_call_details)
 
     # Redact streaming response
     if (
@@ -171,30 +203,14 @@ def perform_redaction(model_call_details: dict, result):
         elif isinstance(_result, dict) and "choices" in _result:
             # Handle dict representation of ModelResponse (e.g., from model_dump())
             if _result.get("choices") is not None:
-                for choice in _result["choices"]:
-                    if isinstance(choice, dict):
-                        if "message" in choice and isinstance(choice["message"], dict):
-                            choice["message"]["content"] = "redacted-by-litellm"
-                            if "reasoning_content" in choice["message"]:
-                                choice["message"][
-                                    "reasoning_content"
-                                ] = "redacted-by-litellm"
-                            if "thinking_blocks" in choice["message"]:
-                                choice["message"]["thinking_blocks"] = None
-                            if "audio" in choice["message"]:
-                                choice["message"]["audio"] = None
-                        elif "delta" in choice and isinstance(choice["delta"], dict):
-                            choice["delta"]["content"] = "redacted-by-litellm"
-                            if "reasoning_content" in choice["delta"]:
-                                choice["delta"][
-                                    "reasoning_content"
-                                ] = "redacted-by-litellm"
-                            if "thinking_blocks" in choice["delta"]:
-                                choice["delta"]["thinking_blocks"] = None
-                            if "audio" in choice["delta"]:
-                                choice["delta"]["audio"] = None
-                    else:
-                        _redact_choice_content(choice)
+                _redact_model_response_dict_choices(
+                    _result["choices"], "redacted-by-litellm"
+                )
+        elif isinstance(_result, dict) and "output" in _result:
+            if isinstance(_result.get("output"), list):
+                _redact_responses_api_output_dict(
+                    _result["output"], "redacted-by-litellm"
+                )
         elif isinstance(_result, litellm.ResponsesAPIResponse):
             if hasattr(_result, "output"):
                 _redact_responses_api_output(_result.output)

litellm/litellm_core_utils/secret_redaction.py

@@ -0,0 +1,81 @@
+"""
+Credential/secret redaction utilities.
+
+This module owns the compiled regex and the public `redact_string` helper so
+that any part of the codebase (logging, exception mapping, etc.) can scrub
+secrets from strings without depending on the logging-configuration module.
+"""
+
+import re
+from typing import List
+
+_REDACTED = "REDACTED"
+
+
+def _build_secret_patterns() -> "re.Pattern[str]":
+    patterns: List[str] = [
+        # PEM private key / certificate blocks
+        r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----",
+        # GCP OAuth2 access tokens (ya29.*)
+        r"\bya29\.[A-Za-z0-9_.~+/-]+",
+        # Credential %s formatting (space separator, no key= prefix)
+        r"(?:client_secret|azure_password|azure_username)\s+[^\s,'\"})\]{}>]+",
+        # AWS access key IDs
+        r"(?:AKIA|ASIA)[0-9A-Z]{16}",
+        # AWS secrets / session tokens / access key IDs (key=value)
+        r"(?:aws_secret_access_key|aws_session_token|aws_access_key_id)"
+        r"\s*[:=]\s*[A-Za-z0-9/+=]{20,}",
+        # Bearer tokens (OAuth, JWT, etc.)
+        r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*",
+        # Basic auth headers
+        r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}",
+        # OpenAI / Anthropic sk- prefixed keys
+        r"sk-[A-Za-z0-9\-_]{20,}",
+        # Generic api_key / api-key / apikey (handles 'key': 'value' dict repr)
+        r"(?:api[_-]?key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]{8,}",
+        # x-api-key / api-key header values (handles 'key': 'value' dict repr)
+        r"(?:x-api-key|api-key)['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Anthropic internal header keys
+        r"x-ak-[A-Za-z0-9\-_]{20,}",
+        # Google API keys (bare key value)
+        r"AIza[0-9A-Za-z\-_]{35}",
+        # URL query-param key=VALUE (e.g. ?key=AIza... or &key=...) — catches the
+        # full "key=<secret>" fragment so the value is redacted regardless of format.
+        r"(?<=[?&])key=[^\s&'\"]{8,}",
+        # Password / secret params (handles key=value and 'key': 'value')
+        # Word boundary prevents O(n^2) backtracking on long word-char runs.
+        r"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)"
+        r"['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+",
+        # Database connection string credentials (scheme://user:pass@host)
+        r"(?<=://)[^\s'\"]*:[^\s'\"@]+(?=@)",
+        # Databricks personal access tokens
+        r"dapi[0-9a-f]{32}",
+        # ── Key-name-based redaction ──
+        # Catches secrets inside dicts/config dumps by matching on the KEY name
+        # regardless of what the value looks like.
+        # e.g. 'master_key': 'any-value-here', "database_url": "postgres://..."
+        # private_key with PEM-aware value capture
+        r"""private_key['\"]?\s*[:=]\s*['\"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'\"})\]{}>]+)""",
+        r"(?:master_key|database_url|db_url|connection_string|"
+        r"signing_key|encryption_key|"
+        r"auth_token|access_token|refresh_token|"
+        r"slack_webhook_url|webhook_url|"
+        r"database_connection_string|"
+        r"huggingface_token|jwt_secret)"
+        r"""['\"]?\s*[:=]\s*['\"]?[^\s,'\"})\]{}>]+""",
+        # Raw JWTs (without Bearer prefix)
+        r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*",
+        # Azure SAS tokens in URLs
+        r"[?&]sig=[A-Za-z0-9%+/=]+",
+        # Full JSON service-account blobs (single-line and multi-line)
+        r'\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}',
+    ]
+    return re.compile("|".join(patterns), re.IGNORECASE)
+
+
+_SECRET_RE = _build_secret_patterns()
+
+
+def redact_string(value: str) -> str:
+    """Scrub known secret/credential patterns from *value* and return the result."""
+    return _SECRET_RE.sub(_REDACTED, value)

litellm/litellm_core_utils/streaming_handler.py

@@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
                 asyncio.create_task(
                     self.logging_obj.async_failure_handler(e, traceback_exception)
                 )
-            raise e
+            self._handle_stream_fallback_error(e)
         except Exception as e:
             traceback_exception = traceback.format_exc()
             if self.logging_obj is not None:

litellm/litellm_core_utils/url_utils.py

@@ -22,7 +22,7 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
 import socket
 from ipaddress import ip_address, ip_network
 from typing import Any, List, Set, Tuple
-from urllib.parse import urlparse, urlunparse
+from urllib.parse import quote, urlparse, urlunparse
 
 import httpx
 
@@ -46,6 +46,46 @@ class SSRFError(ValueError):
     pass
 
 
+def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str:
+    """Percent-encode one user-controlled URL path segment.
+
+    ``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986
+    unreserved characters such as ``.`` unescaped, so reject standalone dot
+    segments before they can be appended to an upstream URL and normalized by
+    the HTTP client.
+    """
+    if value is None:
+        raise ValueError(f"{field_name} is required")
+
+    value_str = str(value)
+    if value_str == "":
+        raise ValueError(f"{field_name} is required")
+    if value_str in {".", ".."}:
+        raise ValueError(f"{field_name} cannot be a dot path segment")
+
+    return quote(value_str, safe="")
+
+
+def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str:
+    """Percent-encode a user-controlled URL path made of multiple segments.
+
+    Empty segments are rejected, so leading, trailing, or consecutive slashes
+    fail closed instead of being normalized by the HTTP client.
+    """
+    if value is None:
+        raise ValueError(f"{field_name} is required")
+
+    value_str = str(value)
+    if value_str == "":
+        raise ValueError(f"{field_name} is required")
+
+    encoded_segments = []
+    for segment in value_str.split("/"):
+        encoded_segments.append(encode_url_path_segment(segment, field_name=field_name))
+
+    return "/".join(encoded_segments)
+
+
 def _is_blocked_ip(addr: str) -> bool:
     """Return True for any IP not safe to reach from a user-supplied URL.
 
@@ -199,6 +239,47 @@ def validate_url(url: str) -> Tuple[str, str]:
     return rewritten, host_header
 
 
+def assert_same_origin(candidate_url: str, expected_url: str) -> None:
+    """Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``.
+
+    Use when an upstream API returns a URL meant for follow-up requests
+    (e.g. an async-job polling URL that will be hit with the operator's
+    API key in the headers). The upstream is trusted because the operator
+    configured ``api_base``, but the URL it hands back must actually point
+    back at the same origin or we'd be blindly forwarding credentials
+    wherever the upstream told us to.
+
+    Hostnames are compared case-insensitively. Default ports are made
+    explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...``
+    and ``https://api.example.com/...`` are treated as the same origin.
+
+    Error messages identify *which* component mismatched but never echo
+    the operator's ``expected`` host or the candidate's hostname back to
+    the caller — in the SSRF threat model the caller is the attacker,
+    and reflecting host info would be a secondary leak of operator
+    infrastructure details.
+    """
+    candidate = urlparse(candidate_url)
+    expected = urlparse(expected_url)
+
+    if candidate.scheme not in _ALLOWED_SCHEMES:
+        raise SSRFError("URL scheme is not allowed")
+
+    if candidate.scheme != expected.scheme:
+        raise SSRFError("Origin mismatch on scheme")
+
+    candidate_host = _normalize_host(candidate.hostname or "")
+    expected_host = _normalize_host(expected.hostname or "")
+    if not candidate_host or candidate_host != expected_host:
+        raise SSRFError("Origin mismatch on host")
+
+    default_port = 443 if candidate.scheme == "https" else 80
+    candidate_port = candidate.port if candidate.port is not None else default_port
+    expected_port = expected.port if expected.port is not None else default_port
+    if candidate_port != expected_port:
+        raise SSRFError("Origin mismatch on port")
+
+
 _MAX_REDIRECTS = 10
 
 

litellm/llms/anthropic/batches/transformation.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cas
 import httpx
 from httpx import Headers, Response
 
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
@@ -122,7 +123,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
             Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id}
         """
         api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
-        return f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}"
+        encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
+        return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}"
 
     def transform_retrieve_batch_request(
         self,

litellm/llms/anthropic/chat/transformation.py

@@ -1553,25 +1553,43 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             )
         data["output_config"] = output_config
 
-    def _transform_response_for_json_mode(
+    def _resolve_json_mode_non_streaming(
         self,
         json_mode: Optional[bool],
         tool_calls: List[ChatCompletionToolCallChunk],
-    ) -> Optional[LitellmMessage]:
-        _message: Optional[LitellmMessage] = None
-        if json_mode is True and len(tool_calls) == 1:
-            # check if tool name is the default tool name
-            json_mode_content_str: Optional[str] = None
-            if (
-                "name" in tool_calls[0]["function"]
-                and tool_calls[0]["function"]["name"] == RESPONSE_FORMAT_TOOL_NAME
-            ):
-                json_mode_content_str = tool_calls[0]["function"].get("arguments")
-            if json_mode_content_str is not None:
-                _message = AnthropicConfig._convert_tool_response_to_message(
-                    tool_calls=tool_calls,
-                )
-        return _message
+    ) -> Tuple[
+        Optional[LitellmMessage],
+        List[ChatCompletionToolCallChunk],
+        Optional[str],
+    ]:
+        """Strip internal response_format tool calls; merge payload into content when mixed with user tools."""
+        if json_mode is not True or not tool_calls:
+            return None, tool_calls, None
+
+        json_indices = [
+            i
+            for i, t in enumerate(tool_calls)
+            if t.get("function", {}).get("name") == RESPONSE_FORMAT_TOOL_NAME
+        ]
+        if not json_indices:
+            return None, tool_calls, None
+
+        if len(json_indices) == len(tool_calls):
+            json_tool = tool_calls[json_indices[0]]
+            if json_tool.get("function", {}).get("arguments") is None:
+                return None, tool_calls, None
+            _message = AnthropicConfig._convert_tool_response_to_message(
+                tool_calls=[json_tool]
+            )
+            return _message, [], None
+
+        first_json = tool_calls[json_indices[0]]
+        json_msg = AnthropicConfig._convert_tool_response_to_message([first_json])
+        extra_content: Optional[str] = (
+            json_msg.content if json_msg is not None else None
+        )
+        filtered_tools = [t for i, t in enumerate(tool_calls) if i not in json_indices]
+        return None, filtered_tools, extra_content
 
     def extract_response_content(self, completion_response: dict) -> Tuple[
         str,
@@ -1931,19 +1949,27 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig):
             tool_calls,
         )
 
+        json_mode_message, tool_calls_for_message, json_extra_content = (
+            self._resolve_json_mode_non_streaming(
+                json_mode=json_mode,
+                tool_calls=tool_calls,
+            )
+        )
+        merged_text = text_content or ""
+        if json_extra_content:
+            merged_text = (
+                merged_text + json_extra_content if merged_text else json_extra_content
+            )
+
         _message = litellm.Message(
-            tool_calls=tool_calls,
-            content=text_content or None,
+            tool_calls=tool_calls_for_message,
+            content=merged_text or None,
             provider_specific_fields=provider_specific_fields,
             thinking_blocks=thinking_blocks,
             reasoning_content=reasoning_content,
         )
         _message.provider_specific_fields = provider_specific_fields
 
-        json_mode_message = self._transform_response_for_json_mode(
-            json_mode=json_mode,
-            tool_calls=tool_calls,
-        )
         if json_mode_message is not None:
             completion_response["stop_reason"] = "stop"
             _message = json_mode_message

litellm/llms/anthropic/experimental_pass_through/adapters/handler.py

@@ -27,6 +27,16 @@ from litellm.utils import get_model_info
 if TYPE_CHECKING:
     pass
 
+
+# Anthropic-only fields that the translator above already maps into the
+# OpenAI-format completion_kwargs (output_config → reasoning_effort /
+# response_format, etc.). They must be filtered out of the raw
+# extra_kwargs re-merge below or non-Anthropic backends reject the call
+# with 400 "Extra inputs are not permitted". Add new entries here when
+# extending AnthropicMessagesRequestOptionalParams with another Anthropic-
+# specific key.
+ANTHROPIC_ONLY_REQUEST_KEYS: frozenset[str] = frozenset({"output_config"})
+
 ########################################################
 # init adapter
 ANTHROPIC_ADAPTER = AnthropicAdapter()
@@ -202,8 +212,12 @@ class LiteLLMMessagesToCompletionTransformationHandler:
             request_data["output_format"] = output_format
 
         # Extract output_config from extra_kwargs so the translator can use it
-        # (e.g. output_config.effort for adaptive thinking → reasoning_effort)
-        extra_kwargs = extra_kwargs or {}
+        # (e.g. output_config.effort for adaptive thinking → reasoning_effort,
+        # output_config.format → response_format for structured outputs).
+        # Use explicit None check rather than `or {}` so an explicit empty dict
+        # caller-passed argument is preserved (matters for tests that drive
+        # the fallback inference path).
+        extra_kwargs = extra_kwargs if extra_kwargs is not None else {}
         if "output_config" in extra_kwargs:
             request_data["output_config"] = extra_kwargs["output_config"]
 
@@ -225,8 +239,23 @@ class LiteLLMMessagesToCompletionTransformationHandler:
                 "include_usage": True,
             }
 
-        excluded_keys = {"anthropic_messages"}
-        extra_kwargs = extra_kwargs or {}
+        # Keys that must NOT be forwarded as raw extras into the OpenAI-format
+        # ``completion_kwargs`` after translation. The translator above has
+        # already consumed the meaningful parts of these inputs (e.g.
+        # ``output_config.format`` → ``response_format``, ``output_config.effort``
+        # → ``reasoning_effort`` for non-Claude targets). Re-adding the raw
+        # Anthropic-shaped key here causes 400 "Extra inputs are not permitted"
+        # on non-Anthropic backends (Azure OpenAI, Fireworks, Bedrock Nova,
+        # etc.) and is silently lossy on Anthropic-family targets, which would
+        # see the translated key ``response_format`` AND a duplicate, conflicting
+        # ``output_config``.
+        #
+        # Maintainability: when adding a new Anthropic-only request param to
+        # ``AnthropicMessagesRequestOptionalParams``, also extend
+        # ``ANTHROPIC_ONLY_REQUEST_KEYS`` here so it doesn't silently leak.
+        excluded_keys = ANTHROPIC_ONLY_REQUEST_KEYS | {"anthropic_messages"}
+        # NOTE: extra_kwargs was already coerced from None to {} at the top of
+        # this method (line ~220). It is guaranteed to be a dict here.
         for key, value in extra_kwargs.items():
             if (
                 key == "litellm_logging_obj"

litellm/llms/anthropic/experimental_pass_through/adapters/transformation.py

@@ -667,7 +667,7 @@ class LiteLLMAnthropicMessagesAdapter:
 
     @staticmethod
     def translate_anthropic_thinking_to_reasoning_effort(
-        thinking: Dict[str, Any]
+        thinking: Dict[str, Any],
     ) -> Optional[str]:
         """
         Translate Anthropic's thinking parameter to OpenAI's reasoning_effort.
@@ -1084,10 +1084,23 @@ class LiteLLMAnthropicMessagesAdapter:
         anthropic_message_request: AnthropicMessagesRequest,
         new_kwargs: ChatCompletionRequest,
     ) -> None:
-        """Translate output_format to response_format when applicable."""
-        if "output_format" not in anthropic_message_request:
-            return
-        output_format = anthropic_message_request["output_format"]
+        """Translate Anthropic structured-output config to OpenAI ``response_format``.
+
+        Accepts either the legacy top-level ``output_format`` field OR the
+        newer ``output_config.format`` (sub-key on ``output_config``) so that
+        both shapes flow through to non-Anthropic backends as
+        ``response_format``. Without the ``output_config.format`` branch,
+        callers using the new Anthropic Structured Outputs API would have
+        their schema silently dropped on the adapter path — only the legacy
+        top-level ``output_format`` was being mapped.
+
+        ``output_format`` takes precedence when both are provided.
+        """
+        output_format: Any = anthropic_message_request.get("output_format")
+        if not output_format:
+            output_config = anthropic_message_request.get("output_config")
+            if isinstance(output_config, dict):
+                output_format = output_config.get("format")
         if not output_format:
             return
         response_format = self.translate_anthropic_output_format_to_openai(

litellm/llms/anthropic/files/handler.py

@@ -9,6 +9,7 @@ import litellm
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
 from litellm.litellm_core_utils.litellm_logging import Logging
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
 from litellm.types.llms.openai import (
     FileContentRequest,
@@ -89,7 +90,10 @@ class AnthropicFilesHandler:
             raise ValueError("Missing Anthropic API Key")
 
         # Construct the Anthropic batch results URL
-        results_url = f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}/results"
+        encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
+        results_url = (
+            f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results"
+        )
 
         # Prepare headers
         headers = {

litellm/llms/anthropic/files/transformation.py

@@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union, cast
 import httpx
 from openai.types.file_deleted import FileDeleted
 
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.transformation import (
@@ -185,7 +186,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
             AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
             or ANTHROPIC_FILES_API_BASE
         )
-        return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
 
     def transform_retrieve_file_response(
         self,
@@ -206,7 +208,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
             AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
             or ANTHROPIC_FILES_API_BASE
         )
-        return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
 
     def transform_delete_file_response(
         self,
@@ -268,7 +271,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
             AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
             or ANTHROPIC_FILES_API_BASE
         )
-        return f"{api_base.rstrip('/')}/v1/files/{file_id}/content", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
 
     def transform_file_content_response(
         self,

litellm/llms/anthropic/skills/transformation.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
 import httpx
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.skills.transformation import (
     BaseSkillsAPIConfig,
     LiteLLMLoggingObj,
@@ -81,7 +82,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
             api_base = AnthropicModelInfo.get_api_base()
 
         if skill_id:
-            return f"{api_base}/v1/skills/{skill_id}"
+            encoded_skill_id = encode_url_path_segment(skill_id, field_name="skill_id")
+            return f"{api_base}/v1/skills/{encoded_skill_id}"
         return f"{api_base}/v1/{endpoint}"
 
     def transform_create_skill_request(

litellm/llms/azure/azure.py

@@ -16,6 +16,7 @@ import litellm
 from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
+from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -792,6 +793,7 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                     client=client,
                     litellm_params=litellm_params,
                     api_base=api_base,
+                    api_version=api_version,
                 )
             azure_client = self.get_azure_openai_client(
                 api_version=api_version,
@@ -898,6 +900,17 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 operation_location_url = response.headers["operation-location"]
             else:
                 raise AzureOpenAIError(status_code=500, message=response.text)
+            # Reject polling URLs that don't share an origin with ``api_base``.
+            # Without this an upstream-controlled or attacker-controlled
+            # value would receive the operator's Azure API key in the
+            # request headers below. VERIA-51.
+            try:
+                assert_same_origin(operation_location_url, api_base)
+            except SSRFError as ssrf_err:
+                raise AzureOpenAIError(
+                    status_code=502,
+                    message=f"Rejected polling URL: {ssrf_err}",
+                )
             response = await async_handler.get(
                 url=operation_location_url,
                 headers=headers,
@@ -908,8 +921,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
             start_time = time.time()
             if "status" not in response.json():
-                raise Exception(
-                    "Expected 'status' in response. Got={}".format(response.json())
+                # Don't reflect the raw response body — when the polling
+                # URL points at an internal JSON API (cloud metadata
+                # service etc.) reflecting it here turns Blind SSRF into
+                # Full-Read SSRF. VERIA-51.
+                raise AzureOpenAIError(
+                    status_code=502,
+                    message="Polling response missing 'status' field",
                 )
             while response.json()["status"] not in ["succeeded", "failed"]:
                 if time.time() - start_time > timeout_secs:
@@ -1009,6 +1027,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
                 operation_location_url = response.headers["operation-location"]
             else:
                 raise AzureOpenAIError(status_code=500, message=response.text)
+            try:
+                assert_same_origin(operation_location_url, api_base)
+            except SSRFError as ssrf_err:
+                raise AzureOpenAIError(
+                    status_code=502,
+                    message=f"Rejected polling URL: {ssrf_err}",
+                )
             response = sync_handler.get(
                 url=operation_location_url,
                 headers=headers,
@@ -1019,8 +1044,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
             timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
             start_time = time.time()
             if "status" not in response.json():
-                raise Exception(
-                    "Expected 'status' in response. Got={}".format(response.json())
+                raise AzureOpenAIError(
+                    status_code=502,
+                    message="Polling response missing 'status' field",
                 )
             while response.json()["status"] not in ["succeeded", "failed"]:
                 if time.time() - start_time > timeout_secs:

litellm/llms/azure/cost_calculation.py

@@ -12,7 +12,10 @@ from litellm.utils import get_model_info
 
 
 def cost_per_token(
-    model: str, usage: Usage, response_time_ms: Optional[float] = 0.0
+    model: str,
+    usage: Usage,
+    response_time_ms: Optional[float] = 0.0,
+    service_tier: Optional[str] = None,
 ) -> Tuple[float, float]:
     """
     Calculates the cost per token for a given model, prompt tokens, and completion tokens.
@@ -47,4 +50,5 @@ def cost_per_token(
         model=model,
         usage=usage,
         custom_llm_provider="azure",
+        service_tier=service_tier,
     )

litellm/llms/azure/responses/transformation.py

@@ -5,6 +5,7 @@ import httpx
 from openai.types.responses import ResponseReasoningItem
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure.common_utils import BaseAzureLLM
 from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
 from litellm.types.llms.openai import *
@@ -201,7 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         # Insert the response_id at the end of the path component
         # Remove trailing slash if present to avoid double slashes
         path = parsed_url.path.rstrip("/")
-        new_path = f"{path}/{response_id}"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        new_path = f"{path}/{encoded_response_id}"
 
         # Reconstruct the URL with all original components but with the modified path
         constructed_url = urlunparse(
@@ -322,7 +326,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
         # Insert the response_id and /cancel at the end of the path component
         # Remove trailing slash if present to avoid double slashes
         path = parsed_url.path.rstrip("/")
-        new_path = f"{path}/{response_id}/cancel"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        new_path = f"{path}/{encoded_response_id}/cancel"
 
         # Reconstruct the URL with all original components but with the modified path
         cancel_url = urlunparse(

litellm/llms/azure_ai/agents/handler.py

@@ -36,6 +36,7 @@ from typing import (
 import httpx
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.azure_ai.agents.transformation import (
     AzureAIAgentsConfig,
     AzureAIAgentsError,
@@ -75,20 +76,29 @@ class AzureAIAgentsHandler:
     def _build_messages_url(
         self, api_base: str, thread_id: str, api_version: str
     ) -> str:
-        return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
+        encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
+        return (
+            f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
+        )
 
     def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
-        return f"{api_base}/threads/{thread_id}/runs?api-version={api_version}"
+        encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
+        return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}"
 
     def _build_run_status_url(
         self, api_base: str, thread_id: str, run_id: str, api_version: str
     ) -> str:
-        return f"{api_base}/threads/{thread_id}/runs/{run_id}?api-version={api_version}"
+        encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
+        encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
+        return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}"
 
     def _build_list_messages_url(
         self, api_base: str, thread_id: str, api_version: str
     ) -> str:
-        return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
+        encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
+        return (
+            f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
+        )
 
     def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str:
         """URL for the create-thread-and-run endpoint (supports streaming)."""

litellm/llms/azure_ai/cost_calculator.py

@@ -65,6 +65,7 @@ def cost_per_token(
     usage: Usage,
     response_time_ms: Optional[float] = 0.0,
     request_model: Optional[str] = None,
+    service_tier: Optional[str] = None,
 ) -> Tuple[float, float]:
     """
     Calculate the cost per token for Azure AI models.
@@ -102,6 +103,7 @@ def cost_per_token(
             model=model,
             usage=usage,
             custom_llm_provider="azure_ai",
+            service_tier=service_tier,
         )
     except Exception as e:
         # For Model Router, the model name (e.g., "azure-model-router") may not be in the cost map

litellm/llms/azure_ai/ocr/document_intelligence/transformation.py

@@ -17,11 +17,13 @@ from urllib.parse import quote
 import httpx
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.constants import (
     AZURE_DOCUMENT_INTELLIGENCE_API_VERSION,
     AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI,
     AZURE_OPERATION_POLLING_TIMEOUT,
 )
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.ocr.transformation import (
     BaseOCRConfig,
     DocumentType,
@@ -217,11 +219,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
         if "/" in model:
             # Extract the last part after the last slash
             model_id = model.split("/")[-1]
+        encoded_model_id = encode_url_path_segment(model_id, field_name="model_id")
 
         # Azure Document Intelligence analyze endpoint
         # Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/)
         url = (
-            f"{api_base}/documentintelligence/documentModels/{model_id}:analyze"
+            f"{api_base}/documentintelligence/documentModels/{encoded_model_id}:analyze"
             f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
         )
 
@@ -599,6 +602,16 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
                         "Azure Document Intelligence returned 202 but no Operation-Location header found"
                     )
 
+                # Reject cross-origin polling URLs — the auth headers
+                # below would otherwise leak to whatever URL the upstream
+                # (or an attacker-controlled upstream) returns. VERIA-51.
+                try:
+                    assert_same_origin(operation_url, str(raw_response.request.url))
+                except SSRFError as ssrf_err:
+                    raise ValueError(
+                        f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
+                    )
+
                 # Get headers for polling (need auth)
                 poll_headers = {
                     "Ocp-Apim-Subscription-Key": raw_response.request.headers.get(
@@ -711,6 +724,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
                         "Azure Document Intelligence returned 202 but no Operation-Location header found"
                     )
 
+                # Reject cross-origin polling URLs (see sync path). VERIA-51.
+                try:
+                    assert_same_origin(operation_url, str(raw_response.request.url))
+                except SSRFError as ssrf_err:
+                    raise ValueError(
+                        f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
+                    )
+
                 # Get headers for polling (need auth)
                 poll_headers = {
                     "Ocp-Apim-Subscription-Key": raw_response.request.headers.get(

litellm/llms/base_llm/rerank/transformation.py

@@ -33,6 +33,7 @@ class BaseRerankConfig(ABC):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         return {}
 

litellm/llms/bedrock/base_aws_llm.py

@@ -1,6 +1,7 @@
 import hashlib
 import json
 import os
+import re
 import urllib.parse
 from datetime import datetime
 from typing import (
@@ -37,6 +38,11 @@ else:
     AWSPreparedRequest = Any
 
 
+# Real AWS region names are lowercase letters, digits, and hyphens
+# (e.g. "us-east-1", "eu-west-2", "us-gov-west-1", "cn-north-1").
+_VALID_AWS_REGION_PATTERN = re.compile(r"\A[a-z0-9-]+\Z")
+
+
 class Boto3CredentialsInfo(BaseModel):
     credentials: Credentials
     aws_region_name: str
@@ -284,6 +290,9 @@ class BaseAWSLLM:
             if not region:  # Check if region is empty
                 return None
 
+            if not _VALID_AWS_REGION_PATTERN.match(region):
+                return None
+
             return region
         except Exception:
             # Catch any unexpected errors and return None
@@ -481,6 +490,7 @@ class BaseAWSLLM:
             str: The AWS region name
         """
         aws_region_name = optional_params.get("aws_region_name", None)
+        self._validate_aws_region_name(aws_region_name)
         ### SET REGION NAME ###
         if aws_region_name is None:
             # check model arn #
@@ -519,8 +529,25 @@ class BaseAWSLLM:
             except Exception:
                 aws_region_name = "us-west-2"
 
+        self._validate_aws_region_name(aws_region_name)
         return aws_region_name
 
+    @staticmethod
+    def _validate_aws_region_name(aws_region_name: Optional[str]) -> None:
+        """
+        Validate that an AWS region name conforms to the expected format
+        (lowercase alphanumerics and hyphens). Raises ValueError otherwise.
+        """
+        if aws_region_name is None:
+            return
+        if not isinstance(aws_region_name, str) or not _VALID_AWS_REGION_PATTERN.match(
+            aws_region_name
+        ):
+            raise ValueError(
+                f"Invalid AWS region format: {aws_region_name!r}. "
+                "Region names must contain only lowercase letters, digits, and hyphens."
+            )
+
     def get_aws_region_name_for_non_llm_api_calls(
         self,
         aws_region_name: Optional[str] = None,
@@ -532,6 +559,7 @@ class BaseAWSLLM:
 
         For non-llm api calls eg. Guardrails, Vector Stores we just need to check the dynamic param or env vars.
         """
+        self._validate_aws_region_name(aws_region_name)
         if aws_region_name is None:
             # check env #
             litellm_aws_region_name = get_secret("AWS_REGION_NAME", None)
@@ -549,6 +577,8 @@ class BaseAWSLLM:
 
             if aws_region_name is None:
                 aws_region_name = "us-west-2"
+
+        self._validate_aws_region_name(aws_region_name)
         return aws_region_name
 
     @staticmethod

litellm/llms/bedrock/chat/invoke_agent/transformation.py

@@ -12,6 +12,7 @@ import httpx
 
 from litellm._logging import verbose_logger
 from litellm._uuid import uuid
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.prompt_templates.common_utils import (
     convert_content_list_to_str,
 )
@@ -97,8 +98,15 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
 
         agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model)
         session_id = self._get_session_id(optional_params)
+        encoded_agent_id = encode_url_path_segment(agent_id, field_name="agent_id")
+        encoded_agent_alias_id = encode_url_path_segment(
+            agent_alias_id, field_name="agent_alias_id"
+        )
+        encoded_session_id = encode_url_path_segment(
+            session_id, field_name="session_id"
+        )
 
-        endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text"
+        endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text"
 
         return endpoint_url
 

litellm/llms/bedrock/count_tokens/transformation.py

@@ -201,13 +201,14 @@ class BedrockCountTokensConfig(BaseAWSLLM):
         # Remove bedrock/ prefix if present
         if model_id.startswith("bedrock/"):
             model_id = model_id[8:]  # Remove "bedrock/" prefix
+        encoded_model_id = self.encode_model_id(model_id=model_id)
 
         base_url, _ = self.get_runtime_endpoint(
             api_base=api_base,
             aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
             aws_region_name=aws_region_name,
         )
-        endpoint = f"{base_url}/model/{model_id}/count-tokens"
+        endpoint = f"{base_url}/model/{encoded_model_id}/count-tokens"
 
         return endpoint
 

litellm/llms/bedrock/vector_stores/transformation.py

@@ -5,6 +5,7 @@ from urllib.parse import urlparse
 import httpx
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
 from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
 from litellm.types.integrations.rag.bedrock_knowledgebase import (
@@ -209,7 +210,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
         if isinstance(query, list):
             query = " ".join(query)
 
-        url = f"{api_base}/{vector_store_id}/retrieve"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}/retrieve"
 
         request_body: Dict[str, Any] = {
             "retrievalQuery": BedrockKBRetrievalQuery(text=query),

litellm/llms/black_forest_labs/image_edit/handler.py

@@ -15,6 +15,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -331,6 +332,17 @@ class BlackForestLabsImageEdit:
                 message="No polling_url in BFL response",
             )
 
+        # Reject cross-origin polling URLs — the ``x-key`` auth header
+        # would otherwise leak to whatever URL the upstream returns.
+        # VERIA-51.
+        try:
+            assert_same_origin(polling_url, str(initial_response.request.url))
+        except SSRFError as ssrf_err:
+            raise BlackForestLabsError(
+                status_code=502,
+                message=f"Rejected polling URL: {ssrf_err}",
+            )
+
         # Get just the auth header for polling
         polling_headers = {"x-key": headers.get("x-key", "")}
 
@@ -416,6 +428,17 @@ class BlackForestLabsImageEdit:
                 message="No polling_url in BFL response",
             )
 
+        # Reject cross-origin polling URLs — the ``x-key`` auth header
+        # would otherwise leak to whatever URL the upstream returns.
+        # VERIA-51.
+        try:
+            assert_same_origin(polling_url, str(initial_response.request.url))
+        except SSRFError as ssrf_err:
+            raise BlackForestLabsError(
+                status_code=502,
+                message=f"Rejected polling URL: {ssrf_err}",
+            )
+
         # Get just the auth header for polling
         polling_headers = {"x-key": headers.get("x-key", "")}
 

litellm/llms/black_forest_labs/image_generation/handler.py

@@ -15,6 +15,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -317,6 +318,17 @@ class BlackForestLabsImageGeneration:
                 message="No polling_url in BFL response",
             )
 
+        # Reject cross-origin polling URLs — the ``x-key`` auth header
+        # would otherwise leak to whatever URL the upstream returns.
+        # VERIA-51.
+        try:
+            assert_same_origin(polling_url, str(initial_response.request.url))
+        except SSRFError as ssrf_err:
+            raise BlackForestLabsError(
+                status_code=502,
+                message=f"Rejected polling URL: {ssrf_err}",
+            )
+
         # Get just the auth header for polling
         polling_headers = {"x-key": headers.get("x-key", "")}
 
@@ -402,6 +414,17 @@ class BlackForestLabsImageGeneration:
                 message="No polling_url in BFL response",
             )
 
+        # Reject cross-origin polling URLs — the ``x-key`` auth header
+        # would otherwise leak to whatever URL the upstream returns.
+        # VERIA-51.
+        try:
+            assert_same_origin(polling_url, str(initial_response.request.url))
+        except SSRFError as ssrf_err:
+            raise BlackForestLabsError(
+                status_code=502,
+                message=f"Rejected polling URL: {ssrf_err}",
+            )
+
         # Get just the auth header for polling
         polling_headers = {"x-key": headers.get("x-key", "")}
 

litellm/llms/bytez/chat/transformation.py

@@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
 
 import httpx
 
+from litellm.litellm_core_utils.url_utils import encode_url_path_segments
 from litellm.litellm_core_utils.exception_mapping_utils import exception_type
 from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@@ -149,7 +150,8 @@ class BytezChatConfig(BaseConfig):
         litellm_params: dict,
         stream: Optional[bool] = None,
     ) -> str:
-        return f"{API_BASE}/{model}"
+        encoded_model = encode_url_path_segments(model, field_name="model")
+        return f"{API_BASE}/{encoded_model}"
 
     def transform_request(
         self,

litellm/llms/cloudflare/chat/transformation.py

@@ -5,6 +5,7 @@ from typing import AsyncIterator, Iterator, List, Optional, Union
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segments
 from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.chat.transformation import (
     BaseConfig,
@@ -89,7 +90,8 @@ class CloudflareChatConfig(BaseConfig):
             api_base = (
                 f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
             )
-        return api_base + model
+        encoded_model = encode_url_path_segments(model, field_name="model")
+        return api_base + encoded_model
 
     def get_supported_openai_params(self, model: str) -> List[str]:
         return [
@@ -147,9 +149,9 @@ class CloudflareChatConfig(BaseConfig):
     ) -> ModelResponse:
         completion_response = raw_response.json()
 
-        model_response.choices[0].message.content = completion_response["result"][  # type: ignore
-            "response"
-        ]
+        # Support both "response" and "response_text" keys (newer models like Nemotron use "response_text")
+        result = completion_response["result"]
+        model_response.choices[0].message.content = result.get("response") if result.get("response") is not None else result.get("response_text", "")  # type: ignore
 
         prompt_tokens = litellm.utils.get_token_count(messages=messages, model=model)
         completion_tokens = len(
@@ -199,8 +201,10 @@ class CloudflareChatResponseIterator(BaseModelResponseIterator):
 
             index = int(chunk.get("index", 0))
 
-            if "response" in chunk:
+            if "response" in chunk and chunk["response"] is not None:
                 text = chunk["response"]
+            elif "response_text" in chunk and chunk["response_text"] is not None:
+                text = chunk["response_text"]
 
             returned_chunk = GenericStreamingChunk(
                 text=text,

litellm/llms/cohere/rerank/transformation.py

@@ -111,6 +111,7 @@ class CohereRerankConfig(BaseRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         if "query" not in optional_rerank_params:
             raise ValueError("query is required for Cohere rerank")

litellm/llms/cohere/rerank_v2/transformation.py

@@ -71,6 +71,7 @@ class CohereRerankV2Config(CohereRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         if "query" not in optional_rerank_params:
             raise ValueError("query is required for Cohere rerank")

litellm/llms/custom_httpx/container_handler.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.custom_httpx.http_handler import (
     AsyncHTTPHandler,
     HTTPHandler,
@@ -72,7 +73,8 @@ def _build_url(
 
     # Substitute path parameters
     for param, value in path_params.items():
-        path_template = path_template.replace(f"{{{param}}}", value)
+        encoded_value = encode_url_path_segment(value, field_name=param)
+        path_template = path_template.replace(f"{{{param}}}", encoded_value)
 
     # Parse the api_base to extract existing query params
     parsed_base = httpx.URL(api_base)

litellm/llms/custom_httpx/llm_http_handler.py

@@ -26,6 +26,7 @@ from litellm._logging import _redact_string, verbose_logger
 from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
 from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
 from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.anthropic_messages.transformation import (
     BaseAnthropicMessagesConfig,
 )
@@ -1007,6 +1008,7 @@ class BaseLLMHTTPHandler:
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
         client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None,
+        litellm_params: Optional[Dict[str, Any]] = None,
     ) -> RerankResponse:
         # get config from model, custom llm provider
         headers = provider_config.validate_environment(
@@ -1026,6 +1028,7 @@ class BaseLLMHTTPHandler:
             model=model,
             optional_rerank_params=optional_rerank_params,
             headers=headers,
+            litellm_params=litellm_params,
         )
 
         ## LOGGING
@@ -2535,10 +2538,16 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        delete_kwargs: Dict[str, Any] = {
+            "url": url,
+            "headers": headers,
+            "timeout": timeout,
+        }
+        if data:
+            delete_kwargs["json"] = data
+
         try:
-            response = await async_httpx_client.delete(
-                url=url, headers=headers, json=data, timeout=timeout
-            )
+            response = await async_httpx_client.delete(**delete_kwargs)
 
         except Exception as e:
             raise self._handle_error(
@@ -2619,10 +2628,16 @@ class BaseLLMHTTPHandler:
             },
         )
 
+        delete_kwargs: Dict[str, Any] = {
+            "url": url,
+            "headers": headers,
+            "timeout": timeout,
+        }
+        if data:
+            delete_kwargs["json"] = data
+
         try:
-            response = sync_httpx_client.delete(
-                url=url, headers=headers, json=data, timeout=timeout
-            )
+            response = sync_httpx_client.delete(**delete_kwargs)
 
         except Exception as e:
             raise self._handle_error(
@@ -8934,7 +8949,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         logging_obj.pre_call(
             input="",
@@ -9001,7 +9019,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         logging_obj.pre_call(
             input="",
@@ -9200,7 +9221,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
 
@@ -9283,7 +9307,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
 
@@ -9349,7 +9376,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         logging_obj.pre_call(
             input="",
@@ -9414,7 +9444,10 @@ class BaseLLMHTTPHandler:
             litellm_params=dict(litellm_params),
         )
 
-        url = f"{api_base}/{vector_store_id}"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}"
 
         logging_obj.pre_call(
             input="",

litellm/llms/deepinfra/rerank/transformation.py

@@ -132,6 +132,7 @@ class DeepinfraRerankConfig(BaseRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         # Convert OptionalRerankParams to dict as expected by parent class
         if optional_rerank_params is None:

litellm/llms/elevenlabs/text_to_speech/transformation.py

@@ -11,13 +11,14 @@ import httpx
 from httpx import Headers
 
 import litellm
-from litellm.types.utils import all_litellm_params
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.text_to_speech.transformation import (
     BaseTextToSpeechConfig,
     TextToSpeechRequestData,
 )
 from litellm.secret_managers.main import get_secret_str
+from litellm.types.utils import all_litellm_params
 
 from ..common_utils import ElevenLabsException
 
@@ -321,7 +322,8 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
                 "ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`."
             )
 
-        url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}"
+        encoded_voice_id = encode_url_path_segment(voice_id, field_name="voice_id")
+        url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}"
 
         query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {})
         if query_params:

litellm/llms/fireworks_ai/rerank/transformation.py

@@ -127,6 +127,7 @@ class FireworksAIRerankConfig(FireworksAIMixin, BaseRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         """
         Transform request to Fireworks AI rerank format

litellm/llms/gemini/files/transformation.py

@@ -12,6 +12,7 @@ import httpx
 from openai.types.file_deleted import FileDeleted
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.files.transformation import (
     BaseFilesConfig,
@@ -258,10 +259,14 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
             normalized_file_id = file_id
 
         normalized_file_id = normalized_file_id.strip("/")
-        if not normalized_file_id.startswith("files/"):
-            normalized_file_id = f"files/{normalized_file_id}"
+        if normalized_file_id.startswith("files/"):
+            normalized_file_id = normalized_file_id.removeprefix("files/")
 
-        return normalized_file_id
+        encoded_file_id = encode_url_path_segment(
+            normalized_file_id, field_name="file_id"
+        )
+
+        return f"files/{encoded_file_id}"
 
     def transform_retrieve_file_response(
         self,
@@ -337,13 +342,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
         if not api_key:
             raise ValueError("api_key is required")
 
-        # Extract file name from URI if full URI is provided
-        # file_id could be "files/abc123" or "https://generativelanguage.googleapis.com/v1beta/files/abc123"
-        if file_id.startswith("http"):
-            # Extract the file path from full URI
-            file_name = file_id.split("/v1beta/")[-1]
-        else:
-            file_name = file_id if file_id.startswith("files/") else f"files/{file_id}"
+        # Normalize and encode the file name before interpolating it into the URL.
+        file_name = self._normalize_gemini_file_id(file_id)
 
         # Construct the delete URL
         url = f"{api_base}/v1beta/{file_name}"

litellm/llms/gemini/interactions/transformation.py

@@ -15,6 +15,7 @@ import httpx
 
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
 from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
 from litellm.types.interactions import (
@@ -205,8 +206,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
         resolved_api_base = GeminiModelInfo.get_api_base(api_base)
         if not GeminiModelInfo.get_api_key(litellm_params.api_key):
             raise ValueError("Google API key is required")
+        encoded_interaction_id = encode_url_path_segment(
+            interaction_id, field_name="interaction_id"
+        )
         return (
-            f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
+            f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
             {},
         )
 
@@ -238,8 +242,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
         resolved_api_base = GeminiModelInfo.get_api_base(api_base)
         if not GeminiModelInfo.get_api_key(litellm_params.api_key):
             raise ValueError("Google API key is required")
+        encoded_interaction_id = encode_url_path_segment(
+            interaction_id, field_name="interaction_id"
+        )
         return (
-            f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
+            f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
             {},
         )
 
@@ -268,8 +275,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
         resolved_api_base = GeminiModelInfo.get_api_base(api_base)
         if not GeminiModelInfo.get_api_key(litellm_params.api_key):
             raise ValueError("Google API key is required")
+        encoded_interaction_id = encode_url_path_segment(
+            interaction_id, field_name="interaction_id"
+        )
         return (
-            f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel",
+            f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}:cancel",
             {},
         )
 

litellm/llms/hosted_vllm/embedding/README.md

@@ -2,4 +2,15 @@ No transformation is required for hosted_vllm embedding.
 
 VLLM is a superset of OpenAI's `embedding` endpoint.
 
-To pass provider-specific parameters, see [this](https://docs.litellm.ai/docs/completion/provider_specific_params)
+## `encoding_format`
+
+For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request:
+
+1. Explicit value on the embedding call (`encoding_format=...`).
+2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry).
+3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env).
+4. Default **`float`**.
+
+That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly.
+
+To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params).

litellm/llms/hosted_vllm/rerank/transformation.py

@@ -121,6 +121,7 @@ class HostedVLLMRerankConfig(BaseRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         if "query" not in optional_rerank_params:
             raise ValueError("query is required for Hosted VLLM rerank")

litellm/llms/huggingface/rerank/transformation.py

@@ -146,6 +146,7 @@ class HuggingFaceRerankConfig(BaseRerankConfig):
         model: str,
         optional_rerank_params: Union[OptionalRerankParams, dict],
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         if "query" not in optional_rerank_params:
             raise ValueError("query is required for HuggingFace rerank")

litellm/llms/jina_ai/rerank/transformation.py

@@ -74,7 +74,11 @@ class JinaAIRerankConfig(BaseRerankConfig):
         return cleaned_base
 
     def transform_rerank_request(
-        self, model: str, optional_rerank_params: Dict, headers: Dict
+        self,
+        model: str,
+        optional_rerank_params: Dict,
+        headers: Dict,
+        litellm_params: Optional[dict] = None,
     ) -> Dict:
         return {"model": model, **optional_rerank_params}
 

litellm/llms/manus/files/transformation.py

@@ -18,6 +18,7 @@ from openai.types.file_deleted import FileDeleted
 
 import litellm
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.files.transformation import (
@@ -306,7 +307,8 @@ class ManusFilesConfig(BaseFilesConfig):
             optional_params=optional_params,
             litellm_params=litellm_params,
         )
-        return f"{api_base}/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}", {}
 
     def transform_retrieve_file_response(
         self,
@@ -336,7 +338,8 @@ class ManusFilesConfig(BaseFilesConfig):
             optional_params=optional_params,
             litellm_params=litellm_params,
         )
-        return f"{api_base}/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}", {}
 
     def transform_delete_file_response(
         self,
@@ -422,7 +425,8 @@ class ManusFilesConfig(BaseFilesConfig):
             optional_params=optional_params,
             litellm_params=litellm_params,
         )
-        return f"{api_base}/{file_id}/content", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}/content", {}
 
     def transform_file_content_response(
         self,

litellm/llms/manus/responses/transformation.py

@@ -6,6 +6,7 @@ import httpx
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     _safe_convert_created_field,
 )
@@ -270,7 +271,10 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
 
         Reference: https://open.manus.im/docs/openai-compatibility
         """
-        url = f"{api_base}/{response_id}"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        url = f"{api_base}/{encoded_response_id}"
         data: Dict = {}
         return url, data
 

litellm/llms/milvus/vector_stores/transformation.py

@@ -25,7 +25,6 @@ else:
     LiteLLMLoggingObj = Any
 
 MILVUS_OPTIONAL_PARAMS = {
-    "dbName",
     "annsField",
     "limit",
     "filter",
@@ -33,7 +32,6 @@ MILVUS_OPTIONAL_PARAMS = {
     "groupingField",
     "outputFields",
     "searchParams",
-    "partitionNames",
     "consistencyLevel",
 }
 
@@ -173,13 +171,21 @@ class MilvusVectorStoreConfig(BaseVectorStoreConfig):
         url = f"{api_base}/v2/vectordb/entities/search"
 
         # Build the request body for Azure AI Search with vector search
-        request_body = {
+        request_body: Dict[str, Any] = {
             "collectionName": index_name,
             "data": [query_vector],
             "annsField": "book_intro_vector",
             **vector_store_search_optional_params,
         }
 
+        db_name = litellm_params.get("milvus_db_name")
+        if db_name:
+            request_body["dbName"] = db_name
+
+        partition_names = litellm_params.get("milvus_partition_names")
+        if partition_names:
+            request_body["partitionNames"] = partition_names
+
         #########################################################
         # Update logging object with details of the request
         #########################################################

litellm/llms/nvidia_nim/rerank/ranking_transformation.py

@@ -66,6 +66,7 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         """
         Transform request, using clean model name without 'ranking/' prefix.
@@ -75,4 +76,5 @@ class NvidiaNimRankingConfig(NvidiaNimRerankConfig):
             model=clean_model,
             optional_rerank_params=optional_rerank_params,
             headers=headers,
+            litellm_params=litellm_params,
         )

litellm/llms/nvidia_nim/rerank/transformation.py

@@ -177,6 +177,7 @@ class NvidiaNimRerankConfig(BaseRerankConfig):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         """
         Transform request to Nvidia NIM format.

litellm/llms/openai/containers/transformation.py

@@ -6,6 +6,7 @@ import litellm
 from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
     StandardBuiltInToolCostTracking,
 )
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.containers.main import (
     ContainerCreateOptionalRequestParams,
@@ -198,7 +199,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
     ) -> Tuple[str, Dict]:
         """Transform the OpenAI container retrieve request."""
         # For container retrieve, we just need to construct the URL
-        url = join_container_api_base_path(api_base, f"/{container_id}")
+        encoded_container_id = encode_url_path_segment(
+            container_id, field_name="container_id"
+        )
+        url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
 
         # No additional data needed for GET request
         data: Dict[str, Any] = {}
@@ -230,7 +234,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
         - DELETE /v1/containers/{container_id}
         """
         # Construct the URL for container delete
-        url = join_container_api_base_path(api_base, f"/{container_id}")
+        encoded_container_id = encode_url_path_segment(
+            container_id, field_name="container_id"
+        )
+        url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
 
         # No data needed for DELETE request
         data: Dict[str, Any] = {}
@@ -267,7 +274,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
         - GET /v1/containers/{container_id}/files
         """
         # Construct the URL for container files
-        url = join_container_api_base_path(api_base, f"/{container_id}/files")
+        encoded_container_id = encode_url_path_segment(
+            container_id, field_name="container_id"
+        )
+        url = join_container_api_base_path(api_base, f"/{encoded_container_id}/files")
 
         # Prepare query parameters
         params: Dict[str, Any] = {}
@@ -311,8 +321,12 @@ class OpenAIContainerConfig(BaseContainerConfig):
         - GET /v1/containers/{container_id}/files/{file_id}/content
         """
         # Construct the URL for container file content
+        encoded_container_id = encode_url_path_segment(
+            container_id, field_name="container_id"
+        )
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
         url = join_container_api_base_path(
-            api_base, f"/{container_id}/files/{file_id}/content"
+            api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content"
         )
 
         # No query parameters needed

litellm/llms/openai/evals/transformation.py

@@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
 import httpx
 
 from litellm._logging import verbose_logger
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.evals.transformation import (
     BaseEvalsAPIConfig,
     LiteLLMLoggingObj,
@@ -76,7 +77,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
             api_base = "https://api.openai.com"
 
         if eval_id:
-            return f"{api_base}/v1/evals/{eval_id}"
+            encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+            return f"{api_base}/v1/evals/{encoded_eval_id}"
         return f"{api_base}/v1/{endpoint}"
 
     def transform_create_eval_request(
@@ -276,7 +278,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         if litellm_params and litellm_params.api_base:
             api_base = litellm_params.api_base
 
-        url = f"{api_base}/v1/evals/{eval_id}/runs"
+        encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+        url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
 
         # Build request body
         request_body = {k: v for k, v in create_request.items() if v is not None}
@@ -310,7 +313,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         if litellm_params and litellm_params.api_base:
             api_base = litellm_params.api_base
 
-        url = f"{api_base}/v1/evals/{eval_id}/runs"
+        encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+        url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
 
         # Build query parameters
         query_params: Dict[str, Any] = {}
@@ -350,7 +354,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         headers: dict,
     ) -> Tuple[str, Dict]:
         """Transform get run request for OpenAI"""
-        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+        encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+        encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
+        url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
 
         verbose_logger.debug("Get run request - URL: %s", url)
 
@@ -376,7 +382,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         headers: dict,
     ) -> Tuple[str, Dict, Dict]:
         """Transform cancel run request for OpenAI"""
-        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
+        encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+        encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
+        url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel"
 
         # Empty body for cancel request
         request_body: Dict[str, Any] = {}
@@ -405,7 +413,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
         headers: dict,
     ) -> Tuple[str, Dict, Dict]:
         """Transform delete run request for OpenAI"""
-        url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
+        encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
+        encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
+        url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
 
         # Empty body for delete request
         request_body: Dict[str, Any] = {}

litellm/llms/openai/responses/transformation.py

@@ -7,6 +7,7 @@ from pydantic import BaseModel, ValidationError
 import litellm
 from litellm._logging import verbose_logger
 from litellm.litellm_core_utils.core_helpers import process_response_headers
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
     _safe_convert_created_field,
 )
@@ -421,7 +422,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         OpenAI API expects the following request
         - DELETE /v1/responses/{response_id}
         """
-        url = f"{api_base}/{response_id}"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        url = f"{api_base}/{encoded_response_id}"
         data: Dict = {}
         return url, data
 
@@ -457,7 +461,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         OpenAI API expects the following request
         - GET /v1/responses/{response_id}
         """
-        url = f"{api_base}/{response_id}"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        url = f"{api_base}/{encoded_response_id}"
         data: Dict = {}
         return url, data
 
@@ -498,7 +505,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         limit: int = 20,
         order: Literal["asc", "desc"] = "desc",
     ) -> Tuple[str, Dict]:
-        url = f"{api_base}/{response_id}/input_items"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        url = f"{api_base}/{encoded_response_id}/input_items"
         params: Dict[str, Any] = {}
         if after is not None:
             params["after"] = after
@@ -540,7 +550,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
         OpenAI API expects the following request
         - POST /v1/responses/{response_id}/cancel
         """
-        url = f"{api_base}/{response_id}/cancel"
+        encoded_response_id = encode_url_path_segment(
+            response_id, field_name="response_id"
+        )
+        url = f"{api_base}/{encoded_response_id}/cancel"
         data: Dict = {}
         return url, data
 

litellm/llms/openai/vector_store_files/transformation.py

@@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, Tuple, cast
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store_files.transformation import (
     BaseVectorStoreFilesConfig,
 )
@@ -98,7 +99,10 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
             or "https://api.openai.com/v1"
         )
         base_url = base_url.rstrip("/")
-        return f"{base_url}/vector_stores/{vector_store_id}/files"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        return f"{base_url}/vector_stores/{encoded_vector_store_id}/files"
 
     def transform_create_vector_store_file_request(
         self,
@@ -163,7 +167,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
         file_id: str,
         api_base: str,
     ) -> Tuple[str, Dict[str, Any]]:
-        return f"{api_base}/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}", {}
 
     def transform_retrieve_vector_store_file_response(
         self,
@@ -186,7 +191,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
         file_id: str,
         api_base: str,
     ) -> Tuple[str, Dict[str, Any]]:
-        return f"{api_base}/{file_id}/content", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}/content", {}
 
     def transform_retrieve_vector_store_file_content_response(
         self,
@@ -218,7 +224,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
                 payload["attributes"] = filtered_attributes
             else:
                 payload.pop("attributes", None)
-        return f"{api_base}/{file_id}", payload
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}", payload
 
     def transform_update_vector_store_file_response(
         self,
@@ -241,7 +248,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
         file_id: str,
         api_base: str,
     ) -> Tuple[str, Dict[str, Any]]:
-        return f"{api_base}/{file_id}", {}
+        encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
+        return f"{api_base}/{encoded_file_id}", {}
 
     def transform_delete_vector_store_file_response(
         self,

litellm/llms/openai/vector_stores/transformation.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
 import httpx
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
@@ -108,7 +109,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
         litellm_params: dict,
         extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
-        url = f"{api_base}/{vector_store_id}/search"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}/search"
         typed_request_body = VectorStoreSearchRequest(
             query=query,
             filters=vector_store_search_optional_params.get("filters", None),

litellm/llms/openai/videos/transformation.py

@@ -1,11 +1,13 @@
 import mimetypes
 from io import BufferedReader, BytesIO
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
+from urllib.parse import quote
 
 import httpx
 from httpx._types import RequestFiles
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
 from litellm.secret_managers.main import get_secret_str
@@ -220,11 +222,18 @@ class OpenAIVideoConfig(BaseVideoConfig):
         - GET /v1/videos/{video_id}/content?variant=thumbnail
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Construct the URL for video content download
-        url = f"{api_base.rstrip('/')}/{original_video_id}/content"
+        url = f"{api_base.rstrip('/')}/{encoded_video_id}/content"
         if variant is not None:
-            url = f"{url}?variant={variant}"
+            # Encode the user-controlled ``variant`` so a value like
+            # ``thumbnail&extra=1`` cannot inject additional query params
+            # into the upstream request — same hardening rationale as the
+            # path-segment encoding above.
+            url = f"{url}?variant={quote(variant, safe='')}"
 
         # No additional data needed for GET content request
         data: Dict[str, Any] = {}
@@ -247,9 +256,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
         - POST /v1/videos/{video_id}/remix
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Construct the URL for video remix
-        url = f"{api_base.rstrip('/')}/{original_video_id}/remix"
+        url = f"{api_base.rstrip('/')}/{encoded_video_id}/remix"
 
         # Prepare the request data
         data = {"prompt": prompt}
@@ -391,9 +403,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
         - DELETE /v1/videos/{video_id}
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Construct the URL for video delete
-        url = f"{api_base.rstrip('/')}/{original_video_id}"
+        url = f"{api_base.rstrip('/')}/{encoded_video_id}"
 
         # No data needed for DELETE request
         data: Dict[str, Any] = {}
@@ -427,9 +442,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
         """
         # Extract the original video_id (remove provider encoding if present)
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # For video retrieve, we just need to construct the URL
-        url = f"{api_base.rstrip('/')}/{original_video_id}"
+        url = f"{api_base.rstrip('/')}/{encoded_video_id}"
 
         # No additional data needed for GET request
         data: Dict[str, Any] = {}
@@ -494,7 +512,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
         litellm_params: GenericLiteLLMParams,
         headers: dict,
     ) -> Tuple[str, Dict]:
-        url = f"{api_base.rstrip('/')}/characters/{character_id}"
+        original_character_id = extract_original_character_id(character_id)
+        encoded_character_id = encode_url_path_segment(
+            original_character_id, field_name="character_id"
+        )
+        url = f"{api_base.rstrip('/')}/characters/{encoded_character_id}"
         return url, {}
 
     def transform_video_get_character_response(

litellm/llms/openai_like/providers.json

@@ -106,5 +106,13 @@
     "base_url": "https://aihubmix.com/v1",
     "api_key_env": "AIHUBMIX_API_KEY",
     "api_base_env": "AIHUBMIX_API_BASE"
+  },
+  "crusoe": {
+    "base_url": "https://managed-inference-api-proxy.crusoecloud.com/v1",
+    "api_key_env": "CRUSOE_API_KEY",
+    "api_base_env": "CRUSOE_API_BASE",
+    "param_mappings": {
+      "max_completion_tokens": "max_tokens"
+    }
   }
 }

litellm/llms/pg_vector/vector_stores/transformation.py

@@ -1,5 +1,6 @@
 from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.router import GenericLiteLLMParams
@@ -82,7 +83,10 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
         litellm_params: dict,
         extra_body: Optional[Dict[str, Any]] = None,
     ) -> Tuple[str, Dict]:
-        url = f"{api_base}/{vector_store_id}/search"
+        encoded_vector_store_id = encode_url_path_segment(
+            vector_store_id, field_name="vector_store_id"
+        )
+        url = f"{api_base}/{encoded_vector_store_id}/search"
         _, request_body = super().transform_search_vector_store_request(
             vector_store_id=vector_store_id,
             query=query,

litellm/llms/ragflow/chat/transformation.py

@@ -13,6 +13,7 @@ Model name format:
 from typing import List, Optional, Tuple
 
 import litellm
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.openai.openai import OpenAIConfig
 from litellm.secret_managers.main import get_secret, get_secret_str
 from litellm.types.llms.openai import AllMessageValues
@@ -126,10 +127,11 @@ class RAGFlowConfig(OpenAIConfig):
             api_base = api_base[:-3]  # Remove /v1
 
         # Construct the RAGFlow-specific path
+        encoded_entity_id = encode_url_path_segment(entity_id, field_name="entity_id")
         if endpoint_type == "chat":
-            path = f"/api/v1/chats_openai/{entity_id}/chat/completions"
+            path = f"/api/v1/chats_openai/{encoded_entity_id}/chat/completions"
         else:  # agent
-            path = f"/api/v1/agents_openai/{entity_id}/chat/completions"
+            path = f"/api/v1/agents_openai/{encoded_entity_id}/chat/completions"
 
         # Ensure path starts with /
         if not path.startswith("/"):

litellm/llms/runwayml/videos/transformation.py

@@ -6,6 +6,7 @@ from httpx._types import RequestFiles
 
 import litellm
 from litellm.constants import RUNWAYML_DEFAULT_API_VERSION
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.chat.transformation import BaseLLMException
 from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
 from litellm.llms.custom_httpx.http_handler import (
@@ -334,9 +335,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         We'll retrieve the task and extract the video URL.
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Get task status to retrieve video URL
-        url = f"{api_base}/tasks/{original_video_id}"
+        url = f"{api_base}/tasks/{encoded_video_id}"
 
         params: Dict[str, Any] = {}
 
@@ -495,9 +499,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         RunwayML uses task cancellation.
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Construct the URL for task cancellation
-        url = f"{api_base}/tasks/{original_video_id}/cancel"
+        url = f"{api_base}/tasks/{encoded_video_id}/cancel"
 
         data: Dict[str, Any] = {}
 
@@ -533,9 +540,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
         RunwayML uses GET /v1/tasks/{task_id} to retrieve task status.
         """
         original_video_id = extract_original_video_id(video_id)
+        encoded_video_id = encode_url_path_segment(
+            original_video_id, field_name="video_id"
+        )
 
         # Construct the full URL for task status retrieval
-        url = f"{api_base}/tasks/{original_video_id}"
+        url = f"{api_base}/tasks/{encoded_video_id}"
 
         # Empty dict for GET request (no body)
         data: Dict[str, Any] = {}

litellm/llms/vertex_ai/batches/handler.py

@@ -4,7 +4,11 @@ from typing import Any, Coroutine, Dict, Optional, Union
 import httpx
 
 import litellm
-from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
+from litellm.litellm_core_utils.url_utils import (
+    async_safe_get,
+    encode_url_path_segment,
+    safe_get,
+)
 from litellm.llms.custom_httpx.http_handler import (
     _get_httpx_client,
     get_async_httpx_client,
@@ -170,7 +174,8 @@ class VertexAIBatchPrediction(VertexLLM):
         )
 
         # Append batch_id to the URL
-        default_api_base = f"{default_api_base}/{batch_id}"
+        encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
+        default_api_base = f"{default_api_base}/{encoded_batch_id}"
 
         if len(default_api_base.split(":")) > 1:
             endpoint = default_api_base.split(":")[-1]
@@ -413,7 +418,8 @@ class VertexAIBatchPrediction(VertexLLM):
             vertex_project=vertex_project or project_id,
         )
 
-        retrieve_api_base_default = f"{default_api_base}/{batch_id}"
+        encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
+        retrieve_api_base_default = f"{default_api_base}/{encoded_batch_id}"
         cancel_api_base_default = f"{retrieve_api_base_default}:cancel"
 
         _, api_base = self._check_custom_proxy(

litellm/llms/vertex_ai/common_utils.py

@@ -27,6 +27,53 @@ class VertexAIError(BaseLLMException):
         super().__init__(message=message, status_code=status_code, headers=headers)
 
 
+def vertex_request_labels_from_litellm_params(
+    litellm_params: Optional[dict],
+) -> Optional[Dict[str, str]]:
+    """
+    Build Vertex/GCP billing labels from LiteLLM user metadata on ``litellm_params``:
+    ``metadata`` (``completion(..., metadata=...)``) or ``litellm_metadata``,
+    using ``requester_metadata`` string key-value pairs (same convention as Gemini).
+    ``metadata`` is tried first when both are present.
+    """
+    if not litellm_params:
+        return None
+    for key in ("metadata", "litellm_metadata"):
+        if key not in litellm_params:
+            continue
+        metadata = litellm_params[key]
+        if metadata is None or not isinstance(metadata, dict):
+            continue
+        if "requester_metadata" not in metadata:
+            continue
+        rm = metadata["requester_metadata"]
+        if not isinstance(rm, dict):
+            continue
+        labels = {k: v for k, v in rm.items() if isinstance(v, str)}
+        if labels:
+            return labels
+    return None
+
+
+def pop_vertex_request_labels(
+    optional_params: Optional[dict],
+    litellm_params: Optional[dict],
+) -> Optional[Dict[str, str]]:
+    """
+    Resolve labels from optional ``labels`` (Gemini-style) and/or
+    ``litellm_params["metadata"]`` / ``litellm_params["litellm_metadata"]``
+    (``requester_metadata``). Pops ``labels`` from optional_params when present.
+    """
+    labels: Optional[Dict[str, str]] = None
+    if optional_params is not None and "labels" in optional_params:
+        raw = optional_params.pop("labels")
+        if isinstance(raw, dict):
+            labels = {k: v for k, v in raw.items() if isinstance(v, str)}
+    if not labels:
+        labels = vertex_request_labels_from_litellm_params(litellm_params)
+    return labels if labels else None
+
+
 class VertexAIModelRoute(str, Enum):
     """Enum for Vertex AI model routing"""
 
@@ -50,7 +97,7 @@ def get_vertex_ai_model_route(
     Determine which handler to use for a Vertex AI model based on the model name.
 
     Args:
-        model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "openai/gpt-oss-120b")
+        model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "xai/grok-4.1-fast-non-reasoning")
         litellm_params: Optional litellm parameters dict that may contain base_model for routing
 
     Returns:
@@ -66,7 +113,7 @@ def get_vertex_ai_model_route(
         >>> get_vertex_ai_model_route("gemma/gemma-3-12b-it")
         VertexAIModelRoute.GEMMA
 
-        >>> get_vertex_ai_model_route("openai/gpt-oss-120b")
+        >>> get_vertex_ai_model_route("xai/grok-4.1-fast-non-reasoning")
         VertexAIModelRoute.MODEL_GARDEN
 
         >>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"})
@@ -102,8 +149,11 @@ def get_vertex_ai_model_route(
     if "gemma/" in model:
         return VertexAIModelRoute.GEMMA
 
-    # Check for model garden openai models
-    if "openai" in model:
+    # Check for model garden OpenAI-compatible publisher models.
+    # Examples:
+    # - openai/gpt-oss-120b-maas
+    # - xai/grok-4.1-fast-non-reasoning
+    if "openai" in model or model.startswith("xai/"):
         return VertexAIModelRoute.MODEL_GARDEN
 
     # Check for gemini models
@@ -209,8 +259,8 @@ def get_vertex_base_model_name(model: str) -> str:
         >>> get_vertex_base_model_name("gemma/gemma-3-12b-it")
         "gemma-3-12b-it"
 
-        >>> get_vertex_base_model_name("openai/gpt-oss-120b")
-        "gpt-oss-120b"
+        >>> get_vertex_base_model_name("xai/grok-4.1-fast-non-reasoning")
+        "grok-4.1-fast-non-reasoning"
 
         >>> get_vertex_base_model_name("1234567890")
         "1234567890"

litellm/llms/vertex_ai/gemini/transformation.py

@@ -24,6 +24,7 @@ from litellm.litellm_core_utils.prompt_templates.factory import (
     response_schema_prompt,
 )
 from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
+from litellm.llms.vertex_ai.common_utils import pop_vertex_request_labels
 from litellm.types.files import (
     get_file_mime_type_for_file_type,
     get_file_type_from_extension,
@@ -211,6 +212,22 @@ def _process_gemini_media(
             return _apply_gemini_metadata(
                 part, model, media_resolution_enum, video_metadata
             )
+        elif image_url.startswith(
+            "https://generativelanguage.googleapis.com/v1beta/files/"
+        ):
+            # Gemini Files API URIs — the file is already uploaded to Google's
+            # servers; pass the URI through as file_data without fetching it.
+            # These URLs return 403 when accessed directly, so we must not try
+            # to resolve their MIME type via HTTP.
+            if format:
+                file_data = FileDataType(mime_type=format, file_uri=image_url)
+            else:
+                # Gemini Files API references can be passed through as URI-only.
+                file_data = cast(FileDataType, {"file_uri": image_url})
+            part = {"file_data": file_data}
+            return _apply_gemini_metadata(
+                part, model, media_resolution_enum, video_metadata
+            )
         elif (
             "https://" in image_url
             and (image_type := format or _get_image_mime_type_from_url(image_url))
@@ -714,16 +731,8 @@ def _transform_request_body(  # noqa: PLR0915
         optional_params.pop("output_config", None)
         config_fields = GenerationConfig.__annotations__.keys()
 
-        # If the LiteLLM client sends Gemini-supported parameter "labels", add it
-        # as "labels" field to the request sent to the Gemini backend.
-        labels: Optional[dict[str, str]] = optional_params.pop("labels", None)
-        # If the LiteLLM client sends OpenAI-supported parameter "metadata", add it
-        # as "labels" field to the request sent to the Gemini backend.
-        if labels is None and "metadata" in litellm_params:
-            metadata = litellm_params["metadata"]
-            if metadata is not None and "requester_metadata" in metadata:
-                rm = metadata["requester_metadata"]
-                labels = {k: v for k, v in rm.items() if isinstance(v, str)}
+        # labels: optional explicit param and/or metadata.requester_metadata (OpenAI metadata)
+        labels = pop_vertex_request_labels(optional_params, litellm_params)
 
         filtered_params = {
             k: v
@@ -750,16 +759,22 @@ def _transform_request_body(  # noqa: PLR0915
                     ]
 
         data = RequestBody(contents=content)
-        if system_instructions is not None:
-            data["system_instruction"] = system_instructions
-        if tools is not None:
-            data["tools"] = tools
-        if tool_choice is not None:
-            data["toolConfig"] = tool_choice
-        if include_server_side_tool_invocations:
-            if "toolConfig" not in data:
-                data["toolConfig"] = {}
-            data["toolConfig"]["includeServerSideToolInvocations"] = True
+        # Vertex rejects system_instruction/tools/toolConfig alongside cachedContent.
+        # Treat dropping these fields as a request mutation guarded by modify_params.
+        can_send_cache_incompatible_fields = (
+            cached_content is None or litellm.modify_params is False
+        )
+        if can_send_cache_incompatible_fields:
+            if system_instructions is not None:
+                data["system_instruction"] = system_instructions
+            if tools is not None:
+                data["tools"] = tools
+            if tool_choice is not None:
+                data["toolConfig"] = tool_choice
+            if include_server_side_tool_invocations:
+                if "toolConfig" not in data:
+                    data["toolConfig"] = {}
+                data["toolConfig"]["includeServerSideToolInvocations"] = True
         if safety_settings is not None:
             data["safetySettings"] = safety_settings
         if generation_config is not None and len(generation_config) > 0:

litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py

@@ -979,15 +979,8 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig):
                     params["includeThoughts"] = False
                 else:
                     params["includeThoughts"] = True
-                    if thinking_budget >= 10000:
-                        is_gemini3flash = (
-                            "gemini-3-flash-preview" in model.lower()
-                            or "gemini-3-flash" in model.lower()
-                        )
-                        params["thinkingLevel"] = (
-                            "minimal" if is_gemini3flash else "low"
-                        )
-                    else:
+                    # Follow provider defaults unless explicitly opted into legacy behavior.
+                    if litellm.enable_gemini_default_thinking_level_low is True:
                         is_gemini3flash = (
                             "gemini-3-flash-preview" in model.lower()
                             or "gemini-3-flash" in model.lower()

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py

@@ -3,7 +3,7 @@ Google AI Studio /batchEmbedContents Embeddings Endpoint
 """
 
 import json
-from typing import Any, Dict, Literal, Optional, Union
+from typing import Any, Dict, List, Literal, Optional, Tuple, Union
 
 import httpx
 
@@ -13,8 +13,8 @@ from litellm.llms.custom_httpx.http_handler import (
     HTTPHandler,
     get_async_httpx_client,
 )
-from litellm.types.llms.openai import EmbeddingInput
 from litellm.types.llms.vertex_ai import (
+    GeminiEmbeddingInput,
     VertexAIBatchEmbeddingsRequestBody,
     VertexAIBatchEmbeddingsResponseObject,
 )
@@ -23,7 +23,6 @@ from litellm.types.utils import EmbeddingResponse
 from ..gemini.vertex_and_google_ai_studio_gemini import VertexLLM
 from .batch_embed_content_transformation import (
     _is_file_reference,
-    _is_multimodal_input,
     process_embed_content_response,
     process_response,
     transform_openai_input_gemini_content,
@@ -32,9 +31,24 @@ from .batch_embed_content_transformation import (
 
 
 class GoogleBatchEmbeddings(VertexLLM):
+    @staticmethod
+    def _flatten_and_detect_file_refs(
+        input: GeminiEmbeddingInput,
+    ) -> Tuple[List[str], bool]:
+        """Flatten nested input lists and detect file references."""
+        input_list = [input] if isinstance(input, str) else input
+        flat_elements = [
+            e
+            for item in input_list
+            for e in (item if isinstance(item, list) else [item])
+            if isinstance(e, str)
+        ]
+        has_file_refs = any(_is_file_reference(e) for e in flat_elements)
+        return flat_elements, has_file_refs
+
     def _resolve_file_references(
         self,
-        input: EmbeddingInput,
+        input: GeminiEmbeddingInput,
         api_key: str,
         sync_handler: HTTPHandler,
     ) -> Dict[str, Dict[str, str]]:
@@ -42,7 +56,7 @@ class GoogleBatchEmbeddings(VertexLLM):
         Resolve Gemini file references (files/...) to get mime_type and uri.
 
         Args:
-            input: EmbeddingInput that may contain file references
+            input: GeminiEmbeddingInput that may contain file references
             api_key: Gemini API key
             sync_handler: HTTP client
 
@@ -73,7 +87,7 @@ class GoogleBatchEmbeddings(VertexLLM):
 
     async def _async_resolve_file_references(
         self,
-        input: EmbeddingInput,
+        input: GeminiEmbeddingInput,
         api_key: str,
         async_handler: AsyncHTTPHandler,
     ) -> Dict[str, Dict[str, str]]:
@@ -81,7 +95,7 @@ class GoogleBatchEmbeddings(VertexLLM):
         Async version of _resolve_file_references.
 
         Args:
-            input: EmbeddingInput that may contain file references
+            input: GeminiEmbeddingInput that may contain file references
             api_key: Gemini API key
             async_handler: Async HTTP client
 
@@ -110,10 +124,10 @@ class GoogleBatchEmbeddings(VertexLLM):
 
         return resolved_files
 
-    def batch_embeddings(
+    def batch_embeddings(  # noqa: PLR0915
         self,
         model: str,
-        input: EmbeddingInput,
+        input: GeminiEmbeddingInput,
         print_verbose,
         model_response: EmbeddingResponse,
         custom_llm_provider: Literal["gemini", "vertex_ai"],
@@ -151,8 +165,7 @@ class GoogleBatchEmbeddings(VertexLLM):
 
         optional_params = optional_params or {}
 
-        is_multimodal = _is_multimodal_input(input)
-        use_embed_content = is_multimodal or (custom_llm_provider == "vertex_ai")
+        use_embed_content = custom_llm_provider == "vertex_ai"
         mode: Literal["embedding", "batch_embedding"]
         if use_embed_content:
             mode = "embedding"
@@ -215,8 +228,22 @@ class GoogleBatchEmbeddings(VertexLLM):
                 resolved_files=resolved_files,
             )
         else:
+            flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
+            if has_file_refs and not api_key:
+                raise ValueError(
+                    "An API key is required to resolve Gemini file references (files/...). "
+                    "Pass api_key= or set GEMINI_API_KEY."
+                )
+            resolved_files = {}
+            if api_key and has_file_refs:
+                resolved_files = self._resolve_file_references(
+                    input=flat_elements, api_key=api_key, sync_handler=sync_handler
+                )
             request_data = transform_openai_input_gemini_content(
-                input=input, model=model, optional_params=optional_params
+                input=input,
+                model=model,
+                optional_params=optional_params,
+                resolved_files=resolved_files,
             )
 
         ## LOGGING
@@ -264,7 +291,7 @@ class GoogleBatchEmbeddings(VertexLLM):
         url: str,
         data: Optional[Union[VertexAIBatchEmbeddingsRequestBody, dict]],
         model_response: EmbeddingResponse,
-        input: EmbeddingInput,
+        input: GeminiEmbeddingInput,
         timeout: Optional[Union[float, httpx.Timeout]],
         headers={},
         client: Optional[AsyncHTTPHandler] = None,
@@ -303,8 +330,22 @@ class GoogleBatchEmbeddings(VertexLLM):
                 resolved_files=resolved_files,
             )
         else:
+            flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
+            if has_file_refs and not api_key:
+                raise ValueError(
+                    "An API key is required to resolve Gemini file references (files/...). "
+                    "Pass api_key= or set GEMINI_API_KEY."
+                )
+            resolved_files = {}
+            if api_key and has_file_refs:
+                resolved_files = await self._async_resolve_file_references(
+                    input=flat_elements, api_key=api_key, async_handler=async_handler
+                )
             data = transform_openai_input_gemini_content(
-                input=input, model=model, optional_params=optional_params or {}
+                input=input,
+                model=model,
+                optional_params=optional_params or {},
+                resolved_files=resolved_files,
             )
 
         ## LOGGING

litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py

@@ -6,12 +6,12 @@ Why separate file? Make it easy to see how transformation works
 
 from typing import Dict, List, Optional, Tuple
 
-from litellm.types.llms.openai import EmbeddingInput
 from litellm.types.llms.vertex_ai import (
     BlobType,
     ContentType,
     EmbedContentRequest,
     FileDataType,
+    GeminiEmbeddingInput,
     PartType,
     VertexAIBatchEmbeddingsRequestBody,
     VertexAIBatchEmbeddingsResponseObject,
@@ -114,33 +114,77 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
     return media_type, base64_data
 
 
-def _is_multimodal_input(input: EmbeddingInput) -> bool:
+def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool:
     """
-    Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
+    Check if the input contains multimodal data (data URIs, file references,
+    GCS URLs, or nested lists for combined embeddings).
 
     Args:
-        input: EmbeddingInput (str or List[str])
+        input: GeminiEmbeddingInput — str, List[str], or List[List[str]] for combined embeddings
 
     Returns:
-        bool: True if any element is a data URI, file reference, or GCS URL
+        bool: True if any element is multimodal or a nested list
     """
     if isinstance(input, str):
-        input_list = [input]
-    else:
-        input_list = input
+        return _is_multimodal_element(input)
 
-    for element in input_list:
-        if isinstance(element, str):
-            if element.startswith("data:") and ";base64," in element:
-                return True
-            if _is_file_reference(element):
-                return True
-            if _is_gcs_url(element):
+    for element in input:
+        if isinstance(element, list):
+            if any(
+                _is_multimodal_element(sub) for sub in element if isinstance(sub, str)
+            ):
                 return True
+        elif isinstance(element, str) and _is_multimodal_element(element):
+            return True
 
     return False
 
 
+def _is_multimodal_element(element: str) -> bool:
+    """Check if a single string element is multimodal."""
+    if element.startswith("data:") and ";base64," in element:
+        return True
+    if _is_file_reference(element):
+        return True
+    if _is_gcs_url(element):
+        return True
+    return False
+
+
+def _build_part_for_input(
+    element: str,
+    resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
+) -> PartType:
+    """
+    Build a single PartType for an input element, handling text, data URIs,
+    file references, and GCS URLs.
+    """
+    resolved_files = resolved_files or {}
+
+    if element.startswith("data:") and ";base64," in element:
+        mime_type, base64_data = _parse_data_url(element)
+        blob: BlobType = {"mime_type": mime_type, "data": base64_data}
+        return PartType(inline_data=blob)
+    elif _is_gcs_url(element):
+        mime_type = _infer_mime_type_from_gcs_url(element)
+        file_data: FileDataType = {
+            "mime_type": mime_type,
+            "file_uri": element,
+        }
+        return PartType(file_data=file_data)
+    elif _is_file_reference(element):
+        if element not in resolved_files:
+            raise ValueError(f"File reference {element} not resolved")
+        file_info = resolved_files[element]
+        file_data_ref: FileDataType = {
+            "mime_type": file_info["mime_type"],
+            "file_uri": file_info["uri"],
+        }
+        return PartType(file_data=file_data_ref)
+    else:
+        return PartType(text=element)
+
+
 _SUPPORTED_EMBED_PARAMS = {"outputDimensionality", "taskType", "title"}
 
 
@@ -155,37 +199,60 @@ def _filter_embed_params(optional_params: dict) -> dict:
 
 
 def transform_openai_input_gemini_content(
-    input: EmbeddingInput, model: str, optional_params: dict
+    input: GeminiEmbeddingInput,
+    model: str,
+    optional_params: dict,
+    resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
 ) -> VertexAIBatchEmbeddingsRequestBody:
     """
-    The content to embed. Only the parts.text fields will be counted.
+    Transform OpenAI embedding input to Gemini batchEmbedContents format.
+
+    Each input element becomes a separate EmbedContentRequest, supporting
+    text, data URIs, file references, and GCS URLs.
+
+    If an element is a list (nested input), all sub-elements are combined
+    into a single content with multiple parts, producing one combined
+    embedding for the group.
+
+    Examples:
+        input=["text", "image"]         → 2 separate embeddings
+        input=[["text", "image"]]       → 1 combined embedding
+        input=[["text", "image"], "x"]  → 2 embeddings (1 combined + 1 separate)
     """
     gemini_model_name = "models/{}".format(model)
 
     gemini_params = _filter_embed_params(optional_params)
 
+    input_list = [input] if isinstance(input, str) else input
     requests: List[EmbedContentRequest] = []
-    if isinstance(input, str):
+
+    for element in input_list:
+        if isinstance(element, list):
+            if not element:
+                raise ValueError("Nested input list must not be empty")
+            for sub in element:
+                if not isinstance(sub, str):
+                    raise ValueError(
+                        f"Elements inside a nested input list must be strings, got {type(sub)}"
+                    )
+            parts = [
+                _build_part_for_input(sub, resolved_files=resolved_files)
+                for sub in element
+            ]
+        else:
+            parts = [_build_part_for_input(element, resolved_files=resolved_files)]
         request = EmbedContentRequest(
             model=gemini_model_name,
-            content=ContentType(parts=[PartType(text=input)]),
+            content=ContentType(parts=parts),
             **gemini_params,
         )
         requests.append(request)
-    else:
-        for i in input:
-            request = EmbedContentRequest(
-                model=gemini_model_name,
-                content=ContentType(parts=[PartType(text=i)]),
-                **gemini_params,
-            )
-            requests.append(request)
 
     return VertexAIBatchEmbeddingsRequestBody(requests=requests)
 
 
 def transform_openai_input_gemini_embed_content(
-    input: EmbeddingInput,
+    input: GeminiEmbeddingInput,
     model: str,
     optional_params: dict,
     resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
@@ -194,7 +261,7 @@ def transform_openai_input_gemini_embed_content(
     Transform OpenAI embedding input to Gemini embedContent format (multimodal).
 
     Args:
-        input: EmbeddingInput (str or List[str]) with text, data URIs, or file references
+        input: GeminiEmbeddingInput with text, data URIs, or file references
         model: Model name
         optional_params: Additional parameters (taskType, outputDimensionality, etc.)
         resolved_files: Dict mapping file names (files/abc) to {mime_type, uri}
@@ -210,31 +277,14 @@ def transform_openai_input_gemini_embed_content(
     parts: List[PartType] = []
 
     for element in input_list:
+        if isinstance(element, list):
+            raise ValueError(
+                "Nested (combined) embeddings are not supported on the embedContent path. "
+                "Use the batchEmbedContents path or pass a flat list instead."
+            )
         if not isinstance(element, str):
             raise ValueError(f"Unsupported input type: {type(element)}")
-
-        if element.startswith("data:") and ";base64," in element:
-            mime_type, base64_data = _parse_data_url(element)
-            blob: BlobType = {"mime_type": mime_type, "data": base64_data}
-            parts.append(PartType(inline_data=blob))
-        elif _is_gcs_url(element):
-            mime_type = _infer_mime_type_from_gcs_url(element)
-            file_data: FileDataType = {
-                "mime_type": mime_type,
-                "file_uri": element,
-            }
-            parts.append(PartType(file_data=file_data))
-        elif _is_file_reference(element):
-            if element not in resolved_files:
-                raise ValueError(f"File reference {element} not resolved")
-            file_info = resolved_files[element]
-            file_data_ref: FileDataType = {
-                "mime_type": file_info["mime_type"],
-                "file_uri": file_info["uri"],
-            }
-            parts.append(PartType(file_data=file_data_ref))
-        else:
-            parts.append(PartType(text=element))
+        parts.append(_build_part_for_input(element, resolved_files=resolved_files))
 
     request_body: dict = {
         "content": ContentType(parts=parts),
@@ -245,7 +295,7 @@ def transform_openai_input_gemini_embed_content(
 
 
 def process_embed_content_response(
-    input: EmbeddingInput,
+    input: GeminiEmbeddingInput,
     model_response: EmbeddingResponse,
     model: str,
     response_json: dict,
@@ -291,7 +341,7 @@ def process_embed_content_response(
 
 
 def process_response(
-    input: EmbeddingInput,
+    input: GeminiEmbeddingInput,
     model_response: EmbeddingResponse,
     model: str,
     _predictions: VertexAIBatchEmbeddingsResponseObject,
@@ -308,8 +358,29 @@ def process_response(
     model_response.data = openai_embeddings
     model_response.model = model
 
-    input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
-    prompt_tokens = token_counter(model=model, text=input_text)
+    has_nested = isinstance(input, list) and any(isinstance(e, list) for e in input)
+    if _is_multimodal_input(input) or has_nested:
+        input_list = input if isinstance(input, list) else [input]
+        text_elements: List[str] = []
+        for e in input_list:
+            if isinstance(e, list):
+                text_elements.extend(
+                    sub
+                    for sub in e
+                    if isinstance(sub, str) and not _is_multimodal_element(sub)
+                )
+            elif isinstance(e, str) and not _is_multimodal_element(e):
+                text_elements.append(e)
+        if text_elements:
+            input_text = get_formatted_prompt(
+                data={"input": text_elements}, call_type="embedding"
+            )
+            prompt_tokens = token_counter(model=model, text=input_text)
+        else:
+            prompt_tokens = 0
+    else:
+        input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
+        prompt_tokens = token_counter(model=model, text=input_text)
     model_response.usage = Usage(
         prompt_tokens=prompt_tokens, total_tokens=prompt_tokens
     )

litellm/llms/vertex_ai/image_generation/vertex_imagen_transformation.py

@@ -7,7 +7,10 @@ import litellm
 from litellm.llms.base_llm.image_generation.transformation import (
     BaseImageGenerationConfig,
 )
-from litellm.llms.vertex_ai.common_utils import get_vertex_base_url
+from litellm.llms.vertex_ai.common_utils import (
+    get_vertex_base_url,
+    pop_vertex_request_labels,
+)
 from litellm.llms.vertex_ai.gemini.vertex_and_google_ai_studio_gemini import VertexLLM
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import (
@@ -203,13 +206,16 @@ class VertexAIImagenImageGenerationConfig(BaseImageGenerationConfig, VertexLLM):
             "sampleCount": 1,
         }
 
-        # Merge with optional params
+        labels = pop_vertex_request_labels(optional_params, litellm_params)
+        # Merge with optional params (after popping labels so they are not sent as Imagen parameters)
         parameters = {**default_params, **optional_params}
 
-        request_body = {
+        request_body: dict = {
             "instances": [{"prompt": prompt}],
             "parameters": parameters,
         }
+        if labels:
+            request_body["labels"] = labels
 
         return request_body
 

litellm/llms/vertex_ai/rerank/transformation.py

@@ -11,12 +11,15 @@ import httpx
 import litellm
 from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
 from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig
+from litellm.llms.vertex_ai.common_utils import (
+    vertex_request_labels_from_litellm_params,
+)
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.rerank import (
+    RerankBilledUnits,
     RerankResponse,
     RerankResponseMeta,
-    RerankBilledUnits,
     RerankResponseResult,
 )
 
@@ -109,6 +112,7 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
         model: str,
         optional_rerank_params: Dict,
         headers: dict,
+        litellm_params: Optional[dict] = None,
     ) -> dict:
         """
         Transform the request from Cohere format to Vertex AI Discovery Engine format
@@ -145,6 +149,10 @@ class VertexAIRerankConfig(BaseRerankConfig, VertexBase):
         # When return_documents is False, we want to ignore record details (return only IDs)
         request_data["ignoreRecordDetailsInResponse"] = not return_documents
 
+        user_labels = vertex_request_labels_from_litellm_params(litellm_params)
+        if user_labels:
+            request_data["userLabels"] = user_labels
+
         return request_data
 
     def transform_rerank_response(

litellm/llms/vertex_ai/vector_stores/search_api/transformation.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
 import httpx
 
 from litellm import get_model_info
+from litellm.litellm_core_utils.url_utils import encode_url_path_segment
 from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
 from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
 from litellm.types.router import GenericLiteLLMParams
@@ -91,12 +92,18 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
             raise ValueError("vector_store_id is required")
         if api_base:
             return api_base.rstrip("/")
+        encoded_collection_id = encode_url_path_segment(
+            collection_id, field_name="vertex_collection_id"
+        )
+        encoded_datastore_id = encode_url_path_segment(
+            datastore_id, field_name="vector_store_id"
+        )
 
         # Vertex AI Search API endpoint for search
         return (
             f"https://discoveryengine.googleapis.com/v1/"
             f"projects/{vertex_project}/locations/{vertex_location}/"
-            f"collections/{collection_id}/dataStores/{datastore_id}/servingConfigs/default_config"
+            f"collections/{encoded_collection_id}/dataStores/{encoded_datastore_id}/servingConfigs/default_config"
         )
 
     def transform_search_vector_store_request(

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py

@@ -13,6 +13,7 @@ from litellm.types.llms.vertex_ai import VertexPartnerProvider
 from litellm.types.router import GenericLiteLLMParams
 
 from ....vertex_llm_base import VertexBase
+from ..output_params_utils import sanitize_vertex_anthropic_output_params
 
 
 class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, VertexBase):
@@ -158,12 +159,10 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert
             "model", None
         )  # do not pass model in request body to vertex ai
 
-        anthropic_messages_request.pop(
-            "output_format", None
-        )  # do not pass output_format in request body to vertex ai - vertex ai does not support output_format as yet
-
-        anthropic_messages_request.pop(
-            "output_config", None
-        )  # do not pass output_config in request body to vertex ai - vertex ai does not support output_config
+        # Vertex AI Claude accepts ``output_config.format`` (structured outputs)
+        # and ``output_format``, but rejects ``output_config.effort`` with 400
+        # "Extra inputs are not permitted". Sanitize in place so the supported
+        # bits flow through.
+        sanitize_vertex_anthropic_output_params(anthropic_messages_request)
 
         return anthropic_messages_request

litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/output_params_utils.py

@@ -0,0 +1,50 @@
+"""
+Shared sanitization for ``output_config`` / ``output_format`` on Vertex AI
+Claude. Lives in its own module so both the chat-completion transformation
+(``transformation.py``) and the Messages pass-through transformation
+(``experimental_pass_through/transformation.py``) can import it without
+forming a cycle through the parent module's heavier imports.
+
+CodeQL flagged the ``..transformation`` import path as a potential cyclic
+import; extracting the helper into a leaf module resolves the warning and
+keeps the parent module's import surface narrow.
+"""
+
+# Keys inside ``output_config`` that Vertex AI Claude does not accept.
+# Today only ``effort`` triggers "Extra inputs are not permitted"; add new
+# entries here as Vertex parity drifts. Keep this list narrow — anything
+# Vertex DOES accept (e.g. ``format`` for structured outputs) must be
+# preserved so callers can rely on Anthropic-native features.
+VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS: frozenset = frozenset({"effort"})
+
+
+def sanitize_vertex_anthropic_output_params(data: dict) -> None:
+    """
+    Strip Vertex-unsupported keys from ``output_config`` /
+    ``output_format`` in-place; forward whatever remains.
+
+    Behavior:
+      * ``output_config`` containing only unsupported keys (e.g. ``effort``
+        alone) is removed entirely so the request body has no empty dict.
+      * ``output_config`` containing a mix of supported + unsupported keys
+        has the unsupported subset filtered out and the rest forwarded.
+      * ``output_config`` that is supported in full passes through unchanged.
+      * ``output_format`` is forwarded as-is (Vertex AI Claude accepts it).
+      * Non-dict values for ``output_config`` are dropped to avoid sending
+        malformed payloads downstream.
+    """
+    output_config = data.get("output_config")
+    if output_config is None:
+        return
+    if not isinstance(output_config, dict):
+        data.pop("output_config", None)
+        return
+    sanitized = {
+        k: v
+        for k, v in output_config.items()
+        if k not in VERTEX_UNSUPPORTED_OUTPUT_CONFIG_KEYS
+    }
+    if sanitized:
+        data["output_config"] = sanitized
+    else:
+        data.pop("output_config", None)