scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-09 03:18:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_clean_litellm_oss_staging_04_01_2026

Browse source
This commit is contained in:
mateo-berri 2026-05-01 15:54:10 -07:00
commit 04e96a9bdc
221 changed files with 12184 additions and 1583 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

75
.github/workflows/check-lazy-openapi-snapshot.yml vendored
View file

@ -1,75 +0,0 @@
name: Check Lazy OpenAPI Snapshot
on:
pull_request:
branches:
- main
- litellm_internal_staging
- "litellm_**"
permissions:
contents: read
checks: write
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
verify:
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7
with:
version: "0.10.9"
- name: Cache uv dependencies
uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0
with:
path: |
~/.cache/uv
.venv
key: ${{ runner.os }}-uv-${{ hashFiles('uv.lock') }}
restore-keys: |
${{ runner.os }}-uv-
- name: Install dependencies
run: uv sync --frozen --all-groups --all-extras
- name: Regenerate snapshot to /tmp
id: regen
run: |
cp litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.committed.json
uv run --no-sync python -m litellm.proxy._lazy_openapi_snapshot
mv litellm/proxy/_lazy_openapi_snapshot.json /tmp/snapshot.fresh.json
mv /tmp/snapshot.committed.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Compare
id: diff
continue-on-error: true
run: |
diff -q /tmp/snapshot.fresh.json litellm/proxy/_lazy_openapi_snapshot.json
- name: Mark neutral if drift
if: steps.diff.outcome == 'failure'
uses: LouisBrunner/checks-action@6b626ffbad7cc56fd58627f774b9067e6118af23 # v2.0.0
with:
token: ${{ secrets.GITHUB_TOKEN }}
name: lazy-openapi-snapshot
conclusion: neutral
output: |
{
"title": "Lazy openapi snapshot is stale",
"summary": "Run `python -m litellm.proxy._lazy_openapi_snapshot` and commit the regenerated `litellm/proxy/_lazy_openapi_snapshot.json`. Not blocking — the snapshot will regenerate at release if not committed."
}

4
.gitignore vendored
View file

@ -90,7 +90,6 @@ test.py
litellm_config.yaml
!.github/observatory/litellm_config.yaml
.cursor
.vscode/launch.json
litellm/proxy/to_delete_loadtest_work/*
update_model_cost_map.py
tests/test_litellm/proxy/_experimental/mcp_server/test_mcp_server_manager.py
@ -100,4 +99,5 @@ STABILIZATION_TODO.md
**/test-results
**/playwright-report
**/*.storageState.json
**/coverage
**/coverage
test-config

3
Makefile
View file

@ -185,3 +185,6 @@ test-llm-translation-single: install-test-deps
$(UV_RUN) pytest tests/llm_translation/$(FILE) \
--junitxml=test-results/junit.xml \
-v --tb=short --maxfail=100 --timeout=300
test-llm-translation-flush-vcr-cache:
$(UV_RUN) python tests/_flush_vcr_cache.py

2
README.md
View file

@ -68,7 +68,7 @@ Managing LLM calls across providers gets complicated fast — different SDKs, au
<td><img height="60" alt="Stripe" src="https://github.com/user-attachments/assets/f7296d4f-9fbd-460d-9d05-e4df31697c4b" /></td>
<td><img height="60" alt="image" src="https://github.com/user-attachments/assets/436fca71-988b-40bb-b5fe-8450c80fdbd0" /></td>
<td><img height="60" alt="Google ADK" src="https://github.com/user-attachments/assets/caf270a2-5aee-45c4-8222-41a2070c4f19" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/0be4bd8a-7cfa-48d3-9090-f415fe948280" /></td>
<td><img height="60" alt="Greptile" src="https://github.com/user-attachments/assets/3db0ae72-0843-4005-a56d-bba1dde2193d" /></td>
<td><img height="60" alt="OpenHands" src="https://github.com/user-attachments/assets/a6150c4c-149e-4cae-888b-8b92be6e003f" /></td>
<td><h2>Netflix</h2></td>
<td><img height="60" alt="OpenAI Agents SDK" src="https://github.com/user-attachments/assets/c02f7be0-8c2e-4d27-aea7-7c024bfaebc0" /></td>

36
enterprise/litellm_enterprise/proxy/management_endpoints/project_endpoints.py
View file

@ -857,10 +857,16 @@ async def project_info(
where={"team_id": project.team_id}
)
if team:
is_team_member = (
user_api_key_dict.user_id in team.admins
or user_api_key_dict.user_id in team.members
)
caller_user_id = user_api_key_dict.user_id
for m in team.members_with_roles or []:
m_user_id = (
m.get("user_id")
if isinstance(m, dict)
else getattr(m, "user_id", None)
)
if m_user_id == caller_user_id:
is_team_member = True
break
if not (is_admin or is_team_member):
raise HTTPException(
@ -911,20 +917,20 @@ async def list_projects(
include={"litellm_budget_table": True, "object_permission": True}
)
else:
# Get projects for teams the user belongs to
user_teams = await prisma_client.db.litellm_teamtable.find_many(
where={
"OR": [
{"members": {"has": user_api_key_dict.user_id}},
{"admins": {"has": user_api_key_dict.user_id}},
]
}
# Look up the user's team memberships via the reverse-index on
# LiteLLM_UserTable.teams (maintained by team_member_add alongside
# members_with_roles). This avoids a full scan of all team rows.
user_record = await prisma_client.db.litellm_usertable.find_unique(
where={"user_id": user_api_key_dict.user_id},
)
user_team_ids = (
user_record.teams
if user_record is not None and user_record.teams
else []
)
team_ids = [team.team_id for team in user_teams]
projects = await prisma_client.db.litellm_projecttable.find_many(
where={"team_id": {"in": team_ids}},
where={"team_id": {"in": user_team_ids}},
include={"litellm_budget_table": True, "object_permission": True},
)

3
litellm/caching/caching.py
View file

@ -432,9 +432,10 @@ class Cache:
str: The final hashed cache key with the redis namespace.
"""
dynamic_cache_control: DynamicCacheControl = kwargs.get("cache", {})
metadata = kwargs.get("metadata") or {}
namespace = (
dynamic_cache_control.get("namespace")
or kwargs.get("metadata", {}).get("redis_namespace")
or metadata.get("redis_namespace")
or self.namespace
)
if namespace:

99
litellm/caching/caching_handler.py
View file

@ -87,6 +87,18 @@ class CachingHandlerResponse(BaseModel):
in_memory_cache_obj = InMemoryCache()
def _should_defer_streaming_cache_hit_callbacks(*, kwargs: Dict[str, Any]) -> bool:
"""
When stream=True, do not run success callbacks at cache-hit time.
Cached chat/text completion replay uses CustomStreamWrapper; cached Responses
replay uses CachedResponsesAPIStreamingIterator. Both invoke logging success
handlers when the stream finishes; firing them here too would double-count
spend and callback records.
"""
return kwargs.get("stream", False) is True
class LLMCachingHandler:
def __init__(
self,
@ -99,6 +111,7 @@ class LLMCachingHandler:
self.async_streaming_chunks: List[ModelResponse] = []
self.sync_streaming_chunks: List[ModelResponse] = []
self.request_kwargs = request_kwargs
self.preset_cache_key: Optional[str] = None
self.original_function = original_function
self.start_time = start_time
if litellm.cache is not None and isinstance(litellm.cache.cache, RedisCache):
@ -206,7 +219,7 @@ class LLMCachingHandler:
custom_llm_provider=kwargs.get("custom_llm_provider", None),
args=args,
)
if kwargs.get("stream", False) is False:
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
# LOG SUCCESS
self._async_log_cache_hit_on_callbacks(
logging_obj=logging_obj,
@ -215,11 +228,12 @@ class LLMCachingHandler:
end_time=end_time,
cache_hit=cache_hit,
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)
) and hasattr(cached_result, "_hidden_params"):
cache_key = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
return CachingHandlerResponse(cached_result=cached_result)
elif (
@ -265,8 +279,6 @@ class LLMCachingHandler:
kwargs: Dict[str, Any],
args: Optional[Tuple[Any, ...]] = None,
) -> CachingHandlerResponse:
from litellm.utils import CustomStreamWrapper
cached_result: Optional[Any] = None
# Check if caching should be performed BEFORE doing expensive kwargs copy
@ -282,6 +294,11 @@ class LLMCachingHandler:
args,
)
)
if new_kwargs.get("metadata") is None:
new_kwargs.pop("metadata", None)
if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = new_kwargs
print_verbose("Checking Sync Cache")
cached_result = litellm.cache.get_cache(**new_kwargs)
if cached_result is not None:
@ -322,17 +339,19 @@ class LLMCachingHandler:
is_async=False,
)
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
if not _should_defer_streaming_cache_hit_callbacks(kwargs=kwargs):
logging_obj.handle_sync_success_callbacks_for_async_calls(
result=cached_result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
)
cache_key = (
self.preset_cache_key
or self.request_kwargs.get("cache_key")
or litellm.cache.get_cache_key(**self.request_kwargs)
)
cache_key = litellm.cache.get_cache_key(**kwargs)
if (
isinstance(cached_result, BaseModel)
or isinstance(cached_result, CustomStreamWrapper)
) and hasattr(cached_result, "_hidden_params"):
if hasattr(cached_result, "_hidden_params"):
cached_result._hidden_params["cache_key"] = cache_key # type: ignore
return CachingHandlerResponse(cached_result=cached_result)
return CachingHandlerResponse(cached_result=cached_result)
@ -686,6 +705,11 @@ class LLMCachingHandler:
args,
)
)
if new_kwargs.get("metadata") is None:
new_kwargs.pop("metadata", None)
if new_kwargs.get("stream") is True and "cache_key" not in new_kwargs:
new_kwargs["cache_key"] = litellm.cache.get_cache_key(**new_kwargs)
self.request_kwargs = new_kwargs
cached_result: Optional[Any] = None
if call_type == CallTypes.aembedding.value:
if isinstance(new_kwargs["input"], str):
@ -710,14 +734,26 @@ class LLMCachingHandler:
if all(result is None for result in cached_result):
cached_result = None
else:
request_kwargs = new_kwargs.copy()
request_cache_key = request_kwargs.pop("cache_key", None)
if litellm.cache._supports_async() is True:
## check if dual cache is supported ##
self.preset_cache_key = (
request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
)
cached_result = await litellm.cache.async_get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
dynamic_cache_object=self.dual_cache,
cache_key=self.preset_cache_key,
**request_kwargs,
)
else: # fallback for caches that don't support async
self.preset_cache_key = (
request_cache_key or litellm.cache.get_cache_key(**request_kwargs)
)
cached_result = litellm.cache.get_cache(
dynamic_cache_object=self.dual_cache, **new_kwargs
dynamic_cache_object=self.dual_cache,
cache_key=self.preset_cache_key,
**request_kwargs,
)
return cached_result
@ -825,8 +861,27 @@ class LLMCachingHandler:
elif (call_type == "aresponses" or call_type == "responses") and isinstance(
cached_result, dict
):
# Convert cached dict back to ResponsesAPIResponse object
cached_result = ResponsesAPIResponse(**cached_result)
from litellm.responses.streaming_iterator import (
CachedResponsesAPIStreamingIterator,
)
response_obj = ResponsesAPIResponse(**cached_result)
if (
hasattr(response_obj, "_hidden_params")
and response_obj._hidden_params is not None
and isinstance(response_obj._hidden_params, dict)
):
response_obj._hidden_params["cache_hit"] = True
if kwargs.get("stream", False) is True:
cached_result = CachedResponsesAPIStreamingIterator(
response=response_obj,
logging_obj=logging_obj,
request_data=kwargs,
call_type=call_type,
)
else:
cached_result = response_obj
if (
hasattr(cached_result, "_hidden_params")

19
litellm/caching/dual_cache.py
View file

@ -92,6 +92,25 @@ class DualCache(BaseCache):
if default_redis_ttl is not None:
self.default_redis_ttl = default_redis_ttl
def attach_redis_cache(
self,
redis_cache: Optional[RedisCache] = None,
*,
default_redis_ttl: Optional[float] = None,
) -> None:
"""
Attach a Redis backend if this DualCache does not already have one.
No-op when ``redis_cache`` is None or when Redis was already set (constructor
or a prior attach). Use this for lazy wiring after a shared Redis client exists.
Does not backfill in-memory-only keys to Redis.
"""
if redis_cache is None or self.redis_cache is not None:
return
self.redis_cache = redis_cache
if default_redis_ttl is not None:
self.default_redis_ttl = default_redis_ttl
def set_cache(self, key, value, local_only: bool = False, **kwargs):
# Update both Redis and in-memory cache
try:

10
litellm/caching/redis_cache.py
View file

@ -551,6 +551,13 @@ class RedisCache(BaseCache):
async def async_set_cache(self, key, value, **kwargs):
from redis.asyncio import Redis
if key is None:
verbose_logger.debug(
"LiteLLM Redis Caching: async set() skipped — key is None, value=%r",
value,
)
return None
start_time = time.time()
try:
_redis_client: Redis = self.init_async_client() # type: ignore
@ -569,8 +576,9 @@ class RedisCache(BaseCache):
)
)
verbose_logger.error(
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, Writing value=%s",
"LiteLLM Redis Caching: async set() - Got exception from REDIS %s, key=%r, value=%r",
str(e),
key,
value,
)
raise e

15
litellm/integrations/arize/arize_phoenix_client.py
View file

@ -2,11 +2,23 @@
Arize Phoenix API client for fetching prompt versions from Arize Phoenix.
"""
import urllib.parse
from typing import Any, Dict, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_id(identifier: str) -> str:
"""Reject path traversal characters and URL-encode the identifier."""
if any(c in identifier for c in ("/", "\\", "#", "?")):
raise ValueError(
f"Invalid identifier {identifier!r}: contains disallowed characters"
)
if ".." in identifier:
raise ValueError(f"Invalid identifier {identifier!r}: path traversal detected")
return urllib.parse.quote(identifier, safe="")
class ArizePhoenixClient:
"""
Client for interacting with Arize Phoenix API to fetch prompt versions.
@ -53,7 +65,8 @@ class ArizePhoenixClient:
Returns:
Dictionary containing prompt version data, or None if not found
"""
url = f"{self.api_base}/v1/prompt_versions/{prompt_version_id}"
safe_id = _sanitize_id(prompt_version_id)
url = f"{self.api_base}/v1/prompt_versions/{safe_id}"
try:
# Use the underlying httpx client directly to avoid query param extraction

25
litellm/integrations/bitbucket/bitbucket_client.py
View file

@ -3,11 +3,27 @@ BitBucket API client for fetching .prompt files from BitBucket repositories.
"""
import base64
import urllib.parse
from typing import Any, Dict, List, Optional
from litellm.llms.custom_httpx.http_handler import HTTPHandler
def _sanitize_file_path(file_path: str) -> str:
"""Reject path traversal and URL-encode each path segment."""
if "#" in file_path or "?" in file_path:
raise ValueError(
f"Invalid file path {file_path!r}: contains URL special characters"
)
parts = file_path.split("/")
for part in parts:
if part == "..":
raise ValueError(
f"Invalid file path {file_path!r}: path traversal detected"
)
return "/".join(urllib.parse.quote(part, safe="") for part in parts)
class BitBucketClient:
"""
Client for interacting with BitBucket API to fetch .prompt files.
@ -72,7 +88,8 @@ class BitBucketClient:
Returns:
File content as string, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
response = self.http_handler.get(url, headers=self.headers)
@ -119,7 +136,8 @@ class BitBucketClient:
Returns:
List of file paths
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{directory_path}"
safe_dir = _sanitize_file_path(directory_path) if directory_path else ""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_dir}"
try:
response = self.http_handler.get(url, headers=self.headers)
@ -211,7 +229,8 @@ class BitBucketClient:
Returns:
Dictionary containing file metadata, or None if file not found
"""
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{file_path}"
safe_path = _sanitize_file_path(file_path)
url = f"{self.base_url}/repositories/{self.workspace}/{self.repository}/src/{self.branch}/{safe_path}"
try:
# Use GET with Range header to get just the headers (HEAD equivalent)

1
litellm/integrations/prometheus.py
View file

@ -265,6 +265,7 @@ class PrometheusLogger(CustomLogger):
########################################
# LiteLLM Virtual API KEY metrics
########################################
# Remaining MODEL RPM limit for API Key
self.litellm_remaining_api_key_requests_for_model = self._gauge_factory(
"litellm_remaining_api_key_requests_for_model",

22
litellm/litellm_core_utils/cli_token_utils.py
View file

@ -31,15 +31,23 @@ def load_cli_token() -> Optional[dict]:
return None
def get_litellm_gateway_api_key() -> Optional[str]:
def get_litellm_gateway_api_key(
expected_base_url: Optional[str] = None,
) -> Optional[str]:
"""
Get the stored CLI API key for use with LiteLLM SDK.
This function reads the token file created by `litellm-proxy login`
and returns the API key for use in Python scripts.
Args:
expected_base_url: When provided, the key is only returned if it was
originally issued for this URL. Pass the target server URL to
prevent credential leakage when the client is pointed at a
different (possibly malicious) server.
Returns:
str: The API key if found, None otherwise
str: The API key if found (and origin matches), None otherwise
Example:
>>> import litellm
@ -53,6 +61,10 @@ def get_litellm_gateway_api_key() -> Optional[str]:
>>> )
"""
token_data = load_cli_token()
if token_data and "key" in token_data:
return token_data["key"]
return None
if not token_data or "key" not in token_data:
return None
if expected_base_url is not None:
stored_url = token_data.get("base_url")
if stored_url != expected_base_url.rstrip("/"):
return None
return token_data["key"]

6
litellm/litellm_core_utils/llm_request_utils.py
View file

@ -77,8 +77,8 @@ def get_proxy_server_request_headers(litellm_params: Optional[dict]) -> dict:
if litellm_params is None:
return {}
proxy_request_headers = (
litellm_params.get("proxy_server_request", {}).get("headers", {}) or {}
)
proxy_request_headers = (litellm_params.get("proxy_server_request") or {}).get(
"headers"
) or {}
return proxy_request_headers

48
litellm/litellm_core_utils/prompt_templates/factory.py
View file

@ -4582,6 +4582,11 @@ class BedrockConverseMessagesProcessor:
message=cast(ChatCompletionFileObject, element)
)
_parts.append(_part)
elif element["type"] == "document":
_part = BedrockConverseMessagesProcessor._process_document_message(
element
)
_parts.append(_part)
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(
@ -4864,6 +4869,44 @@ class BedrockConverseMessagesProcessor:
image_url=cast(str, file_id or file_data), format=format
)
@staticmethod
def _process_document_message(element: dict) -> BedrockContentBlock:
"""Convert a document content block to a Bedrock DocumentBlock.
Handles the Anthropic-style document format:
{"type": "document", "source": {"type": "base64", "media_type": "application/pdf", "data": "..."}}
"""
source = element["source"]
source_type = source.get("type")
if source_type != "base64":
raise ValueError(
f"Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. "
"Please convert the document to base64 before sending to Bedrock."
)
media_type: str = source["media_type"]
data: str = source["data"]
doc_format = BedrockImageProcessor._validate_format(
mime_type=media_type, image_format=media_type.split("/")[1]
)
# Deterministic name using the same hashing pattern as _create_bedrock_block
HASH_SAMPLE_BYTES = 64 * 1024
normalized = "".join(data.split()).encode("utf-8")
sample = normalized[:HASH_SAMPLE_BYTES]
hasher = hashlib.sha256()
hasher.update(sample)
hasher.update(str(len(normalized)).encode("utf-8"))
content_hash = hasher.hexdigest()[:16]
document_name = f"Document_{content_hash}_{doc_format}"
return BedrockContentBlock(
document=BedrockDocumentBlock(
source=BedrockSourceBlock(bytes=data),
format=doc_format,
name=document_name,
)
)
@staticmethod
def add_thinking_blocks_to_assistant_content(
thinking_blocks: List[BedrockContentBlock],
@ -4961,6 +5004,11 @@ def _bedrock_converse_messages_pt( # noqa: PLR0915
)
)
_parts.append(_part)
elif element["type"] == "document":
_part = BedrockConverseMessagesProcessor._process_document_message(
element
)
_parts.append(_part)
_cache_point_block = (
litellm.AmazonConverseConfig()._get_cache_point_block(
message_block=cast(

2
litellm/litellm_core_utils/streaming_handler.py
View file

@ -2244,7 +2244,7 @@ class CustomStreamWrapper:
asyncio.create_task(
self.logging_obj.async_failure_handler(e, traceback_exception)
)
raise e
self._handle_stream_fallback_error(e)
except Exception as e:
traceback_exception = traceback.format_exc()
if self.logging_obj is not None:

83
litellm/litellm_core_utils/url_utils.py
View file

@ -22,7 +22,7 @@ Admins can opt out via two ``litellm`` globals (wired from proxy config):
import socket
from ipaddress import ip_address, ip_network
from typing import Any, List, Set, Tuple
from urllib.parse import urlparse, urlunparse
from urllib.parse import quote, urlparse, urlunparse
import httpx
@ -46,6 +46,46 @@ class SSRFError(ValueError):
pass
def encode_url_path_segment(value: Any, *, field_name: str = "path parameter") -> str:
"""Percent-encode one user-controlled URL path segment.
``urllib.parse.quote(..., safe="")`` intentionally leaves RFC 3986
unreserved characters such as ``.`` unescaped, so reject standalone dot
segments before they can be appended to an upstream URL and normalized by
the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
if value_str in {".", ".."}:
raise ValueError(f"{field_name} cannot be a dot path segment")
return quote(value_str, safe="")
def encode_url_path_segments(value: Any, *, field_name: str = "path") -> str:
"""Percent-encode a user-controlled URL path made of multiple segments.
Empty segments are rejected, so leading, trailing, or consecutive slashes
fail closed instead of being normalized by the HTTP client.
"""
if value is None:
raise ValueError(f"{field_name} is required")
value_str = str(value)
if value_str == "":
raise ValueError(f"{field_name} is required")
encoded_segments = []
for segment in value_str.split("/"):
encoded_segments.append(encode_url_path_segment(segment, field_name=field_name))
return "/".join(encoded_segments)
def _is_blocked_ip(addr: str) -> bool:
"""Return True for any IP not safe to reach from a user-supplied URL.
@ -199,6 +239,47 @@ def validate_url(url: str) -> Tuple[str, str]:
return rewritten, host_header
def assert_same_origin(candidate_url: str, expected_url: str) -> None:
"""Verify ``candidate_url`` shares scheme, host, and port with ``expected_url``.
Use when an upstream API returns a URL meant for follow-up requests
(e.g. an async-job polling URL that will be hit with the operator's
API key in the headers). The upstream is trusted because the operator
configured ``api_base``, but the URL it hands back must actually point
back at the same origin or we'd be blindly forwarding credentials
wherever the upstream told us to.
Hostnames are compared case-insensitively. Default ports are made
explicit (HTTP→80, HTTPS→443) so ``https://api.example.com:443/...``
and ``https://api.example.com/...`` are treated as the same origin.
Error messages identify *which* component mismatched but never echo
the operator's ``expected`` host or the candidate's hostname back to
the caller — in the SSRF threat model the caller is the attacker,
and reflecting host info would be a secondary leak of operator
infrastructure details.
"""
candidate = urlparse(candidate_url)
expected = urlparse(expected_url)
if candidate.scheme not in _ALLOWED_SCHEMES:
raise SSRFError("URL scheme is not allowed")
if candidate.scheme != expected.scheme:
raise SSRFError("Origin mismatch on scheme")
candidate_host = _normalize_host(candidate.hostname or "")
expected_host = _normalize_host(expected.hostname or "")
if not candidate_host or candidate_host != expected_host:
raise SSRFError("Origin mismatch on host")
default_port = 443 if candidate.scheme == "https" else 80
candidate_port = candidate.port if candidate.port is not None else default_port
expected_port = expected.port if expected.port is not None else default_port
if candidate_port != expected_port:
raise SSRFError("Origin mismatch on port")
_MAX_REDIRECTS = 10

4
litellm/llms/anthropic/batches/transformation.py
View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cas
import httpx
from httpx import Headers, Response
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.types.llms.openai import AllMessageValues, CreateBatchRequest
@ -122,7 +123,8 @@ class AnthropicBatchesConfig(BaseBatchesConfig):
Complete URL for Anthropic batch retrieval: {api_base}/v1/messages/batches/{batch_id}
"""
api_base = api_base or self.anthropic_model_info.get_api_base(api_base)
return f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
return f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}"
def transform_retrieve_batch_request(
self,

6
litellm/llms/anthropic/files/handler.py
View file

@ -9,6 +9,7 @@ import litellm
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import (
FileContentRequest,
@ -89,7 +90,10 @@ class AnthropicFilesHandler:
raise ValueError("Missing Anthropic API Key")
# Construct the Anthropic batch results URL
results_url = f"{api_base.rstrip('/')}/v1/messages/batches/{batch_id}/results"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
results_url = (
f"{api_base.rstrip('/')}/v1/messages/batches/{encoded_batch_id}/results"
)
# Prepare headers
headers = {

10
litellm/llms/anthropic/files/transformation.py
View file

@ -19,6 +19,7 @@ from typing import Any, Dict, List, Optional, Union, cast
import httpx
from openai.types.file_deleted import FileDeleted
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
@ -185,7 +186,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_retrieve_file_response(
self,
@ -206,7 +208,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}", {}
def transform_delete_file_response(
self,
@ -268,7 +271,8 @@ class AnthropicFilesConfig(BaseFilesConfig):
AnthropicModelInfo.get_api_base(litellm_params.get("api_base"))
or ANTHROPIC_FILES_API_BASE
)
return f"{api_base.rstrip('/')}/v1/files/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base.rstrip('/')}/v1/files/{encoded_file_id}/content", {}
def transform_file_content_response(
self,

4
litellm/llms/anthropic/skills/transformation.py
View file

@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.skills.transformation import (
BaseSkillsAPIConfig,
LiteLLMLoggingObj,
@ -81,7 +82,8 @@ class AnthropicSkillsConfig(BaseSkillsAPIConfig):
api_base = AnthropicModelInfo.get_api_base()
if skill_id:
return f"{api_base}/v1/skills/{skill_id}"
encoded_skill_id = encode_url_path_segment(skill_id, field_name="skill_id")
return f"{api_base}/v1/skills/{encoded_skill_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_skill_request(

33
litellm/llms/azure/azure.py
View file

@ -16,6 +16,7 @@ import litellm
from litellm.constants import AZURE_OPERATION_POLLING_TIMEOUT, DEFAULT_MAX_RETRIES
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -899,6 +900,17 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
# Reject polling URLs that don't share an origin with ``api_base``.
# Without this an upstream-controlled or attacker-controlled
# value would receive the operator's Azure API key in the
# request headers below. VERIA-51.
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = await async_handler.get(
url=operation_location_url,
headers=headers,
@ -909,8 +921,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
# Don't reflect the raw response body — when the polling
# URL points at an internal JSON API (cloud metadata
# service etc.) reflecting it here turns Blind SSRF into
# Full-Read SSRF. VERIA-51.
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:
@ -1010,6 +1027,13 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
operation_location_url = response.headers["operation-location"]
else:
raise AzureOpenAIError(status_code=500, message=response.text)
try:
assert_same_origin(operation_location_url, api_base)
except SSRFError as ssrf_err:
raise AzureOpenAIError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
response = sync_handler.get(
url=operation_location_url,
headers=headers,
@ -1020,8 +1044,9 @@ class AzureChatCompletion(BaseAzureLLM, BaseLLM):
timeout_secs: int = AZURE_OPERATION_POLLING_TIMEOUT
start_time = time.time()
if "status" not in response.json():
raise Exception(
"Expected 'status' in response. Got={}".format(response.json())
raise AzureOpenAIError(
status_code=502,
message="Polling response missing 'status' field",
)
while response.json()["status"] not in ["succeeded", "failed"]:
if time.time() - start_time > timeout_secs:

11
litellm/llms/azure/responses/transformation.py
View file

@ -5,6 +5,7 @@ import httpx
from openai.types.responses import ResponseReasoningItem
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
from litellm.types.llms.openai import *
@ -201,7 +202,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}"
# Reconstruct the URL with all original components but with the modified path
constructed_url = urlunparse(
@ -322,7 +326,10 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
# Insert the response_id and /cancel at the end of the path component
# Remove trailing slash if present to avoid double slashes
path = parsed_url.path.rstrip("/")
new_path = f"{path}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
new_path = f"{path}/{encoded_response_id}/cancel"
# Reconstruct the URL with all original components but with the modified path
cancel_url = urlunparse(

18
litellm/llms/azure_ai/agents/handler.py
View file

@ -36,6 +36,7 @@ from typing import (
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.azure_ai.agents.transformation import (
AzureAIAgentsConfig,
AzureAIAgentsError,
@ -75,20 +76,29 @@ class AzureAIAgentsHandler:
def _build_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_runs_url(self, api_base: str, thread_id: str, api_version: str) -> str:
return f"{api_base}/threads/{thread_id}/runs?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return f"{api_base}/threads/{encoded_thread_id}/runs?api-version={api_version}"
def _build_run_status_url(
self, api_base: str, thread_id: str, run_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/runs/{run_id}?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
return f"{api_base}/threads/{encoded_thread_id}/runs/{encoded_run_id}?api-version={api_version}"
def _build_list_messages_url(
self, api_base: str, thread_id: str, api_version: str
) -> str:
return f"{api_base}/threads/{thread_id}/messages?api-version={api_version}"
encoded_thread_id = encode_url_path_segment(thread_id, field_name="thread_id")
return (
f"{api_base}/threads/{encoded_thread_id}/messages?api-version={api_version}"
)
def _build_create_thread_and_run_url(self, api_base: str, api_version: str) -> str:
"""URL for the create-thread-and-run endpoint (supports streaming)."""

23
litellm/llms/azure_ai/ocr/document_intelligence/transformation.py
View file

@ -17,11 +17,13 @@ from urllib.parse import quote
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.constants import (
AZURE_DOCUMENT_INTELLIGENCE_API_VERSION,
AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI,
AZURE_OPERATION_POLLING_TIMEOUT,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.ocr.transformation import (
BaseOCRConfig,
DocumentType,
@ -217,11 +219,12 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
if "/" in model:
# Extract the last part after the last slash
model_id = model.split("/")[-1]
encoded_model_id = encode_url_path_segment(model_id, field_name="model_id")
# Azure Document Intelligence analyze endpoint
# Note: API version 2024-11-30+ uses /documentintelligence/ (not /formrecognizer/)
url = (
f"{api_base}/documentintelligence/documentModels/{model_id}:analyze"
f"{api_base}/documentintelligence/documentModels/{encoded_model_id}:analyze"
f"?api-version={AZURE_DOCUMENT_INTELLIGENCE_API_VERSION}"
)
@ -599,6 +602,16 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"Azure Document Intelligence returned 202 but no Operation-Location header found"
)
# Reject cross-origin polling URLs — the auth headers
# below would otherwise leak to whatever URL the upstream
# (or an attacker-controlled upstream) returns. VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(
f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
)
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get(
@ -711,6 +724,14 @@ class AzureDocumentIntelligenceOCRConfig(BaseOCRConfig):
"Azure Document Intelligence returned 202 but no Operation-Location header found"
)
# Reject cross-origin polling URLs (see sync path). VERIA-51.
try:
assert_same_origin(operation_url, str(raw_response.request.url))
except SSRFError as ssrf_err:
raise ValueError(
f"Azure Document Intelligence: rejected polling URL ({ssrf_err})"
)
# Get headers for polling (need auth)
poll_headers = {
"Ocp-Apim-Subscription-Key": raw_response.request.headers.get(

10
litellm/llms/bedrock/chat/invoke_agent/transformation.py
View file

@ -12,6 +12,7 @@ import httpx
from litellm._logging import verbose_logger
from litellm._uuid import uuid
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import (
convert_content_list_to_str,
)
@ -97,8 +98,15 @@ class AmazonInvokeAgentConfig(BaseConfig, BaseAWSLLM):
agent_id, agent_alias_id = self._get_agent_id_and_alias_id(model)
session_id = self._get_session_id(optional_params)
encoded_agent_id = encode_url_path_segment(agent_id, field_name="agent_id")
encoded_agent_alias_id = encode_url_path_segment(
agent_alias_id, field_name="agent_alias_id"
)
encoded_session_id = encode_url_path_segment(
session_id, field_name="session_id"
)
endpoint_url = f"{endpoint_url}/agents/{agent_id}/agentAliases/{agent_alias_id}/sessions/{session_id}/text"
endpoint_url = f"{endpoint_url}/agents/{encoded_agent_id}/agentAliases/{encoded_agent_alias_id}/sessions/{encoded_session_id}/text"
return endpoint_url

3
litellm/llms/bedrock/count_tokens/transformation.py
View file

@ -201,13 +201,14 @@ class BedrockCountTokensConfig(BaseAWSLLM):
# Remove bedrock/ prefix if present
if model_id.startswith("bedrock/"):
model_id = model_id[8:] # Remove "bedrock/" prefix
encoded_model_id = self.encode_model_id(model_id=model_id)
base_url, _ = self.get_runtime_endpoint(
api_base=api_base,
aws_bedrock_runtime_endpoint=aws_bedrock_runtime_endpoint,
aws_region_name=aws_region_name,
)
endpoint = f"{base_url}/model/{model_id}/count-tokens"
endpoint = f"{base_url}/model/{encoded_model_id}/count-tokens"
return endpoint

6
litellm/llms/bedrock/vector_stores/transformation.py
View file

@ -5,6 +5,7 @@ from urllib.parse import urlparse
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.bedrock.base_aws_llm import BaseAWSLLM
from litellm.types.integrations.rag.bedrock_knowledgebase import (
@ -209,7 +210,10 @@ class BedrockVectorStoreConfig(BaseVectorStoreConfig, BaseAWSLLM):
if isinstance(query, list):
query = " ".join(query)
url = f"{api_base}/{vector_store_id}/retrieve"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/retrieve"
request_body: Dict[str, Any] = {
"retrievalQuery": BedrockKBRetrievalQuery(text=query),

23
litellm/llms/black_forest_labs/image_edit/handler.py
View file

@ -15,6 +15,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -331,6 +332,17 @@ class BlackForestLabsImageEdit:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
@ -416,6 +428,17 @@ class BlackForestLabsImageEdit:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}

23
litellm/llms/black_forest_labs/image_generation/handler.py
View file

@ -15,6 +15,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.url_utils import SSRFError, assert_same_origin
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -317,6 +318,17 @@ class BlackForestLabsImageGeneration:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}
@ -402,6 +414,17 @@ class BlackForestLabsImageGeneration:
message="No polling_url in BFL response",
)
# Reject cross-origin polling URLs — the ``x-key`` auth header
# would otherwise leak to whatever URL the upstream returns.
# VERIA-51.
try:
assert_same_origin(polling_url, str(initial_response.request.url))
except SSRFError as ssrf_err:
raise BlackForestLabsError(
status_code=502,
message=f"Rejected polling URL: {ssrf_err}",
)
# Get just the auth header for polling
polling_headers = {"x-key": headers.get("x-key", "")}

4
litellm/llms/bytez/chat/transformation.py
View file

@ -5,6 +5,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union
import httpx
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
@ -149,7 +150,8 @@ class BytezChatConfig(BaseConfig):
litellm_params: dict,
stream: Optional[bool] = None,
) -> str:
return f"{API_BASE}/{model}"
encoded_model = encode_url_path_segments(model, field_name="model")
return f"{API_BASE}/{encoded_model}"
def transform_request(
self,

4
litellm/llms/cloudflare/chat/transformation.py
View file

@ -5,6 +5,7 @@ from typing import AsyncIterator, Iterator, List, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segments
from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
from litellm.llms.base_llm.chat.transformation import (
BaseConfig,
@ -89,7 +90,8 @@ class CloudflareChatConfig(BaseConfig):
api_base = (
f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run/"
)
return api_base + model
encoded_model = encode_url_path_segments(model, field_name="model")
return api_base + encoded_model
def get_supported_openai_params(self, model: str) -> List[str]:
return [

4
litellm/llms/custom_httpx/container_handler.py
View file

@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Coroutine, Dict, Optional, Type, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.custom_httpx.http_handler import (
AsyncHTTPHandler,
HTTPHandler,
@ -72,7 +73,8 @@ def _build_url(
# Substitute path parameters
for param, value in path_params.items():
path_template = path_template.replace(f"{{{param}}}", value)
encoded_value = encode_url_path_segment(value, field_name=param)
path_template = path_template.replace(f"{{{param}}}", encoded_value)
# Parse the api_base to extract existing query params
parsed_base = httpx.URL(api_base)

31
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -26,6 +26,7 @@ from litellm._logging import _redact_string, verbose_logger
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
from litellm.constants import REALTIME_WEBSOCKET_MAX_MESSAGE_SIZE_BYTES
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -8948,7 +8949,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@ -9015,7 +9019,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@ -9214,7 +9221,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
@ -9297,7 +9307,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
request_body: Dict[str, Any] = dict(vector_store_update_optional_params)
@ -9363,7 +9376,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",
@ -9428,7 +9444,10 @@ class BaseLLMHTTPHandler:
litellm_params=dict(litellm_params),
)
url = f"{api_base}/{vector_store_id}"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}"
logging_obj.pre_call(
input="",

6
litellm/llms/elevenlabs/text_to_speech/transformation.py
View file

@ -11,13 +11,14 @@ import httpx
from httpx import Headers
import litellm
from litellm.types.utils import all_litellm_params
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.text_to_speech.transformation import (
BaseTextToSpeechConfig,
TextToSpeechRequestData,
)
from litellm.secret_managers.main import get_secret_str
from litellm.types.utils import all_litellm_params
from ..common_utils import ElevenLabsException
@ -321,7 +322,8 @@ class ElevenLabsTextToSpeechConfig(BaseTextToSpeechConfig):
"ElevenLabs voice_id is required. Pass `voice` when calling `litellm.speech()`."
)
url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{voice_id}"
encoded_voice_id = encode_url_path_segment(voice_id, field_name="voice_id")
url = f"{base_url}{self.TTS_ENDPOINT_PATH}/{encoded_voice_id}"
query_params = litellm_params.get(self.ELEVENLABS_QUERY_PARAMS_KEY, {})
if query_params:

20
litellm/llms/gemini/files/transformation.py
View file

@ -12,6 +12,7 @@ import httpx
from openai.types.file_deleted import FileDeleted
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.files.transformation import (
BaseFilesConfig,
@ -258,10 +259,14 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
normalized_file_id = file_id
normalized_file_id = normalized_file_id.strip("/")
if not normalized_file_id.startswith("files/"):
normalized_file_id = f"files/{normalized_file_id}"
if normalized_file_id.startswith("files/"):
normalized_file_id = normalized_file_id.removeprefix("files/")
return normalized_file_id
encoded_file_id = encode_url_path_segment(
normalized_file_id, field_name="file_id"
)
return f"files/{encoded_file_id}"
def transform_retrieve_file_response(
self,
@ -337,13 +342,8 @@ class GoogleAIStudioFilesHandler(GeminiModelInfo, BaseFilesConfig):
if not api_key:
raise ValueError("api_key is required")
# Extract file name from URI if full URI is provided
# file_id could be "files/abc123" or "https://generativelanguage.googleapis.com/v1beta/files/abc123"
if file_id.startswith("http"):
# Extract the file path from full URI
file_name = file_id.split("/v1beta/")[-1]
else:
file_name = file_id if file_id.startswith("files/") else f"files/{file_id}"
# Normalize and encode the file name before interpolating it into the URL.
file_name = self._normalize_gemini_file_id(file_id)
# Construct the delete URL
url = f"{api_base}/v1beta/{file_name}"

16
litellm/llms/gemini/interactions/transformation.py
View file

@ -15,6 +15,7 @@ import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.interactions.transformation import BaseInteractionsAPIConfig
from litellm.llms.gemini.common_utils import GeminiError, GeminiModelInfo
from litellm.types.interactions import (
@ -205,8 +206,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
{},
)
@ -238,8 +242,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}",
{},
)
@ -268,8 +275,11 @@ class GoogleAIStudioInteractionsConfig(BaseInteractionsAPIConfig):
resolved_api_base = GeminiModelInfo.get_api_base(api_base)
if not GeminiModelInfo.get_api_key(litellm_params.api_key):
raise ValueError("Google API key is required")
encoded_interaction_id = encode_url_path_segment(
interaction_id, field_name="interaction_id"
)
return (
f"{resolved_api_base}/{self.api_version}/interactions/{interaction_id}:cancel",
f"{resolved_api_base}/{self.api_version}/interactions/{encoded_interaction_id}:cancel",
{},
)

13
litellm/llms/hosted_vllm/embedding/README.md
View file

@ -2,4 +2,15 @@ No transformation is required for hosted_vllm embedding.
VLLM is a superset of OpenAI's `embedding` endpoint.
To pass provider-specific parameters, see [this](https://docs.litellm.ai/docs/completion/provider_specific_params)
## `encoding_format`
For OpenAI-compatible embedding calls (including `openai/...` with a custom `api_base` pointing at vLLM), LiteLLM resolves `encoding_format` when it is not set on the request:
1. Explicit value on the embedding call (`encoding_format=...`).
2. Model config (`litellm_params.encoding_format` on the proxy `model_list` entry).
3. Environment variable `LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT` (e.g. in `.env` or container env).
4. Default **`float`**.
That avoids forwarding `encoding_format=None` to the provider/SDK where some servers behave poorly.
To pass provider-specific parameters, see [provider-specific params](https://docs.litellm.ai/docs/completion/provider_specific_params).

10
litellm/llms/manus/files/transformation.py
View file

@ -18,6 +18,7 @@ from openai.types.file_deleted import FileDeleted
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.files.transformation import (
@ -306,7 +307,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_retrieve_file_response(
self,
@ -336,7 +338,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_delete_file_response(
self,
@ -422,7 +425,8 @@ class ManusFilesConfig(BaseFilesConfig):
optional_params=optional_params,
litellm_params=litellm_params,
)
return f"{api_base}/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}/content", {}
def transform_file_content_response(
self,

6
litellm/llms/manus/responses/transformation.py
View file

@ -6,6 +6,7 @@ import httpx
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@ -270,7 +271,10 @@ class ManusResponsesAPIConfig(OpenAIResponsesAPIConfig):
Reference: https://open.manus.im/docs/openai-compatibility
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data

22
litellm/llms/openai/containers/transformation.py
View file

@ -6,6 +6,7 @@ import litellm
from litellm.litellm_core_utils.llm_cost_calc.tool_call_cost_tracking import (
StandardBuiltInToolCostTracking,
)
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.secret_managers.main import get_secret_str
from litellm.types.containers.main import (
ContainerCreateOptionalRequestParams,
@ -198,7 +199,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
) -> Tuple[str, Dict]:
"""Transform the OpenAI container retrieve request."""
# For container retrieve, we just need to construct the URL
url = join_container_api_base_path(api_base, f"/{container_id}")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No additional data needed for GET request
data: Dict[str, Any] = {}
@ -230,7 +234,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
- DELETE /v1/containers/{container_id}
"""
# Construct the URL for container delete
url = join_container_api_base_path(api_base, f"/{container_id}")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}")
# No data needed for DELETE request
data: Dict[str, Any] = {}
@ -267,7 +274,10 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files
"""
# Construct the URL for container files
url = join_container_api_base_path(api_base, f"/{container_id}/files")
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
url = join_container_api_base_path(api_base, f"/{encoded_container_id}/files")
# Prepare query parameters
params: Dict[str, Any] = {}
@ -311,8 +321,12 @@ class OpenAIContainerConfig(BaseContainerConfig):
- GET /v1/containers/{container_id}/files/{file_id}/content
"""
# Construct the URL for container file content
encoded_container_id = encode_url_path_segment(
container_id, field_name="container_id"
)
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
url = join_container_api_base_path(
api_base, f"/{container_id}/files/{file_id}/content"
api_base, f"/{encoded_container_id}/files/{encoded_file_id}/content"
)
# No query parameters needed

22
litellm/llms/openai/evals/transformation.py
View file

@ -7,6 +7,7 @@ from typing import Any, Dict, Optional, Tuple
import httpx
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.evals.transformation import (
BaseEvalsAPIConfig,
LiteLLMLoggingObj,
@ -76,7 +77,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
api_base = "https://api.openai.com"
if eval_id:
return f"{api_base}/v1/evals/{eval_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
return f"{api_base}/v1/evals/{encoded_eval_id}"
return f"{api_base}/v1/{endpoint}"
def transform_create_eval_request(
@ -276,7 +278,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
if litellm_params and litellm_params.api_base:
api_base = litellm_params.api_base
url = f"{api_base}/v1/evals/{eval_id}/runs"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
# Build request body
request_body = {k: v for k, v in create_request.items() if v is not None}
@ -310,7 +313,8 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
if litellm_params and litellm_params.api_base:
api_base = litellm_params.api_base
url = f"{api_base}/v1/evals/{eval_id}/runs"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs"
# Build query parameters
query_params: Dict[str, Any] = {}
@ -350,7 +354,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict]:
"""Transform get run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
verbose_logger.debug("Get run request - URL: %s", url)
@ -376,7 +382,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict, Dict]:
"""Transform cancel run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}/cancel"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}/cancel"
# Empty body for cancel request
request_body: Dict[str, Any] = {}
@ -405,7 +413,9 @@ class OpenAIEvalsConfig(BaseEvalsAPIConfig):
headers: dict,
) -> Tuple[str, Dict, Dict]:
"""Transform delete run request for OpenAI"""
url = f"{api_base}/v1/evals/{eval_id}/runs/{run_id}"
encoded_eval_id = encode_url_path_segment(eval_id, field_name="eval_id")
encoded_run_id = encode_url_path_segment(run_id, field_name="run_id")
url = f"{api_base}/v1/evals/{encoded_eval_id}/runs/{encoded_run_id}"
# Empty body for delete request
request_body: Dict[str, Any] = {}

21
litellm/llms/openai/responses/transformation.py
View file

@ -7,6 +7,7 @@ from pydantic import BaseModel, ValidationError
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@ -421,7 +422,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- DELETE /v1/responses/{response_id}
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@ -457,7 +461,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- GET /v1/responses/{response_id}
"""
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@ -498,7 +505,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/input_items"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/input_items"
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
@ -540,7 +550,10 @@ class OpenAIResponsesAPIConfig(BaseResponsesAPIConfig):
OpenAI API expects the following request
- POST /v1/responses/{response_id}/cancel
"""
url = f"{api_base}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/cancel"
data: Dict = {}
return url, data

18
litellm/llms/openai/vector_store_files/transformation.py
View file

@ -3,6 +3,7 @@ from typing import Any, Dict, Optional, Tuple, cast
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store_files.transformation import (
BaseVectorStoreFilesConfig,
)
@ -98,7 +99,10 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
or "https://api.openai.com/v1"
)
base_url = base_url.rstrip("/")
return f"{base_url}/vector_stores/{vector_store_id}/files"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
return f"{base_url}/vector_stores/{encoded_vector_store_id}/files"
def transform_create_vector_store_file_request(
self,
@ -163,7 +167,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_retrieve_vector_store_file_response(
self,
@ -186,7 +191,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}/content", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}/content", {}
def transform_retrieve_vector_store_file_content_response(
self,
@ -218,7 +224,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
payload["attributes"] = filtered_attributes
else:
payload.pop("attributes", None)
return f"{api_base}/{file_id}", payload
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", payload
def transform_update_vector_store_file_response(
self,
@ -241,7 +248,8 @@ class OpenAIVectorStoreFilesConfig(BaseVectorStoreFilesConfig):
file_id: str,
api_base: str,
) -> Tuple[str, Dict[str, Any]]:
return f"{api_base}/{file_id}", {}
encoded_file_id = encode_url_path_segment(file_id, field_name="file_id")
return f"{api_base}/{encoded_file_id}", {}
def transform_delete_vector_store_file_response(
self,

6
litellm/llms/openai/vector_stores/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@ -108,7 +109,10 @@ class OpenAIVectorStoreConfig(BaseVectorStoreConfig):
litellm_params: dict,
extra_body: Optional[Dict[str, Any]] = None,
) -> Tuple[str, Dict]:
url = f"{api_base}/{vector_store_id}/search"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/search"
typed_request_body = VectorStoreSearchRequest(
query=query,
filters=vector_store_search_optional_params.get("filters", None),

34
litellm/llms/openai/videos/transformation.py
View file

@ -1,11 +1,13 @@
import mimetypes
from io import BufferedReader, BytesIO
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast
from urllib.parse import quote
import httpx
from httpx._types import RequestFiles
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.openai.image_edit.transformation import ImageEditRequestUtils
from litellm.secret_managers.main import get_secret_str
@ -220,11 +222,18 @@ class OpenAIVideoConfig(BaseVideoConfig):
- GET /v1/videos/{video_id}/content?variant=thumbnail
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video content download
url = f"{api_base.rstrip('/')}/{original_video_id}/content"
url = f"{api_base.rstrip('/')}/{encoded_video_id}/content"
if variant is not None:
url = f"{url}?variant={variant}"
# Encode the user-controlled ``variant`` so a value like
# ``thumbnail&extra=1`` cannot inject additional query params
# into the upstream request — same hardening rationale as the
# path-segment encoding above.
url = f"{url}?variant={quote(variant, safe='')}"
# No additional data needed for GET content request
data: Dict[str, Any] = {}
@ -247,9 +256,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
- POST /v1/videos/{video_id}/remix
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video remix
url = f"{api_base.rstrip('/')}/{original_video_id}/remix"
url = f"{api_base.rstrip('/')}/{encoded_video_id}/remix"
# Prepare the request data
data = {"prompt": prompt}
@ -391,9 +403,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
- DELETE /v1/videos/{video_id}
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for video delete
url = f"{api_base.rstrip('/')}/{original_video_id}"
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No data needed for DELETE request
data: Dict[str, Any] = {}
@ -427,9 +442,12 @@ class OpenAIVideoConfig(BaseVideoConfig):
"""
# Extract the original video_id (remove provider encoding if present)
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# For video retrieve, we just need to construct the URL
url = f"{api_base.rstrip('/')}/{original_video_id}"
url = f"{api_base.rstrip('/')}/{encoded_video_id}"
# No additional data needed for GET request
data: Dict[str, Any] = {}
@ -494,7 +512,11 @@ class OpenAIVideoConfig(BaseVideoConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base.rstrip('/')}/characters/{character_id}"
original_character_id = extract_original_character_id(character_id)
encoded_character_id = encode_url_path_segment(
original_character_id, field_name="character_id"
)
url = f"{api_base.rstrip('/')}/characters/{encoded_character_id}"
return url, {}
def transform_video_get_character_response(

6
litellm/llms/pg_vector/vector_stores/transformation.py
View file

@ -1,5 +1,6 @@
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.openai.vector_stores.transformation import OpenAIVectorStoreConfig
from litellm.secret_managers.main import get_secret_str
from litellm.types.router import GenericLiteLLMParams
@ -82,7 +83,10 @@ class PGVectorStoreConfig(OpenAIVectorStoreConfig):
litellm_params: dict,
extra_body: Optional[Dict[str, Any]] = None,
) -> Tuple[str, Dict]:
url = f"{api_base}/{vector_store_id}/search"
encoded_vector_store_id = encode_url_path_segment(
vector_store_id, field_name="vector_store_id"
)
url = f"{api_base}/{encoded_vector_store_id}/search"
_, request_body = super().transform_search_vector_store_request(
vector_store_id=vector_store_id,
query=query,

6
litellm/llms/ragflow/chat/transformation.py
View file

@ -13,6 +13,7 @@ Model name format:
from typing import List, Optional, Tuple
import litellm
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.openai.openai import OpenAIConfig
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.llms.openai import AllMessageValues
@ -126,10 +127,11 @@ class RAGFlowConfig(OpenAIConfig):
api_base = api_base[:-3] # Remove /v1
# Construct the RAGFlow-specific path
encoded_entity_id = encode_url_path_segment(entity_id, field_name="entity_id")
if endpoint_type == "chat":
path = f"/api/v1/chats_openai/{entity_id}/chat/completions"
path = f"/api/v1/chats_openai/{encoded_entity_id}/chat/completions"
else: # agent
path = f"/api/v1/agents_openai/{entity_id}/chat/completions"
path = f"/api/v1/agents_openai/{encoded_entity_id}/chat/completions"
# Ensure path starts with /
if not path.startswith("/"):

16
litellm/llms/runwayml/videos/transformation.py
View file

@ -6,6 +6,7 @@ from httpx._types import RequestFiles
import litellm
from litellm.constants import RUNWAYML_DEFAULT_API_VERSION
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.base_llm.videos.transformation import BaseVideoConfig
from litellm.llms.custom_httpx.http_handler import (
@ -334,9 +335,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
We'll retrieve the task and extract the video URL.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Get task status to retrieve video URL
url = f"{api_base}/tasks/{original_video_id}"
url = f"{api_base}/tasks/{encoded_video_id}"
params: Dict[str, Any] = {}
@ -495,9 +499,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
RunwayML uses task cancellation.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the URL for task cancellation
url = f"{api_base}/tasks/{original_video_id}/cancel"
url = f"{api_base}/tasks/{encoded_video_id}/cancel"
data: Dict[str, Any] = {}
@ -533,9 +540,12 @@ class RunwayMLVideoConfig(BaseVideoConfig):
RunwayML uses GET /v1/tasks/{task_id} to retrieve task status.
"""
original_video_id = extract_original_video_id(video_id)
encoded_video_id = encode_url_path_segment(
original_video_id, field_name="video_id"
)
# Construct the full URL for task status retrieval
url = f"{api_base}/tasks/{original_video_id}"
url = f"{api_base}/tasks/{encoded_video_id}"
# Empty dict for GET request (no body)
data: Dict[str, Any] = {}

12
litellm/llms/vertex_ai/batches/handler.py
View file

@ -4,7 +4,11 @@ from typing import Any, Coroutine, Dict, Optional, Union
import httpx
import litellm
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.litellm_core_utils.url_utils import (
async_safe_get,
encode_url_path_segment,
safe_get,
)
from litellm.llms.custom_httpx.http_handler import (
_get_httpx_client,
get_async_httpx_client,
@ -170,7 +174,8 @@ class VertexAIBatchPrediction(VertexLLM):
)
# Append batch_id to the URL
default_api_base = f"{default_api_base}/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
default_api_base = f"{default_api_base}/{encoded_batch_id}"
if len(default_api_base.split(":")) > 1:
endpoint = default_api_base.split(":")[-1]
@ -413,7 +418,8 @@ class VertexAIBatchPrediction(VertexLLM):
vertex_project=vertex_project or project_id,
)
retrieve_api_base_default = f"{default_api_base}/{batch_id}"
encoded_batch_id = encode_url_path_segment(batch_id, field_name="batch_id")
retrieve_api_base_default = f"{default_api_base}/{encoded_batch_id}"
cancel_api_base_default = f"{retrieve_api_base_default}:cancel"
_, api_base = self._check_custom_proxy(

15
litellm/llms/vertex_ai/common_utils.py
View file

@ -97,7 +97,7 @@ def get_vertex_ai_model_route(
Determine which handler to use for a Vertex AI model based on the model name.
Args:
model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "openai/gpt-oss-120b")
model: The model name (e.g., "llama3-405b", "gemini-pro", "gemma/gemma-3-12b-it", "xai/grok-4.1-fast-non-reasoning")
litellm_params: Optional litellm parameters dict that may contain base_model for routing
Returns:
@ -113,7 +113,7 @@ def get_vertex_ai_model_route(
>>> get_vertex_ai_model_route("gemma/gemma-3-12b-it")
VertexAIModelRoute.GEMMA
>>> get_vertex_ai_model_route("openai/gpt-oss-120b")
>>> get_vertex_ai_model_route("xai/grok-4.1-fast-non-reasoning")
VertexAIModelRoute.MODEL_GARDEN
>>> get_vertex_ai_model_route("1234567890", {"api_base": "http://10.96.32.8"})
@ -149,8 +149,11 @@ def get_vertex_ai_model_route(
if "gemma/" in model:
return VertexAIModelRoute.GEMMA
# Check for model garden openai models
if "openai" in model:
# Check for model garden OpenAI-compatible publisher models.
# Examples:
# - openai/gpt-oss-120b-maas
# - xai/grok-4.1-fast-non-reasoning
if "openai" in model or model.startswith("xai/"):
return VertexAIModelRoute.MODEL_GARDEN
# Check for gemini models
@ -256,8 +259,8 @@ def get_vertex_base_model_name(model: str) -> str:
>>> get_vertex_base_model_name("gemma/gemma-3-12b-it")
"gemma-3-12b-it"
>>> get_vertex_base_model_name("openai/gpt-oss-120b")
"gpt-oss-120b"
>>> get_vertex_base_model_name("xai/grok-4.1-fast-non-reasoning")
"grok-4.1-fast-non-reasoning"
>>> get_vertex_base_model_name("1234567890")
"1234567890"

26
litellm/llms/vertex_ai/gemini/transformation.py
View file

@ -759,16 +759,22 @@ def _transform_request_body( # noqa: PLR0915
]
data = RequestBody(contents=content)
if system_instructions is not None:
data["system_instruction"] = system_instructions
if tools is not None:
data["tools"] = tools
if tool_choice is not None:
data["toolConfig"] = tool_choice
if include_server_side_tool_invocations:
if "toolConfig" not in data:
data["toolConfig"] = {}
data["toolConfig"]["includeServerSideToolInvocations"] = True
# Vertex rejects system_instruction/tools/toolConfig alongside cachedContent.
# Treat dropping these fields as a request mutation guarded by modify_params.
can_send_cache_incompatible_fields = (
cached_content is None or litellm.modify_params is False
)
if can_send_cache_incompatible_fields:
if system_instructions is not None:
data["system_instruction"] = system_instructions
if tools is not None:
data["tools"] = tools
if tool_choice is not None:
data["toolConfig"] = tool_choice
if include_server_side_tool_invocations:
if "toolConfig" not in data:
data["toolConfig"] = {}
data["toolConfig"]["includeServerSideToolInvocations"] = True
if safety_settings is not None:
data["safetySettings"] = safety_settings
if generation_config is not None and len(generation_config) > 0:

69
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py
View file

@ -3,7 +3,7 @@ Google AI Studio /batchEmbedContents Embeddings Endpoint
"""
import json
from typing import Any, Dict, Literal, Optional, Union
from typing import Any, Dict, List, Literal, Optional, Tuple, Union
import httpx
@ -13,8 +13,8 @@ from litellm.llms.custom_httpx.http_handler import (
HTTPHandler,
get_async_httpx_client,
)
from litellm.types.llms.openai import EmbeddingInput
from litellm.types.llms.vertex_ai import (
GeminiEmbeddingInput,
VertexAIBatchEmbeddingsRequestBody,
VertexAIBatchEmbeddingsResponseObject,
)
@ -23,7 +23,6 @@ from litellm.types.utils import EmbeddingResponse
from ..gemini.vertex_and_google_ai_studio_gemini import VertexLLM
from .batch_embed_content_transformation import (
_is_file_reference,
_is_multimodal_input,
process_embed_content_response,
process_response,
transform_openai_input_gemini_content,
@ -32,9 +31,24 @@ from .batch_embed_content_transformation import (
class GoogleBatchEmbeddings(VertexLLM):
@staticmethod
def _flatten_and_detect_file_refs(
input: GeminiEmbeddingInput,
) -> Tuple[List[str], bool]:
"""Flatten nested input lists and detect file references."""
input_list = [input] if isinstance(input, str) else input
flat_elements = [
e
for item in input_list
for e in (item if isinstance(item, list) else [item])
if isinstance(e, str)
]
has_file_refs = any(_is_file_reference(e) for e in flat_elements)
return flat_elements, has_file_refs
def _resolve_file_references(
self,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
api_key: str,
sync_handler: HTTPHandler,
) -> Dict[str, Dict[str, str]]:
@ -42,7 +56,7 @@ class GoogleBatchEmbeddings(VertexLLM):
Resolve Gemini file references (files/...) to get mime_type and uri.
Args:
input: EmbeddingInput that may contain file references
input: GeminiEmbeddingInput that may contain file references
api_key: Gemini API key
sync_handler: HTTP client
@ -73,7 +87,7 @@ class GoogleBatchEmbeddings(VertexLLM):
async def _async_resolve_file_references(
self,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
api_key: str,
async_handler: AsyncHTTPHandler,
) -> Dict[str, Dict[str, str]]:
@ -81,7 +95,7 @@ class GoogleBatchEmbeddings(VertexLLM):
Async version of _resolve_file_references.
Args:
input: EmbeddingInput that may contain file references
input: GeminiEmbeddingInput that may contain file references
api_key: Gemini API key
async_handler: Async HTTP client
@ -110,10 +124,10 @@ class GoogleBatchEmbeddings(VertexLLM):
return resolved_files
def batch_embeddings(
def batch_embeddings( # noqa: PLR0915
self,
model: str,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
print_verbose,
model_response: EmbeddingResponse,
custom_llm_provider: Literal["gemini", "vertex_ai"],
@ -151,8 +165,7 @@ class GoogleBatchEmbeddings(VertexLLM):
optional_params = optional_params or {}
is_multimodal = _is_multimodal_input(input)
use_embed_content = is_multimodal or (custom_llm_provider == "vertex_ai")
use_embed_content = custom_llm_provider == "vertex_ai"
mode: Literal["embedding", "batch_embedding"]
if use_embed_content:
mode = "embedding"
@ -215,8 +228,22 @@ class GoogleBatchEmbeddings(VertexLLM):
resolved_files=resolved_files,
)
else:
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
if has_file_refs and not api_key:
raise ValueError(
"An API key is required to resolve Gemini file references (files/...). "
"Pass api_key= or set GEMINI_API_KEY."
)
resolved_files = {}
if api_key and has_file_refs:
resolved_files = self._resolve_file_references(
input=flat_elements, api_key=api_key, sync_handler=sync_handler
)
request_data = transform_openai_input_gemini_content(
input=input, model=model, optional_params=optional_params
input=input,
model=model,
optional_params=optional_params,
resolved_files=resolved_files,
)
## LOGGING
@ -264,7 +291,7 @@ class GoogleBatchEmbeddings(VertexLLM):
url: str,
data: Optional[Union[VertexAIBatchEmbeddingsRequestBody, dict]],
model_response: EmbeddingResponse,
input: EmbeddingInput,
input: GeminiEmbeddingInput,
timeout: Optional[Union[float, httpx.Timeout]],
headers={},
client: Optional[AsyncHTTPHandler] = None,
@ -303,8 +330,22 @@ class GoogleBatchEmbeddings(VertexLLM):
resolved_files=resolved_files,
)
else:
flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input)
if has_file_refs and not api_key:
raise ValueError(
"An API key is required to resolve Gemini file references (files/...). "
"Pass api_key= or set GEMINI_API_KEY."
)
resolved_files = {}
if api_key and has_file_refs:
resolved_files = await self._async_resolve_file_references(
input=flat_elements, api_key=api_key, async_handler=async_handler
)
data = transform_openai_input_gemini_content(
input=input, model=model, optional_params=optional_params or {}
input=input,
model=model,
optional_params=optional_params or {},
resolved_files=resolved_files,
)
## LOGGING

183
litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py
View file

@ -6,12 +6,12 @@ Why separate file? Make it easy to see how transformation works
from typing import Dict, List, Optional, Tuple
from litellm.types.llms.openai import EmbeddingInput
from litellm.types.llms.vertex_ai import (
BlobType,
ContentType,
EmbedContentRequest,
FileDataType,
GeminiEmbeddingInput,
PartType,
VertexAIBatchEmbeddingsRequestBody,
VertexAIBatchEmbeddingsResponseObject,
@ -114,33 +114,77 @@ def _parse_data_url(data_url: str) -> Tuple[str, str]:
return media_type, base64_data
def _is_multimodal_input(input: EmbeddingInput) -> bool:
def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool:
"""
Check if the input contains multimodal data (data URIs, file references, or GCS URLs).
Check if the input contains multimodal data (data URIs, file references,
GCS URLs, or nested lists for combined embeddings).
Args:
input: EmbeddingInput (str or List[str])
input: GeminiEmbeddingInput — str, List[str], or List[List[str]] for combined embeddings
Returns:
bool: True if any element is a data URI, file reference, or GCS URL
bool: True if any element is multimodal or a nested list
"""
if isinstance(input, str):
input_list = [input]
else:
input_list = input
return _is_multimodal_element(input)
for element in input_list:
if isinstance(element, str):
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
for element in input:
if isinstance(element, list):
if any(
_is_multimodal_element(sub) for sub in element if isinstance(sub, str)
):
return True
elif isinstance(element, str) and _is_multimodal_element(element):
return True
return False
def _is_multimodal_element(element: str) -> bool:
"""Check if a single string element is multimodal."""
if element.startswith("data:") and ";base64," in element:
return True
if _is_file_reference(element):
return True
if _is_gcs_url(element):
return True
return False
def _build_part_for_input(
element: str,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
) -> PartType:
"""
Build a single PartType for an input element, handling text, data URIs,
file references, and GCS URLs.
"""
resolved_files = resolved_files or {}
if element.startswith("data:") and ";base64," in element:
mime_type, base64_data = _parse_data_url(element)
blob: BlobType = {"mime_type": mime_type, "data": base64_data}
return PartType(inline_data=blob)
elif _is_gcs_url(element):
mime_type = _infer_mime_type_from_gcs_url(element)
file_data: FileDataType = {
"mime_type": mime_type,
"file_uri": element,
}
return PartType(file_data=file_data)
elif _is_file_reference(element):
if element not in resolved_files:
raise ValueError(f"File reference {element} not resolved")
file_info = resolved_files[element]
file_data_ref: FileDataType = {
"mime_type": file_info["mime_type"],
"file_uri": file_info["uri"],
}
return PartType(file_data=file_data_ref)
else:
return PartType(text=element)
_SUPPORTED_EMBED_PARAMS = {"outputDimensionality", "taskType", "title"}
@ -155,37 +199,60 @@ def _filter_embed_params(optional_params: dict) -> dict:
def transform_openai_input_gemini_content(
input: EmbeddingInput, model: str, optional_params: dict
input: GeminiEmbeddingInput,
model: str,
optional_params: dict,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
) -> VertexAIBatchEmbeddingsRequestBody:
"""
The content to embed. Only the parts.text fields will be counted.
Transform OpenAI embedding input to Gemini batchEmbedContents format.
Each input element becomes a separate EmbedContentRequest, supporting
text, data URIs, file references, and GCS URLs.
If an element is a list (nested input), all sub-elements are combined
into a single content with multiple parts, producing one combined
embedding for the group.
Examples:
input=["text", "image"] → 2 separate embeddings
input=[["text", "image"]] → 1 combined embedding
input=[["text", "image"], "x"] → 2 embeddings (1 combined + 1 separate)
"""
gemini_model_name = "models/{}".format(model)
gemini_params = _filter_embed_params(optional_params)
input_list = [input] if isinstance(input, str) else input
requests: List[EmbedContentRequest] = []
if isinstance(input, str):
for element in input_list:
if isinstance(element, list):
if not element:
raise ValueError("Nested input list must not be empty")
for sub in element:
if not isinstance(sub, str):
raise ValueError(
f"Elements inside a nested input list must be strings, got {type(sub)}"
)
parts = [
_build_part_for_input(sub, resolved_files=resolved_files)
for sub in element
]
else:
parts = [_build_part_for_input(element, resolved_files=resolved_files)]
request = EmbedContentRequest(
model=gemini_model_name,
content=ContentType(parts=[PartType(text=input)]),
content=ContentType(parts=parts),
**gemini_params,
)
requests.append(request)
else:
for i in input:
request = EmbedContentRequest(
model=gemini_model_name,
content=ContentType(parts=[PartType(text=i)]),
**gemini_params,
)
requests.append(request)
return VertexAIBatchEmbeddingsRequestBody(requests=requests)
def transform_openai_input_gemini_embed_content(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model: str,
optional_params: dict,
resolved_files: Optional[Dict[str, Dict[str, str]]] = None,
@ -194,7 +261,7 @@ def transform_openai_input_gemini_embed_content(
Transform OpenAI embedding input to Gemini embedContent format (multimodal).
Args:
input: EmbeddingInput (str or List[str]) with text, data URIs, or file references
input: GeminiEmbeddingInput with text, data URIs, or file references
model: Model name
optional_params: Additional parameters (taskType, outputDimensionality, etc.)
resolved_files: Dict mapping file names (files/abc) to {mime_type, uri}
@ -210,31 +277,14 @@ def transform_openai_input_gemini_embed_content(
parts: List[PartType] = []
for element in input_list:
if isinstance(element, list):
raise ValueError(
"Nested (combined) embeddings are not supported on the embedContent path. "
"Use the batchEmbedContents path or pass a flat list instead."
)
if not isinstance(element, str):
raise ValueError(f"Unsupported input type: {type(element)}")
if element.startswith("data:") and ";base64," in element:
mime_type, base64_data = _parse_data_url(element)
blob: BlobType = {"mime_type": mime_type, "data": base64_data}
parts.append(PartType(inline_data=blob))
elif _is_gcs_url(element):
mime_type = _infer_mime_type_from_gcs_url(element)
file_data: FileDataType = {
"mime_type": mime_type,
"file_uri": element,
}
parts.append(PartType(file_data=file_data))
elif _is_file_reference(element):
if element not in resolved_files:
raise ValueError(f"File reference {element} not resolved")
file_info = resolved_files[element]
file_data_ref: FileDataType = {
"mime_type": file_info["mime_type"],
"file_uri": file_info["uri"],
}
parts.append(PartType(file_data=file_data_ref))
else:
parts.append(PartType(text=element))
parts.append(_build_part_for_input(element, resolved_files=resolved_files))
request_body: dict = {
"content": ContentType(parts=parts),
@ -245,7 +295,7 @@ def transform_openai_input_gemini_embed_content(
def process_embed_content_response(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model_response: EmbeddingResponse,
model: str,
response_json: dict,
@ -291,7 +341,7 @@ def process_embed_content_response(
def process_response(
input: EmbeddingInput,
input: GeminiEmbeddingInput,
model_response: EmbeddingResponse,
model: str,
_predictions: VertexAIBatchEmbeddingsResponseObject,
@ -308,8 +358,29 @@ def process_response(
model_response.data = openai_embeddings
model_response.model = model
input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
has_nested = isinstance(input, list) and any(isinstance(e, list) for e in input)
if _is_multimodal_input(input) or has_nested:
input_list = input if isinstance(input, list) else [input]
text_elements: List[str] = []
for e in input_list:
if isinstance(e, list):
text_elements.extend(
sub
for sub in e
if isinstance(sub, str) and not _is_multimodal_element(sub)
)
elif isinstance(e, str) and not _is_multimodal_element(e):
text_elements.append(e)
if text_elements:
input_text = get_formatted_prompt(
data={"input": text_elements}, call_type="embedding"
)
prompt_tokens = token_counter(model=model, text=input_text)
else:
prompt_tokens = 0
else:
input_text = get_formatted_prompt(data={"input": input}, call_type="embedding")
prompt_tokens = token_counter(model=model, text=input_text)
model_response.usage = Usage(
prompt_tokens=prompt_tokens, total_tokens=prompt_tokens
)

9
litellm/llms/vertex_ai/vector_stores/search_api/transformation.py
View file

@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union
import httpx
from litellm import get_model_info
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.llms.base_llm.vector_store.transformation import BaseVectorStoreConfig
from litellm.llms.vertex_ai.vertex_llm_base import VertexBase
from litellm.types.router import GenericLiteLLMParams
@ -91,12 +92,18 @@ class VertexSearchAPIVectorStoreConfig(BaseVectorStoreConfig, VertexBase):
raise ValueError("vector_store_id is required")
if api_base:
return api_base.rstrip("/")
encoded_collection_id = encode_url_path_segment(
collection_id, field_name="vertex_collection_id"
)
encoded_datastore_id = encode_url_path_segment(
datastore_id, field_name="vector_store_id"
)
# Vertex AI Search API endpoint for search
return (
f"https://discoveryengine.googleapis.com/v1/"
f"projects/{vertex_project}/locations/{vertex_location}/"
f"collections/{collection_id}/dataStores/{datastore_id}/servingConfigs/default_config"
f"collections/{encoded_collection_id}/dataStores/{encoded_datastore_id}/servingConfigs/default_config"
)
def transform_search_vector_store_request(

23
litellm/llms/vertex_ai/vertex_model_garden/main.py
View file

@ -27,6 +27,17 @@ from ..common_utils import VertexAIError, get_vertex_base_model_name
from ..vertex_llm_base import VertexBase
def _vertex_model_garden_model_id_in_json_body(model: str) -> bool:
"""
Vertex catalog / publisher models are addressed as publisher/model (e.g.
xai/grok-4.1-fast-reasoning) on the shared OpenAPI URL, with the id in the JSON body.
Deployed Model Garden endpoints are typically a single segment (often numeric)
and use .../endpoints/{ENDPOINT_ID}/chat/completions with an empty model field.
"""
return "/" in model
def create_vertex_url(
vertex_location: str,
vertex_project: str,
@ -34,8 +45,13 @@ def create_vertex_url(
model: str,
api_base: Optional[str] = None,
) -> str:
"""Return the base url for the vertex garden models"""
"""Return the api base for vertex model garden (without /chat/completions)."""
base_url = get_vertex_base_url(vertex_location)
if _vertex_model_garden_model_id_in_json_body(model):
return (
f"{base_url}/v1/projects/{vertex_project}/locations/{vertex_location}"
"/endpoints/openapi"
)
return f"{base_url}/v1beta1/projects/{vertex_project}/locations/{vertex_location}/endpoints/{model}"
@ -129,7 +145,10 @@ class VertexAIModelGardenModels(VertexBase):
vertex_location=vertex_location or "us-central1",
vertex_api_version="v1beta1",
)
model = ""
# Publisher/catalog models: model id must be sent in the JSON body (OpenAPI route).
# Single-segment endpoint ids: model is encoded in the URL path; body model stays empty.
if not _vertex_model_garden_model_id_in_json_body(model):
model = ""
return openai_like_chat_completions.completion(
model=model,
messages=messages,

21
litellm/llms/volcengine/responses/transformation.py
View file

@ -17,6 +17,7 @@ from pydantic import fields as pyd_fields
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.core_helpers import process_response_headers
from litellm.litellm_core_utils.url_utils import encode_url_path_segment
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import (
_safe_convert_created_field,
)
@ -300,7 +301,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@ -333,7 +337,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}"
data: Dict = {}
return url, data
@ -372,7 +379,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
limit: int = 20,
order: Literal["asc", "desc"] = "desc",
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/input_items"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/input_items"
params: Dict[str, Any] = {}
if after is not None:
params["after"] = after
@ -408,7 +418,10 @@ class VolcEngineResponsesAPIConfig(OpenAIResponsesAPIConfig):
litellm_params: GenericLiteLLMParams,
headers: dict,
) -> Tuple[str, Dict]:
url = f"{api_base}/{response_id}/cancel"
encoded_response_id = encode_url_path_segment(
response_id, field_name="response_id"
)
url = f"{api_base}/{encoded_response_id}/cancel"
data: Dict = {}
return url, data

1
litellm/llms/xai/chat/transformation.py
View file

@ -43,6 +43,7 @@ class XAIChatConfig(OpenAIGPTConfig):
"logprobs",
"max_tokens",
"n",
"parallel_tool_calls",
"presence_penalty",
"response_format",
"seed",

13
litellm/main.py
View file

@ -4923,8 +4923,17 @@ def embedding( # noqa: PLR0915
if encoding_format is not None:
optional_params["encoding_format"] = encoding_format
else:
# Omiting causes openai sdk to add default value of "float"
optional_params["encoding_format"] = None
env_fmt = get_secret_str("LITELLM_DEFAULT_EMBEDDING_ENCODING_FORMAT")
if env_fmt is not None and env_fmt.strip().lower() == "none":
optional_params.pop("encoding_format", None)
else:
_default_fmt = (
optional_params.get("encoding_format") or env_fmt or "float"
)
if _default_fmt.strip().lower() == "none":
optional_params.pop("encoding_format", None)
else:
optional_params["encoding_format"] = _default_fmt
api_version = None

66
litellm/model_prices_and_context_window_backup.json
View file

@ -33429,6 +33429,72 @@
"source": "https://console.cloud.google.com/vertex-ai/publishers/openai/model-garden/gpt-oss-120b-maas",
"supports_reasoning": true
},
"vertex_ai/xai/grok-4.1-fast-non-reasoning": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 2000000,
"max_output_tokens": 2000000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"vertex_ai/xai/grok-4.1-fast-reasoning": {
"cache_read_input_token_cost": 5e-08,
"input_cost_per_token": 2e-07,
"litellm_provider": "vertex_ai",
"max_input_tokens": 2000000,
"max_output_tokens": 2000000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"vertex_ai/xai/grok-4.20-non-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 2000000,
"max_output_tokens": 2000000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)",
"supports_function_calling": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"vertex_ai/xai/grok-4.20-reasoning": {
"cache_read_input_token_cost": 2e-07,
"input_cost_per_token": 2e-06,
"litellm_provider": "vertex_ai",
"max_input_tokens": 2000000,
"max_output_tokens": 2000000,
"max_tokens": 2000000,
"mode": "chat",
"output_cost_per_token": 6e-06,
"source": "https://docs.x.ai/docs/models (Vertex AI Model Garden)",
"supports_function_calling": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_tool_choice": true,
"supports_vision": true,
"supports_web_search": true
},
"vertex_ai/qwen/qwen3-235b-a22b-instruct-2507-maas": {
"input_cost_per_token": 2.5e-07,
"litellm_provider": "vertex_ai-qwen_models",

22
litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py
View file

@ -33,10 +33,12 @@ def get_request_base_url(request: Request) -> str:
"""
Get the base URL for the request, considering X-Forwarded-* headers.
When behind a proxy (like nginx), the proxy may set:
- X-Forwarded-Proto: The original protocol (http/https)
- X-Forwarded-Host: The original host (may include port)
- X-Forwarded-Port: The original port (if not in Host header)
X-Forwarded-Proto / X-Forwarded-Host / X-Forwarded-Port are only honoured
when the request comes from a configured trusted proxy
(``use_x_forwarded_for`` enabled AND caller in ``mcp_trusted_proxy_ranges``).
Otherwise the request's literal ``base_url`` is returned, so an
untrusted caller cannot poison OAuth-discovery / redirect_uri values
by injecting headers.
Args:
request: FastAPI Request object
@ -47,34 +49,28 @@ def get_request_base_url(request: Request) -> str:
base_url = str(request.base_url).rstrip("/")
parsed = urlparse(base_url)
# Get forwarded headers
if not IPAddressUtils.is_request_from_trusted_proxy(request):
return base_url
x_forwarded_proto = request.headers.get("X-Forwarded-Proto")
x_forwarded_host = request.headers.get("X-Forwarded-Host")
x_forwarded_port = request.headers.get("X-Forwarded-Port")
# Start with the original scheme
scheme = x_forwarded_proto if x_forwarded_proto else parsed.scheme
# Handle host and port
if x_forwarded_host:
# X-Forwarded-Host may already include port (e.g., "example.com:8080")
if ":" in x_forwarded_host and not x_forwarded_host.startswith("["):
# Host includes port
netloc = x_forwarded_host
elif x_forwarded_port:
# Port is separate
netloc = f"{x_forwarded_host}:{x_forwarded_port}"
else:
# Just host, no explicit port
netloc = x_forwarded_host
else:
# No X-Forwarded-Host, use original netloc
netloc = parsed.netloc
if x_forwarded_port and ":" not in netloc:
# Add forwarded port if not already in netloc
netloc = f"{netloc}:{x_forwarded_port}"
# Reconstruct the URL
return urlunparse((scheme, netloc, parsed.path, "", "", ""))

69
litellm/proxy/_experimental/mcp_server/mcp_server_manager.py
View file

@ -169,6 +169,37 @@ def _deserialize_json_dict(data: Any) -> Optional[Dict[str, str]]:
class MCPServerManager:
_STDIO_ENV_TEMPLATE_PATTERN = re.compile(r"^\$\{(X-[^}]+)\}$")
@staticmethod
def _resolve_oauth2_flow(
*,
auth_type: Optional[MCPAuthType],
oauth2_flow: Optional[str],
token_url: Optional[str],
authorization_url: Optional[str],
client_id: Optional[str],
client_secret: Optional[str],
) -> Optional[Literal["client_credentials", "authorization_code"]]:
"""Infer oauth2_flow for legacy records that omit the field.
DB rows created before oauth2_flow support may have OAuth2 client
credentials + token_url but a null oauth2_flow. Treat these as M2M,
unless authorization_url is present (interactive OAuth).
"""
if oauth2_flow in ("client_credentials", "authorization_code"):
return cast(
Literal["client_credentials", "authorization_code"], oauth2_flow
)
if oauth2_flow:
# Ignore unknown/untyped values and continue legacy inference.
return None
if auth_type != MCPAuth.oauth2:
return None
if authorization_url:
return None
if token_url and client_id and client_secret:
return "client_credentials"
return None
def __init__(self):
self.registry: Dict[str, MCPServer] = {}
self.config_mcp_servers: Dict[str, MCPServer] = {}
@ -342,7 +373,14 @@ class MCPServerManager:
# oauth specific fields
client_id=server_config.get("client_id", None),
client_secret=server_config.get("client_secret", None),
oauth2_flow=server_config.get("oauth2_flow", None),
oauth2_flow=self._resolve_oauth2_flow(
auth_type=auth_type,
oauth2_flow=server_config.get("oauth2_flow", None),
token_url=resolved_token_url,
authorization_url=resolved_authorization_url,
client_id=server_config.get("client_id", None),
client_secret=server_config.get("client_secret", None),
),
scopes=resolved_scopes,
authorization_url=resolved_authorization_url,
token_url=resolved_token_url,
@ -679,7 +717,17 @@ class MCPServerManager:
client_id=client_id_value or getattr(mcp_server, "client_id", None),
client_secret=client_secret_value
or getattr(mcp_server, "client_secret", None),
oauth2_flow=getattr(mcp_server, "oauth2_flow", None),
oauth2_flow=self._resolve_oauth2_flow(
auth_type=auth_type,
oauth2_flow=getattr(mcp_server, "oauth2_flow", None),
token_url=mcp_server.token_url
or getattr(mcp_oauth_metadata, "token_url", None),
authorization_url=mcp_server.authorization_url
or getattr(mcp_oauth_metadata, "authorization_url", None),
client_id=client_id_value or getattr(mcp_server, "client_id", None),
client_secret=client_secret_value
or getattr(mcp_server, "client_secret", None),
),
scopes=resolved_scopes,
authorization_url=mcp_server.authorization_url
or getattr(mcp_oauth_metadata, "authorization_url", None),
@ -2426,7 +2474,7 @@ class MCPServerManager:
)
)
async def _call_regular_mcp_tool(
async def _call_regular_mcp_tool( # noqa: PLR0915
self,
mcp_server: MCPServer,
original_tool_name: str,
@ -2489,7 +2537,11 @@ class MCPServerManager:
# oauth2 headers
extra_headers: Optional[Dict[str, str]] = None
if mcp_server.auth_type == MCPAuth.oauth2:
extra_headers = oauth2_headers
if mcp_server.has_client_credentials:
# For M2M OAuth servers, Authorization must come from token fetch.
extra_headers = None
else:
extra_headers = oauth2_headers
if mcp_server.extra_headers and raw_headers:
if extra_headers is None:
@ -2501,6 +2553,11 @@ class MCPServerManager:
for header in mcp_server.extra_headers:
if not isinstance(header, str):
continue
if (
mcp_server.has_client_credentials
and header.lower() == "authorization"
):
continue
header_value = normalized_raw_headers.get(header.lower())
if header_value is None:
continue
@ -2536,6 +2593,10 @@ class MCPServerManager:
)
extra_headers.update(hook_extra_headers)
# Reset to None if no headers were actually added
if extra_headers is not None and len(extra_headers) == 0:
extra_headers = None
stdio_env = self._build_stdio_env(mcp_server, raw_headers)
client = await self._create_mcp_client(

48
litellm/proxy/_experimental/mcp_server/server.py
View file

@ -153,6 +153,7 @@ if MCP_AVAILABLE:
MCPAuthenticatedUser,
)
from litellm.proxy._experimental.mcp_server.mcp_server_manager import (
MCPServerManager,
global_mcp_server_manager,
)
from litellm.proxy._experimental.mcp_server.openapi_to_mcp_generator import (
@ -900,6 +901,20 @@ if MCP_AVAILABLE:
allowed_mcp_server_id
)
if mcp_server is not None:
# Apply oauth2_flow resolution for legacy DB rows where it may be NULL
resolved_flow = MCPServerManager._resolve_oauth2_flow(
auth_type=mcp_server.auth_type,
oauth2_flow=mcp_server.oauth2_flow,
token_url=mcp_server.token_url,
authorization_url=mcp_server.authorization_url,
client_id=mcp_server.client_id,
client_secret=mcp_server.client_secret,
)
if resolved_flow and resolved_flow != mcp_server.oauth2_flow:
# Create a new instance with the resolved flow for this request
mcp_server = mcp_server.model_copy(
update={"oauth2_flow": resolved_flow}
)
allowed_mcp_servers.append(mcp_server)
if mcp_servers is not None:
@ -1100,8 +1115,13 @@ if MCP_AVAILABLE:
extra_headers: Optional[Dict[str, str]] = None
if server.auth_type == MCPAuth.oauth2:
# Copy to avoid mutating the original dict (important for parallel fetching)
extra_headers = oauth2_headers.copy() if oauth2_headers else None
# For OAuth2 M2M servers, upstream Authorization must come from
# client_credentials token fetch, never from caller headers.
if server.has_client_credentials:
extra_headers = None
else:
# Copy to avoid mutating the original dict (important for parallel fetching)
extra_headers = oauth2_headers.copy() if oauth2_headers else None
if server.extra_headers and raw_headers:
if extra_headers is None:
@ -1114,11 +1134,17 @@ if MCP_AVAILABLE:
for header in server.extra_headers:
if not isinstance(header, str):
continue
if server.has_client_credentials and header.lower() == "authorization":
continue
header_value = normalized_raw_headers.get(header.lower())
if header_value is None:
continue
extra_headers[header] = header_value
# Reset to None if no headers were actually added
if extra_headers is not None and len(extra_headers) == 0:
extra_headers = None
if server_auth_header is None:
server_auth_header = mcp_auth_header
@ -1377,11 +1403,19 @@ if MCP_AVAILABLE:
spend_meta["per_server_tool_counts"] = per_server_tool_counts
end_time = datetime.now()
await litellm_logging_obj.async_success_handler(
result=all_tools,
start_time=list_tools_start_time,
end_time=end_time,
)
try:
await litellm_logging_obj.async_success_handler(
result=all_tools,
start_time=list_tools_start_time,
end_time=end_time,
)
except Exception as log_exc:
# list_tools responses must not be dropped due to non-blocking
# observability/serialization failures.
verbose_logger.warning(
"MCP list_tools success logging failed (continuing): %s",
log_exc,
)
verbose_logger.info(
f"Successfully fetched {len(all_tools)} tools total from all MCP servers"

0
litellm/proxy/_experimental/out/404.html → litellm/proxy/_experimental/out/404/index.html
View file

0
litellm/proxy/_experimental/out/_not-found.html → litellm/proxy/_experimental/out/_not-found/index.html
View file

0
litellm/proxy/_experimental/out/api-reference.html → litellm/proxy/_experimental/out/api-reference/index.html
View file

0
litellm/proxy/_experimental/out/experimental/api-playground.html → litellm/proxy/_experimental/out/experimental/api-playground/index.html
View file

0
litellm/proxy/_experimental/out/experimental/budgets.html → litellm/proxy/_experimental/out/experimental/budgets/index.html
View file

0
litellm/proxy/_experimental/out/experimental/caching.html → litellm/proxy/_experimental/out/experimental/caching/index.html
View file

0
litellm/proxy/_experimental/out/experimental/claude-code-plugins.html → litellm/proxy/_experimental/out/experimental/claude-code-plugins/index.html
View file

0
litellm/proxy/_experimental/out/experimental/old-usage.html → litellm/proxy/_experimental/out/experimental/old-usage/index.html
View file

0
litellm/proxy/_experimental/out/experimental/prompts.html → litellm/proxy/_experimental/out/experimental/prompts/index.html
View file

0
litellm/proxy/_experimental/out/experimental/tag-management.html → litellm/proxy/_experimental/out/experimental/tag-management/index.html
View file

0
litellm/proxy/_experimental/out/guardrails.html → litellm/proxy/_experimental/out/guardrails/index.html
View file

0
litellm/proxy/_experimental/out/login.html → litellm/proxy/_experimental/out/login/index.html
View file

0
litellm/proxy/_experimental/out/logs.html → litellm/proxy/_experimental/out/logs/index.html
View file

0
litellm/proxy/_experimental/out/mcp/oauth/callback.html → litellm/proxy/_experimental/out/mcp/oauth/callback/index.html
View file

0
litellm/proxy/_experimental/out/model-hub.html → litellm/proxy/_experimental/out/model-hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub.html → litellm/proxy/_experimental/out/model_hub/index.html
View file

0
litellm/proxy/_experimental/out/model_hub_table.html → litellm/proxy/_experimental/out/model_hub_table/index.html
View file

0
litellm/proxy/_experimental/out/models-and-endpoints.html → litellm/proxy/_experimental/out/models-and-endpoints/index.html
View file

0
litellm/proxy/_experimental/out/onboarding.html → litellm/proxy/_experimental/out/onboarding/index.html
View file

0
litellm/proxy/_experimental/out/organizations.html → litellm/proxy/_experimental/out/organizations/index.html
View file

0
litellm/proxy/_experimental/out/playground.html → litellm/proxy/_experimental/out/playground/index.html
View file

0
litellm/proxy/_experimental/out/policies.html → litellm/proxy/_experimental/out/policies/index.html
View file

0
litellm/proxy/_experimental/out/settings/admin-settings.html → litellm/proxy/_experimental/out/settings/admin-settings/index.html
View file

0
litellm/proxy/_experimental/out/settings/logging-and-alerts.html → litellm/proxy/_experimental/out/settings/logging-and-alerts/index.html
View file

0
litellm/proxy/_experimental/out/settings/router-settings.html → litellm/proxy/_experimental/out/settings/router-settings/index.html
View file

0
litellm/proxy/_experimental/out/settings/ui-theme.html → litellm/proxy/_experimental/out/settings/ui-theme/index.html
View file

0
litellm/proxy/_experimental/out/skills.html → litellm/proxy/_experimental/out/skills/index.html
View file

0
litellm/proxy/_experimental/out/teams.html → litellm/proxy/_experimental/out/teams/index.html
View file

0
litellm/proxy/_experimental/out/test-key.html → litellm/proxy/_experimental/out/test-key/index.html
View file

0
litellm/proxy/_experimental/out/tools/mcp-servers.html → litellm/proxy/_experimental/out/tools/mcp-servers/index.html
View file

0
litellm/proxy/_experimental/out/tools/vector-stores.html → litellm/proxy/_experimental/out/tools/vector-stores/index.html
View file

0
litellm/proxy/_experimental/out/usage.html → litellm/proxy/_experimental/out/usage/index.html
View file

0
litellm/proxy/_experimental/out/users.html → litellm/proxy/_experimental/out/users/index.html
View file

0
litellm/proxy/_experimental/out/virtual-keys.html → litellm/proxy/_experimental/out/virtual-keys/index.html
View file

28
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -3572,7 +3572,7 @@
"/anthropic/{endpoint}": {
"delete": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)",
"operationId": "anthropic_proxy_route_anthropic__endpoint__put",
"operationId": "anthropic_proxy_route_anthropic__endpoint__delete",
"parameters": [
{
"in": "path",
@ -3616,7 +3616,7 @@
},
"get": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)",
"operationId": "anthropic_proxy_route_anthropic__endpoint__put",
"operationId": "anthropic_proxy_route_anthropic__endpoint__get",
"parameters": [
{
"in": "path",
@ -3660,7 +3660,7 @@
},
"patch": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)",
"operationId": "anthropic_proxy_route_anthropic__endpoint__put",
"operationId": "anthropic_proxy_route_anthropic__endpoint__patch",
"parameters": [
{
"in": "path",
@ -3704,7 +3704,7 @@
},
"post": {
"description": "[Docs](https://docs.litellm.ai/docs/pass_through/anthropic_completion)",
"operationId": "anthropic_proxy_route_anthropic__endpoint__put",
"operationId": "anthropic_proxy_route_anthropic__endpoint__post",
"parameters": [
{
"in": "path",
@ -13260,7 +13260,7 @@
"/langfuse/{endpoint}": {
"delete": {
"description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)",
"operationId": "langfuse_proxy_route_langfuse__endpoint__put",
"operationId": "langfuse_proxy_route_langfuse__endpoint__delete",
"parameters": [
{
"in": "path",
@ -13299,7 +13299,7 @@
},
"get": {
"description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)",
"operationId": "langfuse_proxy_route_langfuse__endpoint__put",
"operationId": "langfuse_proxy_route_langfuse__endpoint__get",
"parameters": [
{
"in": "path",
@ -13338,7 +13338,7 @@
},
"patch": {
"description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)",
"operationId": "langfuse_proxy_route_langfuse__endpoint__put",
"operationId": "langfuse_proxy_route_langfuse__endpoint__patch",
"parameters": [
{
"in": "path",
@ -13377,7 +13377,7 @@
},
"post": {
"description": "Call Langfuse via LiteLLM proxy. Works with Langfuse SDK.\n\n[Docs](https://docs.litellm.ai/docs/pass_through/langfuse)",
"operationId": "langfuse_proxy_route_langfuse__endpoint__put",
"operationId": "langfuse_proxy_route_langfuse__endpoint__post",
"parameters": [
{
"in": "path",
@ -26883,7 +26883,7 @@
"/toolset/{toolset_name}/mcp": {
"delete": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_delete",
"parameters": [
{
"in": "path",
@ -26922,7 +26922,7 @@
},
"get": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_get",
"parameters": [
{
"in": "path",
@ -26961,7 +26961,7 @@
},
"head": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_head",
"parameters": [
{
"in": "path",
@ -27000,7 +27000,7 @@
},
"options": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_options",
"parameters": [
{
"in": "path",
@ -27039,7 +27039,7 @@
},
"patch": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_patch",
"parameters": [
{
"in": "path",
@ -27078,7 +27078,7 @@
},
"post": {
"description": "Namespace a toolset as its own MCP endpoint.\n\nConnecting to /toolset/<name>/mcp exposes exactly the tools defined in\nthe toolset. Access is enforced: non-admin API keys must have the toolset\nlisted in their object_permission.mcp_toolsets grant list, or the request\nwill be rejected with a 403.",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_put",
"operationId": "toolset_mcp_route_toolset__toolset_name__mcp_post",
"parameters": [
{
"in": "path",

40
litellm/proxy/_lazy_openapi_snapshot.py
View file

@ -13,6 +13,7 @@ from pathlib import Path
from typing import Dict, Optional
SNAPSHOT_FILE = Path(__file__).parent / "_lazy_openapi_snapshot.json"
HTTP_METHODS = {"delete", "get", "head", "options", "patch", "post", "put"}
def load_snapshot() -> Optional[Dict[str, Dict]]:
@ -25,6 +26,39 @@ def load_snapshot() -> Optional[Dict[str, Dict]]:
return None
def _normalize_operation_ids(paths: Dict[str, Dict]) -> None:
"""Make FastAPI-generated operation IDs stable for multi-method routes.
FastAPI derives the default operation ID suffix from the first item in the
route's methods set. For routes registered with several HTTP methods, that
set iteration order can vary between processes, which makes the snapshot
drift even when no routes changed.
"""
for path_ops in paths.values():
if not isinstance(path_ops, dict):
continue
methods = {method for method in path_ops if method in HTTP_METHODS}
if not methods:
continue
for method, operation in path_ops.items():
if method not in HTTP_METHODS or not isinstance(operation, dict):
continue
operation_id = operation.get("operationId")
if not isinstance(operation_id, str):
continue
for suffix in methods:
suffix_token = f"_{suffix}"
if operation_id.endswith(suffix_token):
operation["operationId"] = (
operation_id[: -len(suffix_token)] + f"_{method}"
)
break
def generate_snapshot() -> Dict[str, Dict]:
import importlib
@ -52,13 +86,15 @@ def generate_snapshot() -> Dict[str, Dict]:
if not feat_routes:
continue
full = get_openapi(title=app.title, version=app.version, routes=feat_routes)
paths = full.get("paths", {})
_normalize_operation_ids(paths)
# Group all of a feature's routes under one tag.
for path_ops in full.get("paths", {}).values():
for path_ops in paths.values():
for op in path_ops.values():
if isinstance(op, dict):
op["tags"] = [feat.name]
fragments[feat.name] = {
"paths": full.get("paths", {}),
"paths": paths,
"components": {"schemas": full.get("components", {}).get("schemas", {})},
}
return fragments

5
litellm/proxy/_types.py
View file

@ -668,6 +668,8 @@ class LiteLLMRoutes(enum.Enum):
"/models/{model_id}",
"/guardrails/list",
"/v2/guardrails/list",
"/project/list",
"/project/info",
]
+ spend_tracking_routes
+ key_management_routes
@ -692,6 +694,9 @@ class LiteLLMRoutes(enum.Enum):
"/model/{model_id}/update",
"/prompt/list",
"/prompt/info",
# Project read routes - endpoint scopes results to caller's teams (non-admin)
"/project/list",
"/project/info",
# Invitation routes - org/team admins checked in endpoint via _user_has_admin_privileges
"/invitation/new",
"/invitation/delete",

Some files were not shown because too many files have changed in this diff Show more