mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-07 02:59:05 +00:00
Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_anthropic_wif_backend
# Conflicts: # tests/test_litellm/test_lazy_imports.py
This commit is contained in:
mateo-berri
commit
779da8f79b
781 changed files with 13107 additions and 9397 deletions
|
|
@ -105,13 +105,13 @@
|
|||
"limit": 109
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 38283
|
||||
"limit": 38271
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 19584
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 29829
|
||||
"limit": 29814
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 110
|
||||
|
|
|
|||
|
|
@ -13,7 +13,6 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
|
|||
### INIT VARIABLES #########################
|
||||
import threading
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
|
||||
import dotenv as _dotenv
|
||||
|
|
@ -46,6 +45,8 @@ from typing import (
|
|||
TYPE_CHECKING,
|
||||
Union,
|
||||
)
|
||||
from litellm.types.integrations.datadog import DatadogInitParams
|
||||
from litellm.types.integrations.newrelic import NewRelicInitParams
|
||||
from litellm._logging import (
|
||||
set_verbose,
|
||||
_turn_on_debug,
|
||||
|
|
@ -94,7 +95,8 @@ from litellm.constants import (
|
|||
DEFAULT_SOFT_BUDGET,
|
||||
DEFAULT_ALLOWED_FAILS,
|
||||
)
|
||||
# httpx is lazy-loaded via __getattr__
|
||||
import httpx
|
||||
|
||||
# register_async_client_cleanup is lazy-loaded and called on first access
|
||||
|
||||
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
|
||||
|
|
@ -362,6 +364,8 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {}
|
|||
include_cost_in_streaming_usage: bool = False
|
||||
reasoning_auto_summary: bool = False
|
||||
### PROMPTS ####
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
|
||||
prompt_name_config_map: Dict[str, PromptSpec] = {}
|
||||
|
||||
##################
|
||||
|
|
@ -1267,203 +1271,206 @@ openai_video_generation_models = ["sora-2"]
|
|||
# get_llm_provider is lazy-loaded via __getattr__
|
||||
# remove_index_from_tool_calls is lazy-loaded via __getattr__
|
||||
|
||||
# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__
|
||||
# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING
|
||||
# so static type checkers still see them
|
||||
if TYPE_CHECKING:
|
||||
_key_management_settings: KeyManagementSettings
|
||||
# Import KeyManagementSettings here (before utils import) because _key_management_settings
|
||||
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
|
||||
from litellm.types.secret_managers.main import KeyManagementSettings
|
||||
|
||||
from .utils import client
|
||||
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
|
||||
|
||||
from .llms.custom_llm import CustomLLM
|
||||
from .llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
|
||||
from .llms.deprecated_providers.palm import (
|
||||
PalmConfig,
|
||||
) # here to prevent breaking changes
|
||||
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
|
||||
from .llms.gemini.common_utils import GeminiModelInfo
|
||||
# client must be imported immediately as it's used as a decorator at function definition time
|
||||
from .utils import client
|
||||
|
||||
from .llms.vertex_ai.vertex_embeddings.transformation import (
|
||||
VertexAITextEmbeddingConfig,
|
||||
)
|
||||
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
|
||||
# (which imports tiktoken) at import time
|
||||
|
||||
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
|
||||
from .llms.custom_llm import CustomLLM
|
||||
from .llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
|
||||
from .llms.deprecated_providers.palm import (
|
||||
PalmConfig,
|
||||
) # here to prevent breaking changes
|
||||
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
|
||||
from .llms.gemini.common_utils import GeminiModelInfo
|
||||
|
||||
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
|
||||
AmazonTitanV2Config,
|
||||
)
|
||||
from .llms.topaz.common_utils import TopazModelInfo
|
||||
|
||||
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
|
||||
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
|
||||
from .llms.xai.common_utils import XAIModelInfo
|
||||
from .llms.vertex_ai.vertex_embeddings.transformation import (
|
||||
VertexAITextEmbeddingConfig,
|
||||
)
|
||||
|
||||
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
|
||||
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
|
||||
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
|
||||
|
||||
# Import LlmProviders here (before main import) because it's imported during import time
|
||||
# in multiple places including openai.py (via main import)
|
||||
|
||||
## Lazy loading this is not straightforward, will leave it here for now.
|
||||
from .main import *
|
||||
from .compression import compress
|
||||
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
|
||||
AmazonTitanV2Config,
|
||||
)
|
||||
from .llms.topaz.common_utils import TopazModelInfo
|
||||
|
||||
# Skills API
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .evals.main import (
|
||||
create_eval,
|
||||
acreate_eval,
|
||||
list_evals,
|
||||
alist_evals,
|
||||
get_eval,
|
||||
aget_eval,
|
||||
delete_eval,
|
||||
adelete_eval,
|
||||
cancel_eval,
|
||||
acancel_eval,
|
||||
create_run,
|
||||
acreate_run,
|
||||
list_runs,
|
||||
alist_runs,
|
||||
get_run,
|
||||
aget_run,
|
||||
delete_run,
|
||||
adelete_run,
|
||||
cancel_run,
|
||||
acancel_run,
|
||||
)
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
from .exceptions import (
|
||||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
RateLimitErrorCategory,
|
||||
RateLimitType,
|
||||
ServiceUnavailableError,
|
||||
BadGatewayError,
|
||||
OpenAIError,
|
||||
ContextWindowExceededError,
|
||||
ContentPolicyViolationError,
|
||||
BudgetExceededError,
|
||||
APIError,
|
||||
Timeout,
|
||||
APIConnectionError,
|
||||
UnsupportedParamsError,
|
||||
APIResponseValidationError,
|
||||
UnprocessableEntityError,
|
||||
InternalServerError,
|
||||
JSONSchemaValidationError,
|
||||
LITELLM_EXCEPTION_TYPES,
|
||||
MockException,
|
||||
)
|
||||
from .budget_manager import BudgetManager
|
||||
from .proxy.proxy_cli import run_server
|
||||
from .router import Router
|
||||
from .assistants.main import *
|
||||
from .batches.main import *
|
||||
from .images.main import *
|
||||
from .videos.main import *
|
||||
from .batch_completion.main import *
|
||||
from .rerank_api.main import *
|
||||
from .llms.anthropic.experimental_pass_through.messages.handler import *
|
||||
from .responses.main import *
|
||||
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
|
||||
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
|
||||
from .llms.xai.common_utils import XAIModelInfo
|
||||
|
||||
# Interactions API is available as litellm.interactions module
|
||||
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
||||
from . import interactions
|
||||
from .interactions.agents.main import (
|
||||
acreate as acreate_agent,
|
||||
create as create_agent,
|
||||
alist as alist_agents,
|
||||
list as list_agents,
|
||||
aget as aget_agent,
|
||||
get as get_agent,
|
||||
adelete as adelete_agent,
|
||||
delete as delete_agent,
|
||||
alist_versions as alist_agent_versions,
|
||||
list_versions as list_agent_versions,
|
||||
)
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .containers.main import *
|
||||
from .ocr.main import *
|
||||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
_arealtime,
|
||||
acreate_realtime_client_secret,
|
||||
acreate_realtime_transcription_session,
|
||||
arealtime_calls,
|
||||
)
|
||||
from .responses.main import _aresponses_websocket
|
||||
from .fine_tuning.main import *
|
||||
from .files.main import *
|
||||
from .vector_store_files.main import (
|
||||
acreate as avector_store_file_create,
|
||||
adelete as avector_store_file_delete,
|
||||
alist as avector_store_file_list,
|
||||
aretrieve as avector_store_file_retrieve,
|
||||
aretrieve_content as avector_store_file_content,
|
||||
aupdate as avector_store_file_update,
|
||||
create as vector_store_file_create,
|
||||
delete as vector_store_file_delete,
|
||||
list as vector_store_file_list,
|
||||
retrieve as vector_store_file_retrieve,
|
||||
retrieve_content as vector_store_file_content,
|
||||
update as vector_store_file_update,
|
||||
)
|
||||
from .scheduler import *
|
||||
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
|
||||
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
|
||||
|
||||
### ADAPTERS ###
|
||||
import litellm.anthropic_interface as anthropic
|
||||
# Import LlmProviders here (before main import) because it's imported during import time
|
||||
# in multiple places including openai.py (via main import)
|
||||
from litellm.types.utils import LlmProviders
|
||||
|
||||
### Vector Store Registry ###
|
||||
## Lazy loading this is not straightforward, will leave it here for now.
|
||||
from .main import *
|
||||
from .compression import compress
|
||||
|
||||
### RAG ###
|
||||
from . import rag
|
||||
# Skills API
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .evals.main import (
|
||||
create_eval,
|
||||
acreate_eval,
|
||||
list_evals,
|
||||
alist_evals,
|
||||
get_eval,
|
||||
aget_eval,
|
||||
delete_eval,
|
||||
adelete_eval,
|
||||
cancel_eval,
|
||||
acancel_eval,
|
||||
create_run,
|
||||
acreate_run,
|
||||
list_runs,
|
||||
alist_runs,
|
||||
get_run,
|
||||
aget_run,
|
||||
delete_run,
|
||||
adelete_run,
|
||||
cancel_run,
|
||||
acancel_run,
|
||||
)
|
||||
from .integrations import *
|
||||
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
|
||||
from .exceptions import (
|
||||
AuthenticationError,
|
||||
InvalidRequestError,
|
||||
BadRequestError,
|
||||
ImageFetchError,
|
||||
NotFoundError,
|
||||
PermissionDeniedError,
|
||||
RateLimitError,
|
||||
RateLimitErrorCategory,
|
||||
RateLimitType,
|
||||
ServiceUnavailableError,
|
||||
BadGatewayError,
|
||||
OpenAIError,
|
||||
ContextWindowExceededError,
|
||||
ContentPolicyViolationError,
|
||||
BudgetExceededError,
|
||||
APIError,
|
||||
Timeout,
|
||||
APIConnectionError,
|
||||
UnsupportedParamsError,
|
||||
APIResponseValidationError,
|
||||
UnprocessableEntityError,
|
||||
InternalServerError,
|
||||
JSONSchemaValidationError,
|
||||
LITELLM_EXCEPTION_TYPES,
|
||||
MockException,
|
||||
)
|
||||
from .budget_manager import BudgetManager
|
||||
from .proxy.proxy_cli import run_server
|
||||
from .router import Router
|
||||
from .assistants.main import *
|
||||
from .batches.main import *
|
||||
from .images.main import *
|
||||
from .videos.main import *
|
||||
from .batch_completion.main import *
|
||||
from .rerank_api.main import *
|
||||
from .llms.anthropic.experimental_pass_through.messages.handler import *
|
||||
from .responses.main import *
|
||||
|
||||
### CUSTOM LLMs ###
|
||||
|
||||
### CLI UTILITIES ###
|
||||
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
|
||||
|
||||
### PASSTHROUGH ###
|
||||
from .passthrough import allm_passthrough_route, llm_passthrough_route
|
||||
from .google_genai import agenerate_content
|
||||
# Interactions API is available as litellm.interactions module
|
||||
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
|
||||
from . import interactions
|
||||
from .interactions.agents.main import (
|
||||
acreate as acreate_agent,
|
||||
create as create_agent,
|
||||
alist as alist_agents,
|
||||
list as list_agents,
|
||||
aget as aget_agent,
|
||||
get as get_agent,
|
||||
adelete as adelete_agent,
|
||||
delete as delete_agent,
|
||||
alist_versions as alist_agent_versions,
|
||||
list_versions as list_agent_versions,
|
||||
)
|
||||
from .skills.main import (
|
||||
create_skill,
|
||||
acreate_skill,
|
||||
list_skills,
|
||||
alist_skills,
|
||||
get_skill,
|
||||
aget_skill,
|
||||
delete_skill,
|
||||
adelete_skill,
|
||||
)
|
||||
from .containers.main import *
|
||||
from .ocr.main import *
|
||||
from .rust_bridge import rust
|
||||
from .rag.main import *
|
||||
from .sandbox.main import *
|
||||
from .search.main import *
|
||||
from .realtime_api.main import (
|
||||
_arealtime,
|
||||
acreate_realtime_client_secret,
|
||||
acreate_realtime_transcription_session,
|
||||
arealtime_calls,
|
||||
)
|
||||
from .responses.main import _aresponses_websocket
|
||||
from .fine_tuning.main import *
|
||||
from .files.main import *
|
||||
from .vector_store_files.main import (
|
||||
acreate as avector_store_file_create,
|
||||
adelete as avector_store_file_delete,
|
||||
alist as avector_store_file_list,
|
||||
aretrieve as avector_store_file_retrieve,
|
||||
aretrieve_content as avector_store_file_content,
|
||||
aupdate as avector_store_file_update,
|
||||
create as vector_store_file_create,
|
||||
delete as vector_store_file_delete,
|
||||
list as vector_store_file_list,
|
||||
retrieve as vector_store_file_retrieve,
|
||||
retrieve_content as vector_store_file_content,
|
||||
update as vector_store_file_update,
|
||||
)
|
||||
from .scheduler import *
|
||||
|
||||
### ADAPTERS ###
|
||||
from .types.adapter import AdapterItem
|
||||
import litellm.anthropic_interface as anthropic
|
||||
|
||||
adapters: List[AdapterItem] = []
|
||||
|
||||
### Vector Store Registry ###
|
||||
from .vector_stores.vector_store_registry import (
|
||||
VectorStoreRegistry,
|
||||
VectorStoreIndexRegistry,
|
||||
)
|
||||
|
||||
vector_store_registry: Optional[VectorStoreRegistry] = None
|
||||
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
|
||||
|
||||
### RAG ###
|
||||
from . import rag
|
||||
|
||||
### CUSTOM LLMs ###
|
||||
from .types.llms.custom_llm import CustomLLMItem
|
||||
|
||||
custom_provider_map: List[CustomLLMItem] = []
|
||||
_custom_providers: List[str] = [] # internal helper util, used to track names of custom providers
|
||||
disable_hf_tokenizer_download: Optional[bool] = (
|
||||
|
|
@ -1471,6 +1478,13 @@ disable_hf_tokenizer_download: Optional[bool] = (
|
|||
)
|
||||
global_disable_no_log_param: bool = False
|
||||
|
||||
### CLI UTILITIES ###
|
||||
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
|
||||
|
||||
### PASSTHROUGH ###
|
||||
from .passthrough import allm_passthrough_route, llm_passthrough_route
|
||||
from .google_genai import agenerate_content
|
||||
|
||||
### GLOBAL CONFIG ###
|
||||
global_bitbucket_config: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
|
@ -1494,21 +1508,10 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
|
|||
# Lazy loading system for heavy modules to reduce initial import time and memory usage
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import httpx
|
||||
|
||||
from litellm.types.utils import ModelInfo as _ModelInfoType
|
||||
from litellm.types.utils import PriorityReservationSettings
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.caching.caching import Cache
|
||||
from litellm.types.adapter import AdapterItem
|
||||
from litellm.types.integrations.datadog import DatadogInitParams
|
||||
from litellm.types.integrations.newrelic import NewRelicInitParams
|
||||
from litellm.types.llms.custom_llm import CustomLLMItem
|
||||
from litellm.types.prompts.init_prompts import PromptSpec
|
||||
from litellm.vector_stores.vector_store_registry import (
|
||||
VectorStoreIndexRegistry,
|
||||
VectorStoreRegistry,
|
||||
)
|
||||
|
||||
# Type stubs for lazy-loaded configs to help mypy
|
||||
from .llms.bedrock.chat.converse_transformation import (
|
||||
|
|
@ -2184,6 +2187,16 @@ if TYPE_CHECKING:
|
|||
# Track if async client cleanup has been registered (for lazy loading)
|
||||
_async_client_cleanup_registered = False
|
||||
|
||||
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
|
||||
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
|
||||
# For now, this only affects encoding (tiktoken) as it was the only reported issue
|
||||
# See: https://github.com/BerriAI/litellm/issues/18659
|
||||
# This ensures encoding is initialized before VCR starts recording HTTP requests
|
||||
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
|
||||
# Load encoding at import time (pre-#18070 behavior)
|
||||
# This ensures encoding is initialized before VCR starts recording
|
||||
from .main import encoding
|
||||
|
||||
|
||||
def __getattr__(name: str) -> Any:
|
||||
"""Lazy import handler with cached registry for improved performance."""
|
||||
|
|
@ -2263,8 +2276,6 @@ def __getattr__(name: str) -> Any:
|
|||
"openAIGPT5Config": "OpenAIGPT5Config",
|
||||
"nvidiaNimConfig": "NvidiaNimConfig",
|
||||
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
|
||||
"vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig",
|
||||
"_key_management_settings": "KeyManagementSettings",
|
||||
}
|
||||
if name in _config_instances:
|
||||
from ._lazy_imports import get_litellm_globals
|
||||
|
|
@ -2382,30 +2393,7 @@ def __getattr__(name: str) -> Any:
|
|||
|
||||
return locals()[name]
|
||||
|
||||
from ._lazy_imports import lazy_import_litellm_submodule
|
||||
|
||||
submodule: Final = lazy_import_litellm_submodule(name)
|
||||
if submodule is not None:
|
||||
return submodule
|
||||
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
from ._lazy_imports import LiteLLMModule
|
||||
from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES
|
||||
|
||||
sys.modules[__name__].__class__ = LiteLLMModule
|
||||
|
||||
__all__ = list(STAR_IMPORT_PUBLIC_NAMES) # mutable-ok: star imports require __all__ to be a list of str
|
||||
|
||||
|
||||
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
|
||||
|
||||
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
|
||||
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
|
||||
# For now, this only affects encoding (tiktoken) as it was the only reported issue
|
||||
# See: https://github.com/BerriAI/litellm/issues/18659
|
||||
# This ensures encoding is initialized before VCR starts recording HTTP requests
|
||||
# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import
|
||||
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
|
||||
from .main import encoding
|
||||
|
|
|
|||
|
|
@ -16,10 +16,9 @@ until they're actually needed.
|
|||
"""
|
||||
|
||||
import importlib
|
||||
import importlib.util
|
||||
import sys
|
||||
from collections.abc import Callable, Mapping
|
||||
from types import MappingProxyType, ModuleType
|
||||
from types import ModuleType
|
||||
from typing import TYPE_CHECKING, Any, Final, cast
|
||||
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
|
@ -35,8 +34,6 @@ from ._lazy_imports_registry import (
|
|||
_LITELLM_LOGGING_IMPORT_MAP,
|
||||
_LLM_CONFIGS_IMPORT_MAP,
|
||||
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
|
||||
_SDK_MODULE_ALIASES,
|
||||
_SDK_SYMBOLS_IMPORT_MAP,
|
||||
_TOKEN_COUNTER_IMPORT_MAP,
|
||||
_TYPES_IMPORT_MAP,
|
||||
_TYPES_UTILS_IMPORT_MAP,
|
||||
|
|
@ -81,10 +78,7 @@ def _get_utils_globals() -> dict[str, object]:
|
|||
This is where we cache imported attributes so we don't import them twice.
|
||||
When you do `litellm.utils.some_function`, it gets stored in this dictionary.
|
||||
"""
|
||||
cached: Final = sys.modules.get("litellm.utils")
|
||||
if cached is not None:
|
||||
return cached.__dict__
|
||||
return importlib.import_module("litellm.utils").__dict__
|
||||
return sys.modules["litellm.utils"].__dict__
|
||||
|
||||
|
||||
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
|
||||
|
|
@ -220,10 +214,6 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
|
|||
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
|
||||
for name in UTILS_MODULE_NAMES:
|
||||
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
|
||||
for name in _SDK_SYMBOLS_IMPORT_MAP:
|
||||
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols)
|
||||
for name in _SDK_MODULE_ALIASES:
|
||||
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias)
|
||||
|
||||
return _LAZY_IMPORT_REGISTRY
|
||||
|
||||
|
|
@ -360,86 +350,6 @@ def _lazy_import_llm_provider_logic(name: str) -> object:
|
|||
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
|
||||
|
||||
|
||||
def _lazy_import_sdk_symbols(name: str) -> object:
|
||||
"""Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py"""
|
||||
return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols")
|
||||
|
||||
|
||||
def _lazy_import_sdk_module_alias(name: str) -> object:
|
||||
"""Handler for litellm attributes that bind a module (e.g. litellm.anthropic)"""
|
||||
_globals: Final = get_litellm_globals()
|
||||
if name in _globals:
|
||||
return _globals[name]
|
||||
module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name])
|
||||
_globals[name] = module # rebind-ok: caches the resolved module alias on the package
|
||||
return module
|
||||
|
||||
|
||||
_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType(
|
||||
{
|
||||
"batch_completion": ("litellm.batch_completion.main", "batch_completion"),
|
||||
"ocr": ("litellm.ocr.main", "ocr"),
|
||||
"responses": ("litellm.responses.main", "responses"),
|
||||
"search": ("litellm.search.main", "search"),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _shadowable_function_property(name: str) -> property:
|
||||
"""Property keeping litellm.<name> bound to the SDK function even after the import
|
||||
machinery binds the identically named litellm.<name> subpackage onto the litellm module."""
|
||||
module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name]
|
||||
|
||||
def _get(module: ModuleType) -> object:
|
||||
stored: Final = module.__dict__.get(name)
|
||||
if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"):
|
||||
return stored
|
||||
value: Final = _module_attribute(importlib.import_module(module_path), attr_name)
|
||||
module.__dict__[name] = value # rebind-ok: caches the resolved function on the litellm module
|
||||
return value
|
||||
|
||||
def _set(module: ModuleType, value: object) -> None:
|
||||
module.__dict__[name] = value # rebind-ok: property setter must store assignments on the module
|
||||
|
||||
return property(_get, _set)
|
||||
|
||||
|
||||
class LiteLLMModule(ModuleType):
|
||||
"""Module type installed on the litellm package so function names shadowed by
|
||||
same-named subpackages (litellm.responses, ...) keep resolving to the functions."""
|
||||
|
||||
batch_completion = _shadowable_function_property("batch_completion")
|
||||
ocr = _shadowable_function_property("ocr")
|
||||
responses = _shadowable_function_property("responses")
|
||||
search = _shadowable_function_property("search")
|
||||
|
||||
|
||||
def lazy_import_submodule(package: str, name: str) -> "ModuleType | None":
|
||||
"""Resolve <package>.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
|
||||
if name.startswith("__") or not name.isidentifier():
|
||||
return None
|
||||
qualified_name: Final = f"{package}.{name}"
|
||||
try:
|
||||
spec: Final = importlib.util.find_spec(qualified_name)
|
||||
except ModuleNotFoundError:
|
||||
return None
|
||||
if spec is None:
|
||||
return None
|
||||
try:
|
||||
module: Final = importlib.import_module(qualified_name)
|
||||
except ModuleNotFoundError as exc:
|
||||
if exc.name == qualified_name:
|
||||
return None
|
||||
raise
|
||||
sys.modules[package].__dict__[name] = module # rebind-ok: caches the resolved submodule on the package
|
||||
return module
|
||||
|
||||
|
||||
def lazy_import_litellm_submodule(name: str) -> "ModuleType | None":
|
||||
"""Resolve litellm.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
|
||||
return lazy_import_submodule("litellm", name)
|
||||
|
||||
|
||||
def _lazy_import_utils_module(name: str) -> object:
|
||||
"""
|
||||
Handler for utils module lazy imports.
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -6,14 +6,13 @@ import base64
|
|||
from collections.abc import Awaitable, Callable
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS
|
||||
from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig, DocumentType
|
||||
from litellm.types.utils import LIST_BATCHES_SUPPORTED_PROVIDERS, LlmProviders
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.types.utils import ImageResponse
|
||||
|
||||
# Minimal PDF for health checks - base64 encoded 1-page PDF with just "test"
|
||||
TEST_PDF_URL = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
|
||||
|
||||
# Minimal image for health checks - base64 encoded 512x512 blue circle on a white background PNG
|
||||
TEST_IMAGE_BASE64 = "iVBORw0KGgoAAAANSUhEUgAAAgAAAAIACAIAAAB7GkOtAAAJk0lEQVR42u3VQREAIRADwVWCOmTjBVzwSLorCri6nbkAVBpPACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAAAgAAAIAZdY+HgEBgIRr/meeGgGA8EMvDAgAuPh6gACAi68HCAA4+mKAAICjLwYIALj7SoAAgLuvBAgA7r4pAQKAu29KgADg7psSIAA4/SYDCADuvikBAoDTbzKAAOD0mwwgADj9JgMIAE6/yQACgNNvMoAA4PSbDCAAOP0mAwgATr/JAAKA668BIAA4/TKAAOD0mwwgALj+pgEIAE6/yQACgOtvGoAA4PSbDCAAuP6mAQgATr/JAAKA628agADg9JsMIAC4/qYBCACuv2kAAoDrbxqAAOD0mwwgALj+pgEIAK6/aQACgOtvGoAA4PqbBiAArr+ZBiAATr+ZDCAArr+ZBiAArr+ZBiAArr+ZBiAArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggArr+ZBggAAmAmAAKA62+mAQKA62+mAQKA62/m1xYAXH/TAAQA1980AAHA9TcNQAAEwEwAEADX30wDEADX30wDEADX30wDEADX30wDEAABMBMABMD1N9MABMD1N9MABEAAzAQAAXD9zTQAAXD9zTRAABAAMwEQAFx/Mw0QAFx/Mw0QAATATAAEANffTAMEANffTAMEAAEwEwABwPU30wABQADMBEAAXH8z0wABcP3NTAMEQADMTAAEwPU3Mw0QAAEwMwEQANffTAMQAAEwEwAEwPU30wAEQADMBAABcP3NNAABEAAzAUAAXH8zDUAABMBMABAA199MAwQAATATAAHA9TfTAAFAAMwEQABw/c00QAAQADMBEAAEwEwABMD1NzMNEAABMDMBEADX38w0QAAEwMwEQAAEwMwEQABcfzPTAAEQADMTAAEQADMTAAFw/c1MAwRAAMxMAARAAMxMAATA9TczDRAAATATAARAAMwEAAFw/c00AAEQADMBQAAEwEwAEADX30wDBAABMBMAAUAAzARAABAAMwEQAFx/Mw0QAAEwMwEQAAEwMwEQAAEwMwEQANffzDRAAATAzARAAATAzARAAATAzARAAATAzARAAFx/M9MAARAAMxMAARAAMxMAARAAMxMAARAAMxMAAXD9zUwDBEAAzEwABEAAzAQAARAAMwFAAATATAAQAAEwEwABQADMBEAAEAAzARAAXH8zDRAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAAATAzARAA19/MNEAANMDM9UcABMBMABAAATATAAHwBAJgJgACgACYCYAAIABmAiAA+IvMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAABMDMBEAANMDMXH8BEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzEwABEAAzAUAABMBMABAADTBz/REAATATAAFAAMwEQAAQADMBEAAEwEwABAANMHP9BUAAzEwABEAAzEwABEAAzEwABEAAzEwABEADzMz1FwABMDMBEAABMDMBEAABMDMBEAANMDPXXwAEwMwEQAAEwMwEQAAEwMwEQAA0wMxcfwEQADMTAAEQADMBQAA0wMz1RwAEwEwAEAABMBMABEADzFx/AUAAzARAABAAMwEQADTAzPUXAATATAAEAAEwEwABQAPMXH8BEAAzEwABEAAzEwAB0AAzc/0FQADMTAAEQAPMzPUXAAEwMwEQAAEwMwEQAA0wM9dfAATAzARAADTAzFx/ARAAMwFAADTAzPVHAATATAAQAA0wc/0RAAEwEwAEQAPMXH8EQADMBAAB0AAz118AEAAzARAANMDM9RcABMBMAAQADTBz/QUAATATAAFAA8xcfwFAA8xcfwFAAMwEQADQADPXXwAEwMwEQAA0wMxcfwHQADNz/QVAAMxMAARAA8xcfwRAA8xcfwRAAMwEAAHQADPXHwHQADPXHwEQADMBQAA0wMz1RwA0wMz1RwAEwEwAEAANMHP9BQANMHP9BQANMHP9BQANMHP9BQABMBMAAUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzFx/AUADzPVHABAAEwAEAA0w1x8BQAPM9UcA0ABz/REANMBcfwQADTDXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwHQADPXHwGQATOnHwHQADPXHwHQADPXXwDQADPXXwDQADPXXwDQADPXXwCQAXP6EQA0wFx/BAANMNcfAUADzPVHAJABc/oRADTAXH8EABkwpx8BQAPM9UcAkAFz+hEANMBcfwQAGTCnHwFAA8z1RwCQAXP6EQBkwJx+BAANMNcfAUAGzOlHAJABc/oRAGTA6QcBQAacfhAAZMDpRwBABpx+BABkwOlHAEAGnH4EAJTA3UcAQAacfgQAlMDdRwBACdx9BACUwN1HAEAJ3H0EAJTA3UcAQAwcfQQAmmLgsyIA0NIDHw4BgJYe+DQIAISHwVMjAJDQDI+AAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACACAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAgAAAIAAACAIAAACAAAAgAAAIAgAAAIAAACAAAAgCAAAAIAABNHpialFcmLajuAAAAAElFTkSuQmCC"
|
||||
|
|
@ -29,6 +28,14 @@ def get_image_file_for_health_check() -> bytes:
|
|||
return base64.b64decode(TEST_IMAGE_BASE64)
|
||||
|
||||
|
||||
def _ocr_health_check_document(model: str, custom_llm_provider: str) -> DocumentType:
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
provider: Final = next((known for known in LlmProviders if known.value == custom_llm_provider), None)
|
||||
config: Final = ProviderConfigManager.get_provider_ocr_config(model=model, provider=provider) if provider else None
|
||||
return (config or BaseOCRConfig()).get_health_check_document()
|
||||
|
||||
|
||||
class HealthCheckHelpers:
|
||||
@staticmethod
|
||||
async def ahealth_check_wildcard_models(
|
||||
|
|
@ -247,9 +254,6 @@ class HealthCheckHelpers:
|
|||
),
|
||||
"ocr": lambda: litellm.aocr(
|
||||
**_filter_model_params(model_params=model_params),
|
||||
document={
|
||||
"type": "document_url",
|
||||
"document_url": TEST_PDF_URL,
|
||||
},
|
||||
document=_ocr_health_check_document(model=model, custom_llm_provider=custom_llm_provider),
|
||||
),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5675,8 +5675,8 @@ class StandardLoggingPayloadSetup:
|
|||
error_code=error_status,
|
||||
error_class=error_class,
|
||||
llm_provider=_llm_provider_in_exception,
|
||||
traceback=traceback_info,
|
||||
error_message=error_message,
|
||||
traceback=_redact_string(traceback_info),
|
||||
error_message=_redact_string(error_message),
|
||||
error_rate_limit_category=rate_limit_category,
|
||||
error_rate_limit_type=rate_limit_type,
|
||||
error_budget_entity_type=budget_error.entity_type if budget_error else None,
|
||||
|
|
@ -5881,6 +5881,7 @@ def _get_status_fields(
|
|||
# Mapping for legacy guardrail status values to new GuardrailStatus values
|
||||
GUARDRAIL_STATUS_MAP: Final[dict[str, GuardrailStatus]] = {
|
||||
"success": "success",
|
||||
"guardrail_flagged": "guardrail_flagged",
|
||||
"blocked": "guardrail_intervened", # legacy
|
||||
"guardrail_intervened": "guardrail_intervened", # direct
|
||||
"failure": "guardrail_failed_to_respond", # legacy
|
||||
|
|
@ -5902,6 +5903,7 @@ def _get_status_fields(
|
|||
GUARDRAIL_STATUS_SEVERITY: Final[tuple[GuardrailStatus, ...]] = (
|
||||
"not_run",
|
||||
"success",
|
||||
"guardrail_flagged",
|
||||
"guardrail_failed_to_respond",
|
||||
"guardrail_intervened",
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
"""Azure AI OCR module."""
|
||||
|
||||
from .cohere_parse_transformation import AzureAICohereParseConfig
|
||||
from .common_utils import get_azure_ai_ocr_config
|
||||
from .document_intelligence.transformation import (
|
||||
AzureDocumentIntelligenceOCRConfig,
|
||||
|
|
@ -7,6 +8,7 @@ from .document_intelligence.transformation import (
|
|||
from .transformation import AzureAIOCRConfig
|
||||
|
||||
__all__ = [
|
||||
"AzureAICohereParseConfig",
|
||||
"AzureAIOCRConfig",
|
||||
"AzureDocumentIntelligenceOCRConfig",
|
||||
"get_azure_ai_ocr_config",
|
||||
|
|
|
|||
91
litellm/llms/azure_ai/ocr/cohere_parse_transformation.py
Normal file
91
litellm/llms/azure_ai/ocr/cohere_parse_transformation.py
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
"""Cohere Parse served from Azure AI Foundry (`/providers/cohere/v2/parse`)."""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm.litellm_core_utils.prompt_templates.image_handling import (
|
||||
async_convert_url_to_base64,
|
||||
convert_url_to_base64,
|
||||
)
|
||||
from litellm.llms.azure_ai.common_utils import get_azure_ai_auth_headers
|
||||
from litellm.llms.cohere.ocr.transformation import COHERE_PARSE_PATH, CohereParseConfig
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
AZURE_AI_API_KEY_ENV_VAR: Final = "AZURE_AI_API_KEY"
|
||||
AZURE_AI_API_BASE_ENV_VAR: Final = "AZURE_AI_API_BASE"
|
||||
AZURE_AI_COHERE_PROVIDER_PATH: Final = "/providers/cohere"
|
||||
AZURE_AI_MODELS_PATH_SUFFIX: Final = "/models"
|
||||
|
||||
|
||||
class AzureAICohereParseConfig(CohereParseConfig):
|
||||
"""Same request and response shape as Cohere Parse, behind Azure AI auth and URL layout.
|
||||
|
||||
Foundry cannot fetch external URLs, so remote images are inlined as base64 data URIs.
|
||||
"""
|
||||
|
||||
def get_api_key_env_var(self) -> str | None:
|
||||
return AZURE_AI_API_KEY_ENV_VAR
|
||||
|
||||
def _llm_provider(self) -> str:
|
||||
return "azure_ai"
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.validate_environment signature
|
||||
) -> dict[str, str]: # mutable-ok: BaseOCRConfig signature
|
||||
resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
|
||||
if resolved_base is None:
|
||||
raise ValueError(
|
||||
f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
|
||||
"or pass api_base parameter"
|
||||
)
|
||||
resolved_key: Final = api_key or get_secret_str(AZURE_AI_API_KEY_ENV_VAR)
|
||||
return { # mutable-ok: BaseOCRConfig signature
|
||||
**get_azure_ai_auth_headers(api_key=resolved_key, litellm_params=litellm_params),
|
||||
"Content-Type": "application/json",
|
||||
**headers,
|
||||
}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.get_complete_url signature
|
||||
) -> str:
|
||||
resolved_base: Final = api_base or get_secret_str(AZURE_AI_API_BASE_ENV_VAR)
|
||||
if resolved_base is None:
|
||||
raise ValueError(
|
||||
f"Missing Azure AI API Base - Set {AZURE_AI_API_BASE_ENV_VAR} environment variable "
|
||||
"or pass api_base parameter"
|
||||
)
|
||||
url: Final = httpx.URL(resolved_base)
|
||||
if not url.is_absolute_url:
|
||||
raise ValueError(
|
||||
"Azure AI API Base must be an absolute URL including scheme (e.g. "
|
||||
f"'https://<resource>.services.ai.azure.com'). Got api_base={resolved_base!r}."
|
||||
)
|
||||
path: Final = url.path.rstrip("/")
|
||||
if path.endswith(COHERE_PARSE_PATH):
|
||||
return str(url.copy_with(path=path))
|
||||
if path.endswith(f"{AZURE_AI_COHERE_PROVIDER_PATH}/v2"):
|
||||
return str(url.copy_with(path=f"{path}/parse"))
|
||||
return str(
|
||||
url.copy_with(
|
||||
path=f"{path.removesuffix(AZURE_AI_MODELS_PATH_SUFFIX)}{AZURE_AI_COHERE_PROVIDER_PATH}{COHERE_PARSE_PATH}"
|
||||
)
|
||||
)
|
||||
|
||||
def _resolve_image_url_sync(self, image_url: str) -> str:
|
||||
return convert_url_to_base64(image_url)
|
||||
|
||||
async def _resolve_image_url_async(self, image_url: str) -> str:
|
||||
return await async_convert_url_to_base64(image_url)
|
||||
|
|
@ -24,6 +24,11 @@ def is_azure_document_intelligence_model(model: str) -> bool:
|
|||
return "doc-intelligence" in lowered or "documentintelligence" in lowered
|
||||
|
||||
|
||||
def is_azure_cohere_parse_model(model: str) -> bool:
|
||||
lowered: Final = model.lower()
|
||||
return "cohere" in lowered and "parse" in lowered
|
||||
|
||||
|
||||
def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
|
||||
"""
|
||||
Determine which Azure AI OCR configuration to use based on the model name.
|
||||
|
|
@ -46,6 +51,7 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
|
|||
>>> get_azure_ai_ocr_config("azure_ai/pixtral-12b-2409")
|
||||
<AzureAIOCRConfig object>
|
||||
"""
|
||||
from litellm.llms.azure_ai.ocr.cohere_parse_transformation import AzureAICohereParseConfig
|
||||
from litellm.llms.azure_ai.ocr.document_intelligence.transformation import (
|
||||
AzureDocumentIntelligenceOCRConfig,
|
||||
)
|
||||
|
|
@ -56,6 +62,10 @@ def get_azure_ai_ocr_config(model: str) -> Optional["BaseOCRConfig"]:
|
|||
verbose_logger.debug("Routing %s to Azure Document Intelligence OCR config", model)
|
||||
return AzureDocumentIntelligenceOCRConfig()
|
||||
|
||||
if is_azure_cohere_parse_model(model):
|
||||
verbose_logger.debug("Routing %s to Azure AI Cohere Parse config", model)
|
||||
return AzureAICohereParseConfig()
|
||||
|
||||
# Default to Mistral-based OCR for other azure_ai models
|
||||
verbose_logger.debug("Routing %s to Azure AI (Mistral) OCR config", model)
|
||||
return AzureAIOCRConfig()
|
||||
|
|
|
|||
|
|
@ -33,6 +33,8 @@ OCR_REQUEST_FORMAT_HEADER: Final = "x-req-format"
|
|||
|
||||
PROVIDER_NATIVE_RESPONSE_KEY: Final = "provider_native_response"
|
||||
|
||||
HEALTH_CHECK_PDF_DATA_URI: Final = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMyAwIG9iago8PC9UeXBlIC9QYWdlCi9QYXJlbnQgMSAwIFIKL01lZGlhQm94IFswIDAgNjEyIDc5Ml0KL0NvbnRlbnRzIDQgMCBSCi9SZXNvdXJjZXMgPDwvRm9udCA8PC9GMSAyIDAgUj4+Pj4+PgplbmRvYmoKNCAwIG9iago8PC9MZW5ndGggNDQ+PgpzdHJlYW0KQlQKL0YxIDI0IFRmCjEwMCA3MDAgVGQKKHRlc3QpIFRqCkVUCmVuZHN0cmVhbQplbmRvYmoKMiAwIG9iago8PC9UeXBlIC9Gb250Ci9TdWJ0eXBlIC9UeXBlMQovQmFzZUZvbnQgL0hlbHZldGljYT4+CmVuZG9iagoxIDAgb2JqCjw8L1R5cGUgL1BhZ2VzCi9LaWRzIFszIDAgUl0KL0NvdW50IDE+PgplbmRvYmoKNSAwIG9iago8PC9UeXBlIC9DYXRhbG9nCi9QYWdlcyAxIDAgUj4+CmVuZG9iagp0cmFpbGVyCjw8L1NpemUgNgovUm9vdCA1IDAgUj4+CnN0YXJ0eHJlZgozMjQKJSVFT0Y="
|
||||
|
||||
|
||||
def parse_ocr_request_format(value: object) -> OCRRequestFormat:
|
||||
if value == "litellm":
|
||||
|
|
@ -142,6 +144,16 @@ class BaseOCRConfig:
|
|||
"""
|
||||
return None
|
||||
|
||||
def supports_rust_bridge(self) -> bool:
|
||||
"""Whether the Rust OCR bridge may serve this config when it is enabled for the provider."""
|
||||
return True
|
||||
|
||||
def get_health_check_document(self) -> DocumentType:
|
||||
return { # mutable-ok: litellm.aocr rejects any document that is not a dict
|
||||
"type": "document_url",
|
||||
"document_url": HEALTH_CHECK_PDF_DATA_URI,
|
||||
}
|
||||
|
||||
def map_ocr_params(
|
||||
self,
|
||||
non_default_params: dict,
|
||||
|
|
|
|||
3
litellm/llms/cohere/ocr/__init__.py
Normal file
3
litellm/llms/cohere/ocr/__init__.py
Normal file
|
|
@ -0,0 +1,3 @@
|
|||
from litellm.llms.cohere.ocr.transformation import CohereParseConfig
|
||||
|
||||
__all__ = ("CohereParseConfig",)
|
||||
301
litellm/llms/cohere/ocr/transformation.py
Normal file
301
litellm/llms/cohere/ocr/transformation.py
Normal file
|
|
@ -0,0 +1,301 @@
|
|||
"""Cohere Parse (`POST /v2/parse`) exposed through LiteLLM's OCR interface."""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.exceptions import BadRequestError, UnsupportedParamsError
|
||||
from litellm.llms.base_llm.ocr.transformation import (
|
||||
OCR_REQUEST_FORMAT_PARAM,
|
||||
BaseOCRConfig,
|
||||
DocumentType,
|
||||
OCRPage,
|
||||
OCRPageImage,
|
||||
OCRRequestData,
|
||||
OCRRequestFormat,
|
||||
OCRResponse,
|
||||
OCRUsageInfo,
|
||||
parse_ocr_request_format,
|
||||
)
|
||||
from litellm.llms.cohere.common_utils import CohereError
|
||||
from litellm.secret_managers.main import get_secret_str
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
COHERE_API_KEY_ENV_VAR: Final = "COHERE_API_KEY"
|
||||
COHERE_PARSE_API_BASE: Final = "https://api.cohere.com"
|
||||
COHERE_PARSE_PATH: Final = "/v2/parse"
|
||||
COHERE_PARSE_OUTPUT_FORMAT_PARAM: Final = "output_format"
|
||||
COHERE_PARSE_OUTPUT_FORMATS: Final = ("markdown", "blocks")
|
||||
COHERE_PARSE_DEFAULT_OUTPUT_FORMAT: Final = "markdown"
|
||||
COHERE_PARSE_SUPPORTED_PARAMS: Final = (COHERE_PARSE_OUTPUT_FORMAT_PARAM, OCR_REQUEST_FORMAT_PARAM)
|
||||
COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI: Final = (
|
||||
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVR4nGP4//8/AAX+Av4N70a4AAAAAElFTkSuQmCC"
|
||||
)
|
||||
COHERE_PARSE_IMAGE_ONLY_MESSAGE: Final = (
|
||||
"Cohere Parse only accepts `image_url` documents (an image URL or a base64 image data URI); "
|
||||
"`document_url` and PDF inputs are not supported."
|
||||
)
|
||||
|
||||
_NATIVE_RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_BOUNDING_BOX_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
class _CohereParseDocument(TypedDict):
|
||||
type: ReadOnly[Literal["image_url"]]
|
||||
image_url: ReadOnly[str]
|
||||
|
||||
|
||||
class _CohereParseRequestBody(TypedDict):
|
||||
model: ReadOnly[str]
|
||||
document: ReadOnly[_CohereParseDocument]
|
||||
output_format: ReadOnly[str]
|
||||
|
||||
|
||||
class _MarkdownPage(TypedDict):
|
||||
index: ReadOnly[int]
|
||||
markdown: ReadOnly[str]
|
||||
images: ReadOnly[Sequence[OCRPageImage] | None]
|
||||
|
||||
|
||||
class _BlocksPage(_MarkdownPage):
|
||||
blocks: ReadOnly[Sequence[Mapping[str, object]]]
|
||||
|
||||
|
||||
class _CohereParseMarkdown(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
content: str = ""
|
||||
images: Sequence[Mapping[str, object]] | None = None
|
||||
|
||||
|
||||
class _CohereParsePage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
index: int | None = None
|
||||
markdown: _CohereParseMarkdown | None = None
|
||||
blocks: Sequence[Mapping[str, object]] | None = None
|
||||
|
||||
|
||||
class _CohereParseBilledUnits(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
pages: int | None = None
|
||||
|
||||
|
||||
class _CohereParseMeta(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
billed_units: _CohereParseBilledUnits | None = None
|
||||
|
||||
|
||||
class _CohereParseResponse(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="allow")
|
||||
|
||||
pages: Sequence[_CohereParsePage] = ()
|
||||
meta: _CohereParseMeta | None = None
|
||||
|
||||
|
||||
def _requested_format(optional_params: Mapping[str, object] | None) -> OCRRequestFormat:
|
||||
if optional_params is None:
|
||||
return "litellm"
|
||||
return "native" if optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native" else "litellm"
|
||||
|
||||
|
||||
def _page_image(image: Mapping[str, object]) -> OCRPageImage:
|
||||
bounding_box: Final = image.get("bounding_box")
|
||||
if not isinstance(bounding_box, Mapping):
|
||||
return OCRPageImage.model_validate(image)
|
||||
bbox: Final = _BOUNDING_BOX_ADAPTER.validate_python(bounding_box)
|
||||
return OCRPageImage.model_validate(MappingProxyType({**image, "bbox": bbox}))
|
||||
|
||||
|
||||
def _normalize_page(page: _CohereParsePage, position: int) -> OCRPage:
|
||||
markdown: Final = page.markdown
|
||||
images: Final = tuple(_page_image(image) for image in markdown.images) if markdown and markdown.images else None
|
||||
normalized: Final[_MarkdownPage] = {
|
||||
"index": page.index if page.index is not None else position,
|
||||
"markdown": markdown.content if markdown else "",
|
||||
"images": images,
|
||||
}
|
||||
if page.blocks is None:
|
||||
return OCRPage.model_validate(normalized)
|
||||
with_blocks: Final[_BlocksPage] = {**normalized, "blocks": page.blocks}
|
||||
return OCRPage.model_validate(with_blocks)
|
||||
|
||||
|
||||
def _billed_pages(parsed: _CohereParseResponse) -> int | None:
|
||||
if parsed.meta is None or parsed.meta.billed_units is None:
|
||||
return None
|
||||
return parsed.meta.billed_units.pages
|
||||
|
||||
|
||||
class CohereParseConfig(BaseOCRConfig):
|
||||
"""Cohere Parse, an image-only document understanding endpoint returning markdown or blocks."""
|
||||
|
||||
def get_supported_ocr_params(self, model: str) -> list[str]: # mutable-ok: BaseOCRConfig signature
|
||||
return list(COHERE_PARSE_SUPPORTED_PARAMS) # mutable-ok: BaseOCRConfig signature
|
||||
|
||||
def get_api_key_env_var(self) -> str | None:
|
||||
return COHERE_API_KEY_ENV_VAR
|
||||
|
||||
def supports_rust_bridge(self) -> bool:
|
||||
return False
|
||||
|
||||
def get_health_check_document(self) -> DocumentType:
|
||||
return { # mutable-ok: litellm.aocr rejects any document that is not a dict
|
||||
"type": "image_url",
|
||||
"image_url": COHERE_PARSE_HEALTH_CHECK_IMAGE_DATA_URI,
|
||||
}
|
||||
|
||||
def _llm_provider(self) -> str:
|
||||
return "cohere"
|
||||
|
||||
def map_ocr_params(
|
||||
self,
|
||||
non_default_params: Mapping[str, object],
|
||||
optional_params: Mapping[str, object],
|
||||
model: str,
|
||||
) -> dict[str, object]: # mutable-ok: BaseOCRConfig signature
|
||||
output_format: Final = non_default_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM)
|
||||
if output_format is not None and output_format not in COHERE_PARSE_OUTPUT_FORMATS:
|
||||
raise UnsupportedParamsError(
|
||||
message=(
|
||||
f"Invalid `{COHERE_PARSE_OUTPUT_FORMAT_PARAM}`: {output_format!r}. "
|
||||
f"Expected one of {', '.join(COHERE_PARSE_OUTPUT_FORMATS)}."
|
||||
),
|
||||
model=model,
|
||||
llm_provider=self._llm_provider(),
|
||||
)
|
||||
requested_format: Final = non_default_params.get(OCR_REQUEST_FORMAT_PARAM)
|
||||
request_format: Final = parse_ocr_request_format(requested_format) if requested_format is not None else None
|
||||
overrides: Final = tuple(
|
||||
(key, value)
|
||||
for key, value in (
|
||||
(COHERE_PARSE_OUTPUT_FORMAT_PARAM, output_format),
|
||||
(OCR_REQUEST_FORMAT_PARAM, request_format),
|
||||
)
|
||||
if value is not None
|
||||
)
|
||||
return {**optional_params, **dict(overrides)} # mutable-ok: BaseOCRConfig signature
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
headers: Mapping[str, str],
|
||||
model: str,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.validate_environment signature
|
||||
) -> dict[str, str]: # mutable-ok: BaseOCRConfig signature
|
||||
resolved_key: Final = api_key or get_secret_str(COHERE_API_KEY_ENV_VAR)
|
||||
if resolved_key is None:
|
||||
raise ValueError(
|
||||
f"Missing {COHERE_API_KEY_ENV_VAR} - set it in the environment or pass api_key to "
|
||||
"litellm.ocr()/litellm.aocr()"
|
||||
)
|
||||
return { # mutable-ok: BaseOCRConfig signature
|
||||
"Authorization": f"Bearer {resolved_key}",
|
||||
"Content-Type": "application/json",
|
||||
**headers,
|
||||
}
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
api_base: str | None,
|
||||
model: str,
|
||||
optional_params: Mapping[str, object],
|
||||
litellm_params: Mapping[str, object] | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.get_complete_url signature
|
||||
) -> str:
|
||||
url: Final = httpx.URL(api_base or COHERE_PARSE_API_BASE)
|
||||
path: Final = url.path.rstrip("/")
|
||||
if path.endswith(COHERE_PARSE_PATH):
|
||||
return str(url.copy_with(path=path))
|
||||
if path.endswith("/v2"):
|
||||
return str(url.copy_with(path=f"{path}/parse"))
|
||||
return str(url.copy_with(path=f"{path}{COHERE_PARSE_PATH}"))
|
||||
|
||||
def _image_url(self, document: DocumentType, model: str) -> str:
|
||||
image_url: Final = document.get("image_url", "")
|
||||
if document.get("type") != "image_url" or not image_url or image_url.startswith("data:application/pdf"):
|
||||
raise BadRequestError(
|
||||
message=COHERE_PARSE_IMAGE_ONLY_MESSAGE,
|
||||
model=model,
|
||||
llm_provider=self._llm_provider(),
|
||||
)
|
||||
return image_url
|
||||
|
||||
def _resolve_image_url_sync(self, image_url: str) -> str:
|
||||
return image_url
|
||||
|
||||
async def _resolve_image_url_async(self, image_url: str) -> str:
|
||||
return image_url
|
||||
|
||||
def _build_request(self, model: str, image_url: str, optional_params: Mapping[str, object]) -> OCRRequestData:
|
||||
body: Final[_CohereParseRequestBody] = {
|
||||
"model": model,
|
||||
"document": {"type": "image_url", "image_url": image_url},
|
||||
"output_format": str(
|
||||
optional_params.get(COHERE_PARSE_OUTPUT_FORMAT_PARAM, COHERE_PARSE_DEFAULT_OUTPUT_FORMAT)
|
||||
),
|
||||
}
|
||||
return OCRRequestData(data=dict(body), files=None) # mutable-ok: OCRRequestData.data is a dict
|
||||
|
||||
def transform_ocr_request(
|
||||
self,
|
||||
model: str,
|
||||
document: DocumentType,
|
||||
optional_params: Mapping[str, object],
|
||||
headers: Mapping[str, str],
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.transform_ocr_request signature
|
||||
) -> OCRRequestData:
|
||||
image_url: Final = self._resolve_image_url_sync(self._image_url(document, model))
|
||||
return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
|
||||
|
||||
async def async_transform_ocr_request(
|
||||
self,
|
||||
model: str,
|
||||
document: DocumentType,
|
||||
optional_params: Mapping[str, object],
|
||||
headers: Mapping[str, str],
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.async_transform_ocr_request signature
|
||||
) -> OCRRequestData:
|
||||
image_url: Final = await self._resolve_image_url_async(self._image_url(document, model))
|
||||
return self._build_request(model=model, image_url=image_url, optional_params=optional_params)
|
||||
|
||||
def transform_ocr_response(
|
||||
self,
|
||||
model: str,
|
||||
raw_response: httpx.Response,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
optional_params: Mapping[str, object] | None = None,
|
||||
**kwargs: object, # kwargs-ok: BaseOCRConfig.transform_ocr_response signature
|
||||
) -> OCRResponse:
|
||||
native: Final = _NATIVE_RESPONSE_ADAPTER.validate_python(raw_response.json())
|
||||
parsed: Final = _CohereParseResponse.model_validate(native)
|
||||
pages: Final = [ # mutable-ok: OCRResponse.pages is a list
|
||||
_normalize_page(page, position) for position, page in enumerate(parsed.pages)
|
||||
]
|
||||
billed_pages: Final = _billed_pages(parsed)
|
||||
response: Final = OCRResponse(
|
||||
pages=pages,
|
||||
model=model,
|
||||
usage_info=OCRUsageInfo(pages_processed=billed_pages if billed_pages is not None else len(pages)),
|
||||
)
|
||||
if _requested_format(optional_params) == "native":
|
||||
response.set_provider_native_response(native)
|
||||
return response
|
||||
|
||||
def get_error_class(
|
||||
self,
|
||||
error_message: str,
|
||||
status_code: int,
|
||||
headers: Mapping[str, str],
|
||||
) -> Exception:
|
||||
return CohereError(status_code=status_code, message=error_message)
|
||||
|
|
@ -10243,6 +10243,16 @@
|
|||
],
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/mistral/"
|
||||
},
|
||||
"azure_ai/Cohere-parse-v5": {
|
||||
"deprecation_date": "2026-12-15",
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "ocr",
|
||||
"ocr_cost_per_page": 0.0015,
|
||||
"source": "https://cohere.com/blog/parse",
|
||||
"supported_endpoints": [
|
||||
"/v1/ocr"
|
||||
]
|
||||
},
|
||||
"azure_ai/doc-intelligence/prebuilt-read": {
|
||||
"litellm_provider": "azure_ai",
|
||||
"ocr_cost_per_page": 0.0015,
|
||||
|
|
@ -14116,6 +14126,15 @@
|
|||
"output_vector_size": 1536,
|
||||
"supports_embedding_image_input": true
|
||||
},
|
||||
"cohere/parse-v5.0": {
|
||||
"litellm_provider": "cohere",
|
||||
"mode": "ocr",
|
||||
"ocr_cost_per_page": 0.0015,
|
||||
"source": "https://cohere.com/blog/parse",
|
||||
"supported_endpoints": [
|
||||
"/v1/ocr"
|
||||
]
|
||||
},
|
||||
"cohere.rerank-v3-5:0": {
|
||||
"input_cost_per_query": 0.002,
|
||||
"input_cost_per_token": 0.0,
|
||||
|
|
|
|||
|
|
@ -191,6 +191,8 @@ def _prepare_ocr_request(
|
|||
def _rust_ocr_supported(prepared_request: _PreparedOCRRequest) -> bool:
|
||||
if prepared_request.optional_params.get(OCR_REQUEST_FORMAT_PARAM) == "native":
|
||||
return False
|
||||
if not prepared_request.provider_config.supports_rust_bridge():
|
||||
return False
|
||||
return prepared_request.custom_llm_provider in _RUST_OCR_PROVIDERS
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -559,6 +559,7 @@
|
|||
"moderations": false,
|
||||
"batches": false,
|
||||
"rerank": true,
|
||||
"ocr": true,
|
||||
"a2a": true,
|
||||
"interactions": true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1 @@
|
|||
from types import ModuleType
|
||||
from typing import Final
|
||||
|
||||
|
||||
def __getattr__(name: str) -> ModuleType:
|
||||
from litellm._lazy_imports import lazy_import_submodule
|
||||
|
||||
submodule: Final = lazy_import_submodule(__name__, name)
|
||||
if submodule is not None:
|
||||
return submodule
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
from . import *
|
||||
|
|
|
|||
|
|
@ -1,16 +1,19 @@
|
|||
"""
|
||||
MCP Guardrail Handler for Unified Guardrails.
|
||||
|
||||
Converts an MCP call_tool (name + arguments) into a single OpenAI-compatible
|
||||
tool_call and passes it to apply_guardrail. Works with the synthetic payload
|
||||
from ProxyLogging._convert_mcp_to_llm_format.
|
||||
Converts an MCP call_tool (name + arguments) into the OpenAI-compatible shape
|
||||
apply_guardrail expects: the tool as a single-entry ``tools`` definition, and
|
||||
every string leaf of the call arguments as ``texts`` so text guardrails can
|
||||
detect and mask sensitive values in the payload. Works with the synthetic
|
||||
request from ProxyLogging._convert_mcp_to_llm_format.
|
||||
|
||||
Note: For MCP tool definitions (schema) -> OpenAI tools=[], see
|
||||
litellm.experimental_mcp_client.tools.transform_mcp_tool_to_openai_tool
|
||||
when you have a full MCP Tool from list_tools. Here we only have the call
|
||||
payload (name + arguments) so we just build the tool_call.
|
||||
payload (name + arguments) so we just build the tool definition.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
|
@ -20,6 +23,8 @@ from litellm._logging import verbose_proxy_logger
|
|||
from litellm.experimental_mcp_client.tools import transform_mcp_tool_to_openai_tool
|
||||
from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation
|
||||
from litellm.proxy._experimental.mcp_server.utils import (
|
||||
MAX_STRUCTURED_CONTENT_SCAN_DEPTH,
|
||||
JSONLeafPath,
|
||||
json_string_leaves,
|
||||
json_unrewritable_labels,
|
||||
mcp_content_item_text,
|
||||
|
|
@ -42,6 +47,72 @@ if TYPE_CHECKING:
|
|||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
|
||||
|
||||
def _blocked(reason: str) -> HTTPException:
|
||||
return HTTPException(status_code=400, detail={"error": f"Content blocked: {reason}"})
|
||||
|
||||
|
||||
def _too_deeply_nested() -> HTTPException:
|
||||
return _blocked(
|
||||
f"MCP tool call arguments exceed the maximum nesting depth of {MAX_STRUCTURED_CONTENT_SCAN_DEPTH} "
|
||||
"and cannot be scanned by the configured guardrail"
|
||||
)
|
||||
|
||||
|
||||
def _argument_replacements(
|
||||
argument_leaves: tuple[tuple[JSONLeafPath, str], ...],
|
||||
masked_texts: Sequence[str] | None,
|
||||
) -> Mapping[JSONLeafPath, str]:
|
||||
"""Positionally pair the guardrail's returned texts with the leaves they came from.
|
||||
|
||||
Only leaves the guardrail actually rewrote are returned, so a guardrail that
|
||||
detects nothing leaves the outbound tool call byte-identical. A guardrail that
|
||||
returns the wrong number of texts fails closed, because a positional write-back
|
||||
would scramble the arguments rather than mask them.
|
||||
"""
|
||||
if masked_texts is not None and len(masked_texts) != len(argument_leaves):
|
||||
raise _blocked(
|
||||
f"guardrail returned {len(masked_texts)} texts for {len(argument_leaves)} MCP tool call argument strings, "
|
||||
"so the redaction cannot be mapped back to the arguments"
|
||||
)
|
||||
return {path: masked for (path, original), masked in zip(argument_leaves, masked_texts or ()) if masked != original}
|
||||
|
||||
|
||||
def _conflicting_rewrite_paths(
|
||||
scanned_leaves: tuple[tuple[JSONLeafPath, str], ...],
|
||||
current_leaves: tuple[tuple[JSONLeafPath, str], ...],
|
||||
replacements: Mapping[JSONLeafPath, str],
|
||||
) -> tuple[JSONLeafPath, ...]:
|
||||
"""Paths another guardrail already rewrote differently from what this one wants.
|
||||
|
||||
Guardrails opted into ``run_in_parallel`` all scan the same payload snapshot, so
|
||||
each one returns a full replacement string derived from the *original* leaf. Two
|
||||
of them rewriting one leaf to different values cannot be merged: writing either
|
||||
result discards the other guardrail's redaction. A leaf still holding the text
|
||||
this guardrail was handed, or already holding this guardrail's own replacement,
|
||||
is safe to write; the latter is how a guardrail that masks the arguments itself
|
||||
as well as through ``texts`` gets there first. Anything else fails closed,
|
||||
including a payload reshaped so the leaves no longer line up, because the
|
||||
write-back is positional and would land a redaction on the wrong value.
|
||||
"""
|
||||
if tuple(path for path, _ in scanned_leaves) != tuple(path for path, _ in current_leaves):
|
||||
return tuple(replacements)
|
||||
return tuple(
|
||||
path
|
||||
for (path, scanned), (_, current) in zip(scanned_leaves, current_leaves)
|
||||
if path in replacements and current not in (scanned, replacements[path])
|
||||
)
|
||||
|
||||
|
||||
def _conflicting_rewrite(paths: tuple[JSONLeafPath, ...]) -> HTTPException:
|
||||
return _blocked(
|
||||
"two guardrails running concurrently rewrote the same MCP tool call "
|
||||
f"argument{'s' if len(paths) > 1 else ''} "
|
||||
f"({', '.join('.'.join(str(part) for part in path) for path in paths)}); "
|
||||
"their redactions cannot be merged. Remove run_in_parallel from one of them so they "
|
||||
"run in sequence."
|
||||
)
|
||||
|
||||
|
||||
class MCPGuardrailTranslationHandler(BaseTranslation):
|
||||
"""Guardrail translation handler for MCP tool calls (passes a single tool_call to guardrail)."""
|
||||
|
||||
|
|
@ -52,10 +123,8 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
litellm_logging_obj: "LiteLLMLoggingObj | None" = None,
|
||||
) -> dict[str, Any]:
|
||||
mcp_tool_name: Final = data.get("mcp_tool_name") or data.get("name")
|
||||
mcp_arguments = data.get("mcp_arguments") or data.get("arguments")
|
||||
mcp_arguments: Final[object] = data.get("mcp_arguments") or data.get("arguments")
|
||||
mcp_tool_description: Final = data.get("mcp_tool_description") or data.get("description")
|
||||
if mcp_arguments is None or not isinstance(mcp_arguments, dict):
|
||||
mcp_arguments = {}
|
||||
|
||||
if not mcp_tool_name:
|
||||
verbose_proxy_logger.debug("MCP Guardrail: mcp_tool_name missing")
|
||||
|
|
@ -84,16 +153,37 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
strict=fn.get("strict", False) or False, # Default to False if None
|
||||
),
|
||||
}
|
||||
argument_leaves: Final = json_string_leaves(mcp_arguments)
|
||||
if argument_leaves is None:
|
||||
raise _too_deeply_nested()
|
||||
inputs: Final[GenericGuardrailAPIInputs] = GenericGuardrailAPIInputs(
|
||||
tools=[tool_def],
|
||||
texts=[text for _, text in argument_leaves],
|
||||
)
|
||||
|
||||
await guardrail_to_apply.apply_guardrail(
|
||||
guarded: Final = await guardrail_to_apply.apply_guardrail(
|
||||
inputs=inputs,
|
||||
request_data=data,
|
||||
input_type="request",
|
||||
logging_obj=litellm_logging_obj,
|
||||
)
|
||||
replacements: Final = _argument_replacements(
|
||||
argument_leaves=argument_leaves,
|
||||
masked_texts=guarded.get("texts") if guarded else None,
|
||||
)
|
||||
if not replacements:
|
||||
return data
|
||||
|
||||
current_arguments: Final[object] = data.get("mcp_arguments") or data.get("arguments")
|
||||
current_leaves: Final = json_string_leaves(current_arguments)
|
||||
if current_leaves is None:
|
||||
raise _too_deeply_nested()
|
||||
conflicting: Final = _conflicting_rewrite_paths(argument_leaves, current_leaves, replacements)
|
||||
if conflicting:
|
||||
raise _conflicting_rewrite(conflicting)
|
||||
masked_arguments: Final = with_json_string_leaves(current_arguments, replacements)
|
||||
data["mcp_arguments"] = masked_arguments # rebind-ok: preserve the mask for the outbound MCP call
|
||||
data["modified_arguments"] = masked_arguments # rebind-ok: expose the applied mask to the caller
|
||||
return data
|
||||
|
||||
async def process_output_response(
|
||||
|
|
@ -131,14 +221,8 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
structured_leaves: Final = json_string_leaves(structured) if structured is not None else ()
|
||||
structured_labels: Final = json_unrewritable_labels(structured) if structured is not None else ()
|
||||
if structured_leaves is None or structured_labels is None:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": (
|
||||
"Content blocked: MCP tool result structuredContent is nested too deeply to be scanned "
|
||||
"by the configured guardrail"
|
||||
)
|
||||
},
|
||||
raise _blocked(
|
||||
"MCP tool result structuredContent is nested too deeply to be scanned by the configured guardrail"
|
||||
)
|
||||
|
||||
if not text_blocks and not structured_leaves and not structured_labels:
|
||||
|
|
@ -158,12 +242,10 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
if masked_texts is None:
|
||||
return response
|
||||
if len(masked_texts) != len(originals):
|
||||
verbose_proxy_logger.warning(
|
||||
"MCP Guardrail: guardrail returned %d texts for %d tool result texts; leaving the result unmasked",
|
||||
len(masked_texts),
|
||||
len(originals),
|
||||
raise _blocked(
|
||||
f"guardrail returned {len(masked_texts)} texts for {len(originals)} MCP tool result texts, "
|
||||
"so the redaction cannot be mapped back to the result"
|
||||
)
|
||||
return response
|
||||
|
||||
split: Final = len(text_blocks)
|
||||
if content is not None:
|
||||
|
|
@ -173,15 +255,10 @@ class MCPGuardrailTranslationHandler(BaseTranslation):
|
|||
|
||||
label_start: Final = split + len(structured_leaves)
|
||||
if any(masked != original for original, masked in zip(structured_labels, masked_texts[label_start:])):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail={
|
||||
"error": (
|
||||
"Content blocked: MCP tool result matched a masking rule on a non-rewritable field "
|
||||
"(a structuredContent key or numeric value), which cannot be redacted without changing "
|
||||
"the payload contract"
|
||||
)
|
||||
},
|
||||
raise _blocked(
|
||||
"MCP tool result matched a masking rule on a non-rewritable field "
|
||||
"(a structuredContent key or numeric value), which cannot be redacted without changing "
|
||||
"the payload contract"
|
||||
)
|
||||
|
||||
structured_replacements: Final = {
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
|
|
@ -1,9 +1,9 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
|
||||
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js"],"default"]
|
||||
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
|
||||
2:I[347257,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientPageRoot"]
|
||||
3:I[871135,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js"],"default"]
|
||||
6:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"OutletBoundary"]
|
||||
7:"$Sreact.suspense"
|
||||
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3wdy9040h4b13.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/13rzpi4q1z_e8.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/0wh5uu7sl34-i.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/0cotqb-2hzyvs.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/16xdxq7qvv37h.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/2tj1x2xl0npv1.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/0z7zg9587od6_.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/1v3m908ycsmt4.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
|
||||
0:{"rsc":["$","$1","c",{"children":[["$","$L2",null,{"Component":"$3","serverProvidedParams":{"searchParams":{},"params":{},"promises":["$@4","$@5"]}}],[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/0di-9qm-8ex8r.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3dy-3uqjux30s.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/3155srena77mb.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0limvbttcca8i.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/2ty4asibief-4.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/07cqsb7poupf9.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/2dvjnwfxzyldc.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/0ab_ntohf1wik.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0esaql-j_8-p2.js","async":true}],["$","script","script-9",{"src":"/litellm-asset-prefix/_next/static/chunks/0dylouuq8ak8p.js","async":true}],["$","script","script-10",{"src":"/litellm-asset-prefix/_next/static/chunks/2bl93j-9lt0zm.js","async":true}],["$","script","script-11",{"src":"/litellm-asset-prefix/_next/static/chunks/2eonl4rcemkdj.js","async":true}],["$","script","script-12",{"src":"/litellm-asset-prefix/_next/static/chunks/2yrtzeoze9bgu.js","async":true}]],["$","$L6",null,{"children":["$","$7",null,{"name":"Next.MetadataOutlet","children":"$@8"}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
|
||||
4:{}
|
||||
5:"$0:rsc:props:children:0:props:serverProvidedParams:params"
|
||||
8:null
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
|
||||
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/1a3mamulxkyhw.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/27gjrlkmq245y.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/34_wtpkkvqa3n.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/1l61r88q65pjd.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
|
||||
2:I[92825,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ClientSegmentRoot"]
|
||||
3:I[216370,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js"],"default"]
|
||||
4:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
5:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/2k6lzy5s7rafp.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/2yqxc2yxa1-go.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/2mo45qar55a-z.js","async":true}],["$","script","script-3",{"src":"/litellm-asset-prefix/_next/static/chunks/0aoel7yrv88fp.js","async":true}],["$","script","script-4",{"src":"/litellm-asset-prefix/_next/static/chunks/3ytz29phknzsy.js","async":true}],["$","script","script-5",{"src":"/litellm-asset-prefix/_next/static/chunks/2i0218zvrsasm.js","async":true}],["$","script","script-6",{"src":"/litellm-asset-prefix/_next/static/chunks/257-u3v7vdxzj.js","async":true}],["$","script","script-7",{"src":"/litellm-asset-prefix/_next/static/chunks/1nfnjvxf_0-3n.js","async":true}],["$","script","script-8",{"src":"/litellm-asset-prefix/_next/static/chunks/0t8t3-_8y1jh9.js","async":true}]],["$","$L2",null,{"Component":"$3","slots":{"children":["$","$L4",null,{"parallelRouterKey":"children","template":["$","$L5",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]},"serverProvidedParams":{"params":{},"promises":["$@6"]}}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
|
||||
6:"$0:rsc:props:children:1:props:serverProvidedParams:params"
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
|
|
@ -1,6 +1,6 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
|
||||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
|
||||
2:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ViewportBoundary"]
|
||||
3:I[897367,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"MetadataBoundary"]
|
||||
4:"$Sreact.suspense"
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
|
||||
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
|
||||
5:I[27201,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"IconMark"]
|
||||
0:{"rsc":["$","$1","h",{"children":[null,["$","$L2",null,{"children":[["$","meta","0",{"charSet":"utf-8"}],["$","meta","1",{"name":"viewport","content":"width=device-width, initial-scale=1"}]]}],["$","div",null,{"hidden":true,"children":["$","$L3",null,{"children":["$","$4",null,{"name":"Next.Metadata","children":[["$","title","0",{"children":"LiteLLM Dashboard"}],["$","meta","1",{"name":"description","content":"LiteLLM Proxy Admin UI"}],["$","link","2",{"rel":"icon","href":"/favicon.ico?favicon.3arlap5n8tyzg.ico","sizes":"48x48","type":"image/x-icon"}],["$","link","3",{"rel":"icon","href":"/get_favicon"}],["$","$L5","4",{}]]}]}]}],["$","meta",null,{"name":"next-size-adjust","content":""}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
1:"$Sreact.fragment"
|
||||
2:I[363178,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
|
||||
3:I[12985,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
|
||||
4:I[867271,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
|
||||
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
8:I[713354,["/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
|
||||
2:I[363178,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"ThemeProvider"]
|
||||
3:I[12985,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"NuqsAdapter"]
|
||||
4:I[867271,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
5:I[557951,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"AuthProvider"]
|
||||
6:I[339756,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
7:I[837457,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"default"]
|
||||
8:I[713354,["/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js"],"Toaster"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/3s39b43k2vde7.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3kest3gurc9op.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
|
||||
0:{"rsc":["$","$1","c",{"children":[[["$","link","0",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","precedence":"next"}],["$","link","1",{"rel":"stylesheet","href":"/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","precedence":"next"}],["$","script","script-0",{"src":"/litellm-asset-prefix/_next/static/chunks/03pu_dx0gqja9.js","async":true}],["$","script","script-1",{"src":"/litellm-asset-prefix/_next/static/chunks/3fn8zqlfrwowr.js","async":true}],["$","script","script-2",{"src":"/litellm-asset-prefix/_next/static/chunks/1ntn7efqc-iiw.js","async":true}]],["$","html",null,{"lang":"en","suppressHydrationWarning":true,"children":["$","body",null,{"className":"inter_5972bc34-module__OU16Qa__className","children":["$","$L2",null,{"attribute":"class","defaultTheme":"light","enableSystem":true,"disableTransitionOnChange":true,"children":["$","$L3",null,{"children":["$","$L4",null,{"children":[["$","$L5",null,{"children":["$","$L6",null,{"parallelRouterKey":"children","template":["$","$L7",null,{}],"notFound":[[["$","title",null,{"children":"404: This page could not be found."}],["$","div",null,{"style":{"fontFamily":"system-ui,\"Segoe UI\",Roboto,Helvetica,Arial,sans-serif,\"Apple Color Emoji\",\"Segoe UI Emoji\"","height":"100vh","textAlign":"center","display":"flex","flexDirection":"column","alignItems":"center","justifyContent":"center"},"children":["$","div",null,{"children":[["$","style",null,{"dangerouslySetInnerHTML":{"__html":"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}"}}],["$","h1",null,{"className":"next-error-h1","style":{"display":"inline-block","margin":"0 20px 0 0","padding":"0 23px 0 0","fontSize":24,"fontWeight":500,"verticalAlign":"top","lineHeight":"49px"},"children":404}],["$","div",null,{"style":{"display":"inline-block"},"children":["$","h2",null,{"style":{"fontSize":14,"fontWeight":400,"lineHeight":"49px","margin":0},"children":"This page could not be found."}]}]]}]}]],[]]}]}],["$","$L8",null,{}]]}]}]}]}]}]]}],"isPartial":false,"staleTime":300,"varyParams":null,"buildId":"912qRXFjlEYHK3EAPXXTc"}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
:HL["/litellm-asset-prefix/_next/static/chunks/1kid9zr1--h6y.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3idmblk6vi8i5.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/chunks/3lvwyc5xjv11f.css","style"]
|
||||
:HL["/litellm-asset-prefix/_next/static/media/83afe278b6a6bb3c-s.p.2bn3s6zvc0dyp.woff2","font",{"crossOrigin":"","type":"font/woff2"}]
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"YAsRgSxdV-OcBfib_67Dt"}
|
||||
0:{"tree":{"name":"","param":null,"prefetchHints":16,"slots":{"children":{"name":"(dashboard)","param":null,"prefetchHints":0,"slots":{"children":{"name":"__PAGE__","param":null,"prefetchHints":0,"slots":null}}}}},"staleTime":300,"buildId":"912qRXFjlEYHK3EAPXXTc"}
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue