scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-15 23:31:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 165

Merge remote-tracking branch 'origin/litellm_internal_staging' into litellm_fix_ai_gateway_image_build

Browse source
This commit is contained in:
mateo-berri 2026-09-05 16:23:24 -07:00
commit b64d1472e0
16 changed files with 433 additions and 3200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

410
litellm/__init__.py
View file

@ -13,7 +13,6 @@ warnings.filterwarnings("ignore", message=".*`ReadOnly` qualifier.*")
### INIT VARIABLES #########################
import threading
import os
import sys
# Load .env before any other litellm imports so env vars (e.g. LITELLM_UI_SESSION_DURATION) are available
import dotenv as _dotenv
@ -46,6 +45,8 @@ from typing import (
TYPE_CHECKING,
Union,
)
from litellm.types.integrations.datadog import DatadogInitParams
from litellm.types.integrations.newrelic import NewRelicInitParams
from litellm._logging import (
set_verbose,
_turn_on_debug,
@ -94,7 +95,8 @@ from litellm.constants import (
DEFAULT_SOFT_BUDGET,
DEFAULT_ALLOWED_FAILS,
)
# httpx is lazy-loaded via __getattr__
import httpx
# register_async_client_cleanup is lazy-loaded and called on first access
litellm_mode = os.getenv("LITELLM_MODE", "DEV") # "PRODUCTION", "DEV"
@ -362,6 +364,8 @@ guardrail_name_config_map: Dict[str, GuardrailItem] = {}
include_cost_in_streaming_usage: bool = False
reasoning_auto_summary: bool = False
### PROMPTS ####
from litellm.types.prompts.init_prompts import PromptSpec
prompt_name_config_map: Dict[str, PromptSpec] = {}
##################
@ -1267,203 +1271,206 @@ openai_video_generation_models = ["sora-2"]
# get_llm_provider is lazy-loaded via __getattr__
# remove_index_from_tool_calls is lazy-loaded via __getattr__
# SDK symbols previously imported eagerly here are lazy-loaded via __getattr__
# (_SDK_SYMBOLS_IMPORT_MAP in _lazy_imports_registry.py); mirrored under TYPE_CHECKING
# so static type checkers still see them
if TYPE_CHECKING:
_key_management_settings: KeyManagementSettings
# Import KeyManagementSettings here (before utils import) because _key_management_settings
# is accessed during import time in secret_managers/main.py (via dd_tracing -> datadog -> _service_logger -> utils)
from litellm.types.secret_managers.main import KeyManagementSettings
from .utils import client
_key_management_settings: KeyManagementSettings = KeyManagementSettings()
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.deprecated_providers.palm import (
PalmConfig,
) # here to prevent breaking changes
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
from .llms.gemini.common_utils import GeminiModelInfo
# client must be imported immediately as it's used as a decorator at function definition time
from .utils import client
from .llms.vertex_ai.vertex_embeddings.transformation import (
VertexAITextEmbeddingConfig,
)
# Note: Most other utils imports are lazy-loaded via __getattr__ to avoid loading utils.py
# (which imports tiktoken) at import time
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
from .llms.custom_llm import CustomLLM
from .llms.anthropic.common_utils import AnthropicModelInfo
from .llms.ai21.chat.transformation import AI21ChatConfig, AI21ChatConfig as AI21Config
from .llms.deprecated_providers.palm import (
PalmConfig,
) # here to prevent breaking changes
from .llms.deprecated_providers.aleph_alpha import AlephAlphaConfig
from .llms.gemini.common_utils import GeminiModelInfo
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from .llms.topaz.common_utils import TopazModelInfo
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
from .llms.vertex_ai.vertex_embeddings.transformation import (
VertexAITextEmbeddingConfig,
)
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
vertexAITextEmbeddingConfig = VertexAITextEmbeddingConfig()
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
## Lazy loading this is not straightforward, will leave it here for now.
from .main import *
from .compression import compress
from .llms.bedrock.embed.amazon_titan_v2_transformation import (
AmazonTitanV2Config,
)
from .llms.topaz.common_utils import TopazModelInfo
# Skills API
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .evals.main import (
create_eval,
acreate_eval,
list_evals,
alist_evals,
get_eval,
aget_eval,
delete_eval,
adelete_eval,
cancel_eval,
acancel_eval,
create_run,
acreate_run,
list_runs,
alist_runs,
get_run,
aget_run,
delete_run,
adelete_run,
cancel_run,
acancel_run,
)
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
from .exceptions import (
AuthenticationError,
InvalidRequestError,
BadRequestError,
ImageFetchError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitErrorCategory,
RateLimitType,
ServiceUnavailableError,
BadGatewayError,
OpenAIError,
ContextWindowExceededError,
ContentPolicyViolationError,
BudgetExceededError,
APIError,
Timeout,
APIConnectionError,
UnsupportedParamsError,
APIResponseValidationError,
UnprocessableEntityError,
InternalServerError,
JSONSchemaValidationError,
LITELLM_EXCEPTION_TYPES,
MockException,
)
from .budget_manager import BudgetManager
from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
# OpenAIOSeriesConfig is lazy loaded - openaiOSeriesConfig will be created on first access
# OpenAIGPTConfig, OpenAIGPT5Config, etc. are lazy loaded - instances will be created on first access
from .llms.xai.common_utils import XAIModelInfo
# Interactions API is available as litellm.interactions module
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
from . import interactions
from .interactions.agents.main import (
acreate as acreate_agent,
create as create_agent,
alist as alist_agents,
list as list_agents,
aget as aget_agent,
get as get_agent,
adelete as adelete_agent,
delete as delete_agent,
alist_versions as alist_agent_versions,
list_versions as list_agent_versions,
)
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .containers.main import *
from .ocr.main import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
arealtime_calls,
)
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
acreate as avector_store_file_create,
adelete as avector_store_file_delete,
alist as avector_store_file_list,
aretrieve as avector_store_file_retrieve,
aretrieve_content as avector_store_file_content,
aupdate as avector_store_file_update,
create as vector_store_file_create,
delete as vector_store_file_delete,
list as vector_store_file_list,
retrieve as vector_store_file_retrieve,
retrieve_content as vector_store_file_content,
update as vector_store_file_update,
)
from .scheduler import *
# PublicAI now uses JSON-based configuration (see litellm/llms/openai_like/providers.json)
# All remaining configs are now lazy loaded - see _lazy_imports_registry.py
### ADAPTERS ###
import litellm.anthropic_interface as anthropic
# Import LlmProviders here (before main import) because it's imported during import time
# in multiple places including openai.py (via main import)
from litellm.types.utils import LlmProviders
### Vector Store Registry ###
## Lazy loading this is not straightforward, will leave it here for now.
from .main import *
from .compression import compress
### RAG ###
from . import rag
# Skills API
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .evals.main import (
create_eval,
acreate_eval,
list_evals,
alist_evals,
get_eval,
aget_eval,
delete_eval,
adelete_eval,
cancel_eval,
acancel_eval,
create_run,
acreate_run,
list_runs,
alist_runs,
get_run,
aget_run,
delete_run,
adelete_run,
cancel_run,
acancel_run,
)
from .integrations import *
from .llms.custom_httpx.async_client_cleanup import close_litellm_async_clients
from .exceptions import (
AuthenticationError,
InvalidRequestError,
BadRequestError,
ImageFetchError,
NotFoundError,
PermissionDeniedError,
RateLimitError,
RateLimitErrorCategory,
RateLimitType,
ServiceUnavailableError,
BadGatewayError,
OpenAIError,
ContextWindowExceededError,
ContentPolicyViolationError,
BudgetExceededError,
APIError,
Timeout,
APIConnectionError,
UnsupportedParamsError,
APIResponseValidationError,
UnprocessableEntityError,
InternalServerError,
JSONSchemaValidationError,
LITELLM_EXCEPTION_TYPES,
MockException,
)
from .budget_manager import BudgetManager
from .proxy.proxy_cli import run_server
from .router import Router
from .assistants.main import *
from .batches.main import *
from .images.main import *
from .videos.main import *
from .batch_completion.main import *
from .rerank_api.main import *
from .llms.anthropic.experimental_pass_through.messages.handler import *
from .responses.main import *
### CUSTOM LLMs ###
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route
from .google_genai import agenerate_content
# Interactions API is available as litellm.interactions module
# Usage: litellm.interactions.create(), litellm.interactions.get(), etc.
from . import interactions
from .interactions.agents.main import (
acreate as acreate_agent,
create as create_agent,
alist as alist_agents,
list as list_agents,
aget as aget_agent,
get as get_agent,
adelete as adelete_agent,
delete as delete_agent,
alist_versions as alist_agent_versions,
list_versions as list_agent_versions,
)
from .skills.main import (
create_skill,
acreate_skill,
list_skills,
alist_skills,
get_skill,
aget_skill,
delete_skill,
adelete_skill,
)
from .containers.main import *
from .ocr.main import *
from .rust_bridge import rust
from .rag.main import *
from .sandbox.main import *
from .search.main import *
from .realtime_api.main import (
_arealtime,
acreate_realtime_client_secret,
acreate_realtime_transcription_session,
arealtime_calls,
)
from .responses.main import _aresponses_websocket
from .fine_tuning.main import *
from .files.main import *
from .vector_store_files.main import (
acreate as avector_store_file_create,
adelete as avector_store_file_delete,
alist as avector_store_file_list,
aretrieve as avector_store_file_retrieve,
aretrieve_content as avector_store_file_content,
aupdate as avector_store_file_update,
create as vector_store_file_create,
delete as vector_store_file_delete,
list as vector_store_file_list,
retrieve as vector_store_file_retrieve,
retrieve_content as vector_store_file_content,
update as vector_store_file_update,
)
from .scheduler import *
### ADAPTERS ###
from .types.adapter import AdapterItem
import litellm.anthropic_interface as anthropic
adapters: List[AdapterItem] = []
### Vector Store Registry ###
from .vector_stores.vector_store_registry import (
VectorStoreRegistry,
VectorStoreIndexRegistry,
)
vector_store_registry: Optional[VectorStoreRegistry] = None
vector_store_index_registry: Optional[VectorStoreIndexRegistry] = None
### RAG ###
from . import rag
### CUSTOM LLMs ###
from .types.llms.custom_llm import CustomLLMItem
custom_provider_map: List[CustomLLMItem] = []
_custom_providers: List[str] = [] # internal helper util, used to track names of custom providers
disable_hf_tokenizer_download: Optional[bool] = (
@ -1471,6 +1478,13 @@ disable_hf_tokenizer_download: Optional[bool] = (
)
global_disable_no_log_param: bool = False
### CLI UTILITIES ###
from litellm.litellm_core_utils.cli_token_utils import get_litellm_gateway_api_key
### PASSTHROUGH ###
from .passthrough import allm_passthrough_route, llm_passthrough_route
from .google_genai import agenerate_content
### GLOBAL CONFIG ###
global_bitbucket_config: Optional[Dict[str, Any]] = None
@ -1494,21 +1508,10 @@ def set_global_gitlab_config(config: Dict[str, Any]) -> None:
# Lazy loading system for heavy modules to reduce initial import time and memory usage
if TYPE_CHECKING:
import httpx
from litellm.types.utils import ModelInfo as _ModelInfoType
from litellm.types.utils import PriorityReservationSettings
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.caching.caching import Cache
from litellm.types.adapter import AdapterItem
from litellm.types.integrations.datadog import DatadogInitParams
from litellm.types.integrations.newrelic import NewRelicInitParams
from litellm.types.llms.custom_llm import CustomLLMItem
from litellm.types.prompts.init_prompts import PromptSpec
from litellm.vector_stores.vector_store_registry import (
VectorStoreIndexRegistry,
VectorStoreRegistry,
)
# Type stubs for lazy-loaded configs to help mypy
from .llms.bedrock.chat.converse_transformation import (
@ -2184,6 +2187,16 @@ if TYPE_CHECKING:
# Track if async client cleanup has been registered (for lazy loading)
_async_client_cleanup_registered = False
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
# See: https://github.com/BerriAI/litellm/issues/18659
# This ensures encoding is initialized before VCR starts recording HTTP requests
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
# Load encoding at import time (pre-#18070 behavior)
# This ensures encoding is initialized before VCR starts recording
from .main import encoding
def __getattr__(name: str) -> Any:
"""Lazy import handler with cached registry for improved performance."""
@ -2263,8 +2276,6 @@ def __getattr__(name: str) -> Any:
"openAIGPT5Config": "OpenAIGPT5Config",
"nvidiaNimConfig": "NvidiaNimConfig",
"nvidiaNimEmbeddingConfig": "NvidiaNimEmbeddingConfig",
"vertexAITextEmbeddingConfig": "VertexAITextEmbeddingConfig",
"_key_management_settings": "KeyManagementSettings",
}
if name in _config_instances:
from ._lazy_imports import get_litellm_globals
@ -2382,30 +2393,7 @@ def __getattr__(name: str) -> Any:
return locals()[name]
from ._lazy_imports import lazy_import_litellm_submodule
submodule: Final = lazy_import_litellm_submodule(name)
if submodule is not None:
return submodule
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from ._lazy_imports import LiteLLMModule
from ._lazy_imports_registry import STAR_IMPORT_PUBLIC_NAMES
sys.modules[__name__].__class__ = LiteLLMModule
__all__ = list(STAR_IMPORT_PUBLIC_NAMES) # mutable-ok: star imports require __all__ to be a list of str
# ALL_LITELLM_RESPONSE_TYPES is lazy-loaded via __getattr__ to avoid loading utils at import time
# Eager loading for backwards compatibility with VCR and other HTTP recording tools
# When LITELLM_DISABLE_LAZY_LOADING is set, lazy-loaded attributes are loaded at import time
# For now, this only affects encoding (tiktoken) as it was the only reported issue
# See: https://github.com/BerriAI/litellm/issues/18659
# This ensures encoding is initialized before VCR starts recording HTTP requests
# This block stays at the bottom so __getattr__ can resolve attributes main.py needs during its import
if os.getenv("LITELLM_DISABLE_LAZY_LOADING", "").lower() in ("1", "true", "yes", "on"):
from .main import encoding

94
litellm/_lazy_imports.py
View file

@ -16,10 +16,9 @@ until they're actually needed.
"""
import importlib
import importlib.util
import sys
from collections.abc import Callable, Mapping
from types import MappingProxyType, ModuleType
from types import ModuleType
from typing import TYPE_CHECKING, Any, Final, cast
from typing_extensions import ReadOnly, TypedDict
@ -35,8 +34,6 @@ from ._lazy_imports_registry import (
_LITELLM_LOGGING_IMPORT_MAP,
_LLM_CONFIGS_IMPORT_MAP,
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
_SDK_MODULE_ALIASES,
_SDK_SYMBOLS_IMPORT_MAP,
_TOKEN_COUNTER_IMPORT_MAP,
_TYPES_IMPORT_MAP,
_TYPES_UTILS_IMPORT_MAP,
@ -81,10 +78,7 @@ def _get_utils_globals() -> dict[str, object]:
This is where we cache imported attributes so we don't import them twice.
When you do `litellm.utils.some_function`, it gets stored in this dictionary.
"""
cached: Final = sys.modules.get("litellm.utils")
if cached is not None:
return cached.__dict__
return importlib.import_module("litellm.utils").__dict__
return sys.modules["litellm.utils"].__dict__
def _get_module_level_client_timeout(litellm_globals: Mapping[str, Any]) -> "float | httpx.Timeout | None":
@ -220,10 +214,6 @@ def _get_lazy_import_registry() -> dict[str, Callable[[str], object]]:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_llm_provider_logic
for name in UTILS_MODULE_NAMES:
_LAZY_IMPORT_REGISTRY[name] = _lazy_import_utils_module
for name in _SDK_SYMBOLS_IMPORT_MAP:
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_symbols)
for name in _SDK_MODULE_ALIASES:
_LAZY_IMPORT_REGISTRY.setdefault(name, _lazy_import_sdk_module_alias)
return _LAZY_IMPORT_REGISTRY
@ -360,86 +350,6 @@ def _lazy_import_llm_provider_logic(name: str) -> object:
return _generic_lazy_import(name, _LLM_PROVIDER_LOGIC_IMPORT_MAP, "LLM provider logic")
def _lazy_import_sdk_symbols(name: str) -> object:
"""Handler for SDK symbols previously imported eagerly at the bottom of litellm/__init__.py"""
return _generic_lazy_import(name, _SDK_SYMBOLS_IMPORT_MAP, "SDK symbols")
def _lazy_import_sdk_module_alias(name: str) -> object:
"""Handler for litellm attributes that bind a module (e.g. litellm.anthropic)"""
_globals: Final = get_litellm_globals()
if name in _globals:
return _globals[name]
module: Final = importlib.import_module(_SDK_MODULE_ALIASES[name])
_globals[name] = module # rebind-ok: caches the resolved module alias on the package
return module
_SHADOWABLE_SDK_FUNCTIONS: Final = MappingProxyType(
{
"batch_completion": ("litellm.batch_completion.main", "batch_completion"),
"ocr": ("litellm.ocr.main", "ocr"),
"responses": ("litellm.responses.main", "responses"),
"search": ("litellm.search.main", "search"),
}
)
def _shadowable_function_property(name: str) -> property:
"""Property keeping litellm.<name> bound to the SDK function even after the import
machinery binds the identically named litellm.<name> subpackage onto the litellm module."""
module_path, attr_name = _SHADOWABLE_SDK_FUNCTIONS[name]
def _get(module: ModuleType) -> object:
stored: Final = module.__dict__.get(name)
if stored is not None and not (isinstance(stored, ModuleType) and stored.__name__ == f"litellm.{name}"):
return stored
value: Final = _module_attribute(importlib.import_module(module_path), attr_name)
module.__dict__[name] = value # rebind-ok: caches the resolved function on the litellm module
return value
def _set(module: ModuleType, value: object) -> None:
module.__dict__[name] = value # rebind-ok: property setter must store assignments on the module
return property(_get, _set)
class LiteLLMModule(ModuleType):
"""Module type installed on the litellm package so function names shadowed by
same-named subpackages (litellm.responses, ...) keep resolving to the functions."""
batch_completion = _shadowable_function_property("batch_completion")
ocr = _shadowable_function_property("ocr")
responses = _shadowable_function_property("responses")
search = _shadowable_function_property("search")
def lazy_import_submodule(package: str, name: str) -> "ModuleType | None":
"""Resolve <package>.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
if name.startswith("__") or not name.isidentifier():
return None
qualified_name: Final = f"{package}.{name}"
try:
spec: Final = importlib.util.find_spec(qualified_name)
except ModuleNotFoundError:
return None
if spec is None:
return None
try:
module: Final = importlib.import_module(qualified_name)
except ModuleNotFoundError as exc:
if exc.name == qualified_name:
return None
raise
sys.modules[package].__dict__[name] = module # rebind-ok: caches the resolved submodule on the package
return module
def lazy_import_litellm_submodule(name: str) -> "ModuleType | None":
"""Resolve litellm.<name> as a submodule (e.g. litellm.utils) when no other handler matches"""
return lazy_import_submodule("litellm", name)
def _lazy_import_utils_module(name: str) -> object:
"""
Handler for utils module lazy imports.

2693
litellm/_lazy_imports_registry.py
View file

File diff suppressed because it is too large Load diff

12
litellm/proxy/__init__.py
View file

@ -1,11 +1 @@
from types import ModuleType
from typing import Final
def __getattr__(name: str) -> ModuleType:
from litellm._lazy_imports import lazy_import_submodule
submodule: Final = lazy_import_submodule(__name__, name)
if submodule is not None:
return submodule
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
from . import *

6
litellm/proxy/hooks/proxy_track_cost_callback.py
View file

@ -254,7 +254,7 @@ class _ProxyDBLogger(CustomLogger):
key_alias: Final = cast(str | None, metadata.get("user_api_key_alias", None))
end_user_max_budget: Final = metadata.get("user_api_end_user_max_budget", None)
sl_object: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
response_cost = (
response_cost: Final = (
sl_object.get("response_cost", None) if sl_object is not None else kwargs.get("response_cost", None)
)
tags: Final = _get_request_tags_for_cost_tracking(
@ -269,10 +269,6 @@ class _ProxyDBLogger(CustomLogger):
if response_cost is not None:
user_api_key: Final = metadata.get("user_api_key", None)
if kwargs.get("cache_hit", False) is True:
response_cost = 0.0
verbose_proxy_logger.debug("Cache Hit: response_cost %s, for user_id %s", response_cost, user_id)
verbose_proxy_logger.debug(
"user_api_key %s, user_id %s, team_id %s, end_user_id %s",
user_api_key,

2
tests/e2e/CLAUDE.md
View file

@ -77,7 +77,7 @@ Mark live tests with `@pytest.mark.e2e` (on the class or the module). Pure cover
The seam is `provider_edge.py`: `start_provider_edge` boots an in-process HTTP server (one shared instance per pytest process, `e2e_config.provider_edge_base` is the accessor) that mounts each supported provider under a path prefix (`EDGE_MOUNTS`: `/openai` -> `https://api.openai.com`, `/anthropic` -> `https://api.anthropic.com`). A test participates by registering its deployment with `api_base=provider_edge_base("openai")` plus the provider's path suffix; `quota_management/spend_tracking/test_provider_edge_spend_e2e.py` is the reference. In live mode the accessor returns None and the deployment defaults to the real provider, so an edge-wired test runs in all three modes unchanged. Non-wired tests hit their providers live in every mode. The edge binds `E2E_PROVIDER_EDGE_BIND_HOST` (default 127.0.0.1) and advertises `E2E_PROVIDER_EDGE_ADVERTISE_HOST` in the api_base it hands out, for proxies running in containers
A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket
A bundle (default `tests/e2e/.fixtures`, override with `E2E_FIXTURE_DIR`) is a directory: `manifest.json` carries the record timestamp, harness git version, and format version, and each test gets a subdirectory holding one JSON file per provider call in call order (`0000-post-openai-v1-chat-completions.json`). Request headers are never stored (provider credentials never touch disk), non-JSON request bodies store a canonicalized sha256 digest instead of the bytes, `multipart/form-data` bodies store their ordinary fields plus a JSON list of the uploaded parts' `[field, filename, content-type]` triples and a digest of their content, so the per-request random boundary and the envelope never reach the key, and responses store status, filtered headers, and the verbatim body base64-encoded, which is part of why bundles are gitignored. Responses come in two shapes told apart by a `kind` tag: an ordinary one holding a single base64 body, and, for a response the provider streamed (`content-type: text/event-stream`), one holding its transfer chunks in order plus why the stream ended early if it did, so replay reproduces the split points the provider chose instead of one coalesced body. `fixture_bundle.py` owns the format, and `BUNDLE_FORMAT_VERSION` is checked on load, so a bundle recorded under older rules is refused by name rather than partially read. Record serves the proxy the same filtered stored response replay will serve later, chunk for chunk on a stream, so the two modes are byte-identical from the proxy's side of the socket. Replay does not reproduce the provider's inter-chunk timing (chunks go out as fast as the socket takes them), so a test that judges streaming on the clock, such as the `stream_event_arrivals` lead between the first content delta and `message_stop`, gates that assertion on `provider_paces_stream()` and proves only the event grammar in replay
Multipart identity is the fiddly corner, and the rules exist because each one had a collision behind it. A part counts as an upload when it carries a filename or declares its own content type, and everything else is an ordinary field. Field names get a `name[n]` suffix on repeats, with a literal `[` doubled first, so a form that repeats `purpose` never keys the same as one that literally sends `purpose[1]`. A field whose name reads as a credential is stored as `<secret>`, which stays key-preserving because the key is recomputed from the stored request rather than saved alongside it, so the live request carrying the real value still matches its redacted fixture. A field value that is not UTF-8 is stored as a base64 sha256 digest, base64 and not hex because the canonicalizer rewrites any 64-character hex run to `<sha256>` and would fold every binary value onto one key. The uploaded parts contribute a JSON list rather than a `field:filename` string, so a separator inside a filename cannot impersonate a field boundary, and their byte length is stored for a reader's benefit but deliberately left out of the key, since the canonicalizer absorbs timestamp and id drift inside a file that changes its length

8
tests/e2e/e2e_config.py
View file

@ -10,6 +10,7 @@ import os
import time
import uuid
from pathlib import Path
from typing import Final
from dotenv import load_dotenv
@ -192,6 +193,13 @@ def provider_edge_base(mount: str) -> str | None:
)
STREAM_MIN_LEAD_SECONDS: Final = 1.0
def provider_paces_stream() -> bool:
return parse_fixture_mode(FIXTURE_MODE_RAW) != "replay"
def unique_marker() -> str:
"""A short unique token per call/run, so concurrent runs and the shared
response cache never collide on prompts, tags, or customer ids. In record

92
tests/e2e/e2e_http.py
View file

@ -16,9 +16,9 @@ requests itself imports.
from __future__ import annotations
import time
from collections.abc import Callable
from collections.abc import Callable, Mapping
from dataclasses import dataclass
from typing import Generator, Generic, Iterator, Literal, NewType, Protocol, TypeVar, cast
from typing import Final, Generator, Generic, Iterator, Literal, NewType, Protocol, TypeVar, cast
import pytest
import requests
@ -142,6 +142,7 @@ class StreamingResponse(BaseModel):
body: str
chunks: int = 0 # streamed events (0 for non-streaming)
stream_events: list[str] = []
stream_event_arrivals: list[float] = []
# First in-stream error event, if any. A streamed call commits its HTTP 200
# before the upstream completes, so upstream failures (e.g. insufficient
# quota) arrive as SSE error events inside an otherwise-successful response;
@ -184,7 +185,20 @@ class BinaryStream(BaseModel):
return "chunked" in (self.transfer_encoding or "")
def _hdr(resp: requests.Response, name: str) -> str | None:
class SseResponse(Protocol):
@property
def status_code(self) -> int: ...
@property
def headers(self) -> Mapping[str, str]: ...
@property
def text(self) -> str: ...
def iter_lines(self) -> Iterator[bytes]: ...
def _hdr(resp: SseResponse, name: str) -> str | None:
value = resp.headers.get(name)
return value if isinstance(value, str) else None
@ -457,7 +471,7 @@ def probe(
return ProbeResult(status_code=resp.status_code, body=resp.text)
def _parse_response_cost(resp: requests.Response) -> float | None:
def _parse_response_cost(resp: SseResponse) -> float | None:
raw = _hdr(resp, "x-litellm-response-cost")
if raw is None or raw == "":
return None
@ -467,11 +481,26 @@ def _parse_response_cost(resp: requests.Response) -> float | None:
return None
def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingResponse:
call_id = _hdr(resp, "x-litellm-call-id")
response_cost = _parse_response_cost(resp)
content_type = _hdr(resp, "content-type")
headers = {name.lower(): value for name, value in resp.headers.items()}
_SSE_DATA_PREFIX: Final = b"data: "
_SSE_DONE: Final = "[DONE]"
def _is_stream_error_line(line: bytes) -> bool:
return (
line.startswith(b"event: error")
or b'"type":"error"' in line
or b'"type": "error"' in line
or line.startswith(b'data: {"error"')
)
def streaming_outcome(
resp: SseResponse, stream: bool, *, sent_at: float, clock: Callable[[], float] = time.monotonic
) -> StreamingResponse:
call_id: Final = _hdr(resp, "x-litellm-call-id")
response_cost: Final = _parse_response_cost(resp)
content_type: Final = _hdr(resp, "content-type")
headers: Final = {name.lower(): value for name, value in resp.headers.items()}
if not stream or not (200 <= resp.status_code < 300):
return StreamingResponse(
status_code=resp.status_code,
@ -481,29 +510,13 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon
headers=headers,
body=resp.text,
)
lines = cast("Iterator[bytes]", resp.iter_lines())
chunks = 0
stream_error: str | None = None
stream_events: list[str] = []
stream_done = False
for line in lines:
if not line:
continue
chunks += 1
decoded_line = line.decode(errors="replace")
if decoded_line.startswith("data: "):
payload = decoded_line.removeprefix("data: ")
if payload == "[DONE]":
stream_done = True
else:
stream_events.append(payload)
if stream_error is None and (
line.startswith(b"event: error")
or b'"type":"error"' in line
or b'"type": "error"' in line
or line.startswith(b'data: {"error"')
):
stream_error = line.decode(errors="replace")[:300]
stamped: Final = tuple((line, clock() - sent_at) for line in resp.iter_lines() if line)
payloads: Final = tuple(
(line.removeprefix(_SSE_DATA_PREFIX).decode(errors="replace"), arrived)
for line, arrived in stamped
if line.startswith(_SSE_DATA_PREFIX)
)
events: Final = tuple((payload, arrived) for payload, arrived in payloads if payload != _SSE_DONE)
return StreamingResponse(
status_code=resp.status_code,
call_id=call_id,
@ -511,10 +524,14 @@ def _streaming_outcome(resp: requests.Response, stream: bool) -> StreamingRespon
content_type=content_type,
headers=headers,
body="<streamed>",
chunks=chunks,
stream_events=stream_events,
stream_done=stream_done,
stream_error=stream_error,
chunks=len(stamped),
stream_events=[payload for payload, _ in events],
stream_event_arrivals=[arrived for _, arrived in events],
stream_done=any(payload == _SSE_DONE for payload, _ in payloads),
stream_error=next(
(line.decode(errors="replace")[:300] for line, _ in stamped if _is_stream_error_line(line)),
None,
),
)
@ -531,6 +548,7 @@ def send(
x-litellm-call-id header. For native/passthrough bodies and for calls judged by
status rather than a typed JSON model (e.g. a budget block is a non-2xx). With
``stream=True`` the SSE body is consumed and its events counted instead."""
sent_at: Final = time.monotonic()
try:
resp = request_with_retry(
lambda: requests.post(
@ -544,7 +562,7 @@ def send(
)
except requests.RequestException as exc:
return StreamingResponse(status_code=-1, body=str(exc))
return _streaming_outcome(resp, stream)
return streaming_outcome(resp, stream, sent_at=sent_at)
def stream(

37
tests/e2e/llm_translation/test_messages_e2e.py
View file

@ -8,8 +8,15 @@ litellm-regression-tests/tests/test_inference_endpoints.py.
from __future__ import annotations
from typing import Final
import pytest
from e2e_config import provider_edge_base, unique_marker
from e2e_config import (
STREAM_MIN_LEAD_SECONDS,
provider_edge_base,
provider_paces_stream,
unique_marker,
)
from e2e_http import assert_client_error, require_successful_call, unwrap
from endpoints_client import EndpointsClient, MessagesResult
from lifecycle import ResourceManager
@ -159,19 +166,21 @@ class TestAnthropicMessages:
"""Edge-wired like its non-streaming siblings, so record and replay both
carry the streamed response.
Asserts the shape of the event sequence, not just that deltas and a stop
appeared somewhere in it: the answer arrives across several deltas, and the
usage event sits between the last of them and ``message_stop``. A replay that
coalesced the response into one buffered body could not satisfy either."""
Asserts what the proxy controls: the event grammar (usage between the last
content delta and ``message_stop``) and, on the clock, that the relay is
incremental. How many deltas a reply is split into is the provider's choice, so
the first content delta must instead reach the client well before
``message_stop``, which a buffered response cannot do. Replay serves chunks back
to back, so only live and record runs judge the timing."""
model, key = self._register(endpoints_client, resources)
result = endpoints_client.proxy.messages_stream(
key,
AnthropicMessagesBody(
model=model,
max_tokens=400,
max_tokens=800,
stream=True,
messages=[ChatMessage(role="user", content="Count from 1 to 100, one number per line.")],
messages=[ChatMessage(role="user", content="Count from 1 to 200, one number per line.")],
),
)
require_successful_call(result)
@ -186,10 +195,7 @@ class TestAnthropicMessages:
delta_positions = [
index for index, event in enumerate(events) if event.type == "content_block_delta"
]
assert len(delta_positions) >= 2, (
f"stream carried {len(delta_positions)} content deltas, so it was not "
f"incremental: {types}"
)
assert delta_positions, f"stream carried no content deltas: {types}"
text = "".join(
event.delta.text
for event in events
@ -209,6 +215,15 @@ class TestAnthropicMessages:
f"usage did not land between the last content delta and message_stop: {types}"
)
first_delta_at: Final = result.stream_event_arrivals[delta_positions[0]]
stop_at: Final = result.stream_event_arrivals[stop_position]
if provider_paces_stream():
assert stop_at - first_delta_at >= STREAM_MIN_LEAD_SECONDS, (
f"first content delta reached the client {first_delta_at:.2f}s after the request "
f"and message_stop {stop_at:.2f}s after it; a relayed stream shows the first delta "
f"at least {STREAM_MIN_LEAD_SECONDS}s before the end, so the response was buffered"
)
@pytest.mark.covers("llm.messages.anthropic.tool_use.nonstream.works")
def test_messages_tool_use(
self, endpoints_client: EndpointsClient, resources: ResourceManager

29
tests/e2e/llm_translation/test_together_ai_e2e.py
View file

@ -23,7 +23,7 @@ from datetime import date
from typing import Final
import pytest
from e2e_config import unique_marker
from e2e_config import STREAM_MIN_LEAD_SECONDS, provider_paces_stream, unique_marker
from e2e_http import StreamingResponse, require_successful_call, unwrap
from lifecycle import ResourceManager
from models import (
@ -81,7 +81,7 @@ PERSON_RESPONSE_FORMAT: dict[str, object] = {
}
WEATHER_PROMPT = "What is the weather in Paris? Use the tool."
WEATHER_REPORT = "Paris: 22 degrees Celsius, clear skies, wind from the northwest at 9 km/h"
COUNTING_PROMPT = "Count from 1 to 20, one number per line."
COUNTING_PROMPT = "Count from 1 to 200, one number per line."
WEATHER_TOOL = ChatTool(
function=ChatToolFunction(
@ -753,7 +753,7 @@ class TestTogetherMessages:
key,
AnthropicMessagesBody(
model=model,
max_tokens=512,
max_tokens=2048,
stream=True,
messages=[ChatMessage(role="user", content=COUNTING_PROMPT)],
),
@ -763,11 +763,24 @@ class TestTogetherMessages:
assert not result.stream_error, f"stream errored: {result.stream_error}"
events = [_MessagesStreamEvent.model_validate_json(event) for event in result.stream_events]
types = [event.type for event in events]
text_deltas = [
delta_positions = [
index for index, event in enumerate(events) if event.type == "content_block_delta"
]
assert delta_positions, f"stream carried no content deltas: {types}"
text = "".join(
event.delta.text
for event in events
if event.type == "content_block_delta" and event.delta is not None and event.delta.text
]
assert len(text_deltas) >= 2, f"stream was not incremental: {types}"
assert "20" in "".join(text_deltas), f"streamed text lost the answer: {text_deltas}"
if event.type == "content_block_delta" and event.delta is not None
)
assert "200" in text, f"streamed text lost the answer: {text[:300]!r}"
assert "message_stop" in types, f"stream never reached message_stop: {types}"
stop_position: Final = types.index("message_stop")
first_delta_at: Final = result.stream_event_arrivals[delta_positions[0]]
stop_at: Final = result.stream_event_arrivals[stop_position]
if provider_paces_stream():
assert stop_at - first_delta_at >= STREAM_MIN_LEAD_SECONDS, (
f"first content delta reached the client {first_delta_at:.2f}s after the request "
f"and message_stop {stop_at:.2f}s after it; a relayed stream shows the first delta "
f"at least {STREAM_MIN_LEAD_SECONDS}s before the end, so the response was buffered"
)

58
tests/e2e/test_e2e_http.py
View file

@ -12,12 +12,14 @@ monkeypatches anything.
from __future__ import annotations
from collections.abc import Callable, Sequence
from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass, field
from types import MappingProxyType
from typing import Final
import pytest
from e2e_http import RETRY_ATTEMPTS, TRANSIENT_STATUSES, request_with_retry
from e2e_http import RETRY_ATTEMPTS, TRANSIENT_STATUSES, request_with_retry, streaming_outcome
@dataclass
@ -80,3 +82,55 @@ class TestTransientRetryPolicy:
assert result is responses[RETRY_ATTEMPTS - 1]
assert sleep.delays == [0.5, 1.0]
assert [r.close_calls for r in responses] == [1, 1, 0, 0]
@dataclass(frozen=True, slots=True)
class FakeSseResponse:
lines: Sequence[bytes]
status_code: int = 200
headers: Mapping[str, str] = MappingProxyType({"content-type": "text/event-stream"})
text: str = ""
def iter_lines(self) -> Iterator[bytes]:
return iter(self.lines)
def _ticking_clock(start: float, step: float) -> Callable[[], float]:
ticks: Final = iter(range(10_000))
return lambda: start + step * next(ticks)
class TestStreamEventArrivals:
def test_each_event_is_stamped_at_the_moment_its_line_arrives(self) -> None:
resp: Final = FakeSseResponse(
lines=(
b"event: message_start",
b'data: {"type":"message_start"}',
b"",
b"event: ping",
b'data: {"type":"ping"}',
b"event: content_block_delta",
b'data: {"type":"content_block_delta"}',
b"data: [DONE]",
)
)
result: Final = streaming_outcome(resp, True, sent_at=100.0, clock=_ticking_clock(start=100.0, step=0.5))
assert result.stream_events == [
'{"type":"message_start"}',
'{"type":"ping"}',
'{"type":"content_block_delta"}',
]
assert result.stream_event_arrivals == [0.5, 1.5, 2.5]
assert result.stream_done
assert result.chunks == 7
def test_a_non_streaming_outcome_carries_no_arrivals(self) -> None:
resp: Final = FakeSseResponse(lines=(), status_code=400, text="bad request")
result: Final = streaming_outcome(resp, True, sent_at=0.0, clock=_ticking_clock(start=0.0, step=1.0))
assert result.stream_events == []
assert result.stream_event_arrivals == []
assert result.body == "bad request"

47
tests/test_litellm/proxy/hooks/test_proxy_track_cost_callback.py
View file

@ -1783,6 +1783,53 @@ async def test_track_cost_callback_enriches_user_id_for_mcp_style_metadata():
)
@pytest.mark.asyncio
async def test_track_cost_callback_keeps_guardrail_cost_on_cache_hit():
"""A cache hit skips the LLM, not the guardrail that screened the prompt, so the
guardrail's provider charge must still reach spend logs and budgets. The payload
already prices the LLM share at 0 on a cache hit, so its response_cost is the
guardrail cost alone and the callback must pass it through untouched."""
logger = _ProxyDBLogger()
kwargs = {
"call_type": "acompletion",
"model": "gpt-4o",
"cache_hit": True,
"response_cost": 0.0,
"litellm_params": {
"metadata": {
"user_api_key": "hashed-key",
"user_api_key_user_id": "user-1",
"user_api_key_team_id": "team-1",
}
},
"standard_logging_object": {
"response_cost": 0.0003,
"request_tags": [],
"metadata": {},
"cost_breakdown": {"guardrail_cost": 0.0003, "total_cost": 0.0003},
},
}
with (
patch("litellm.proxy.proxy_server.increment_spend_counters", new_callable=AsyncMock) as mock_increment, # test-quality-ok: the callback imports this from proxy_server inside its body, so there is no injection seam
patch("litellm.proxy.proxy_server.update_cache", new_callable=AsyncMock), # test-quality-ok: same function-body import, no injection seam
patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging, # test-quality-ok: same function-body import, no injection seam
):
mock_proxy_logging.db_spend_update_writer.update_database = AsyncMock()
mock_proxy_logging.slack_alerting_instance.customer_spend_alert = AsyncMock()
await logger._PROXY_track_cost_callback(
kwargs=kwargs,
completion_response={"id": "cached-call-1"},
start_time=datetime.now(),
end_time=datetime.now(),
)
update_kwargs = mock_proxy_logging.db_spend_update_writer.update_database.await_args.kwargs
assert update_kwargs["response_cost"] == pytest.approx(0.0003)
assert mock_increment.call_args.kwargs["response_cost"] == pytest.approx(0.0003)
@pytest.mark.parametrize(
"call_type, expected",
[

87
tests/test_litellm/test_lazy_imports.py
View file

@ -1,8 +1,5 @@
"""Simple tests for lazy import functionality."""
import importlib
import json
import subprocess
import sys
import pytest
@ -10,10 +7,6 @@ import pytest
import litellm
from litellm._lazy_imports import (
_SDK_MODULE_ALIASES,
_SDK_SYMBOLS_IMPORT_MAP,
lazy_import_litellm_submodule,
_lazy_import_sdk_symbols,
COST_CALCULATOR_NAMES,
LITELLM_LOGGING_NAMES,
UTILS_NAMES,
@ -353,83 +346,3 @@ def test_utils_module_lazy_imports():
assert name in utils_globals
_verify_only_requested_name_imported_in_utils(name, UTILS_MODULE_NAMES)
def test_sdk_symbols_lazy_imports():
"""Every symbol previously imported eagerly in litellm/__init__.py resolves to the source module attribute."""
for name, (module_path, attr_name) in _SDK_SYMBOLS_IMPORT_MAP.items():
resolved = getattr(litellm, name)
expected = getattr(importlib.import_module(module_path), attr_name)
assert resolved is expected, f"litellm.{name} is not {module_path}.{attr_name}"
def test_sdk_module_aliases():
"""Module-valued attributes (litellm.anthropic, litellm.httpx, ...) resolve to the aliased modules."""
for name, module_path in _SDK_MODULE_ALIASES.items():
assert getattr(litellm, name) is importlib.import_module(module_path)
def test_litellm_submodule_fallback():
"""litellm.<submodule> attribute access resolves real submodules and returns None for unknown names."""
assert lazy_import_litellm_submodule("budget_manager") is importlib.import_module("litellm.budget_manager")
assert litellm.utils is importlib.import_module("litellm.utils")
assert lazy_import_litellm_submodule("not_a_real_submodule") is None
with pytest.raises(AttributeError):
_ = litellm.not_a_real_attribute
def test_missing_attribute_stays_attribute_error_when_find_spec_lies(monkeypatch):
"""getattr(litellm, name, default) must not leak ModuleNotFoundError when find_spec is patched to always succeed."""
monkeypatch.setattr(importlib.util, "find_spec", lambda name: object())
assert getattr(litellm, "not_a_real_submodule", None) is None
with pytest.raises(AttributeError):
_ = litellm.not_a_real_attribute
def test_proxy_private_submodule_resolves_in_fresh_process():
"""litellm.proxy._types resolves without an eager proxy import (used by documentation checks)."""
code = "import litellm\nprint(litellm.proxy._types.__name__)\n"
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert result.returncode == 0, result.stderr
assert result.stdout.strip() == "litellm.proxy._types"
def test_lazy_instances_are_singletons():
"""Lazily created instances are cached, so repeated access returns the same object."""
assert litellm._key_management_settings is litellm._key_management_settings
assert litellm.vertexAITextEmbeddingConfig is litellm.vertexAITextEmbeddingConfig
from litellm.types.secret_managers.main import KeyManagementSettings
assert isinstance(litellm._key_management_settings, KeyManagementSettings)
def test_star_import_exports_public_api():
"""`from litellm import *` keeps exporting the full public surface despite lazy loading."""
code = (
"from litellm import *\n"
"import litellm\n"
"missing = [n for n in litellm.__all__ if n not in dir()]\n"
"assert not missing, missing[:20]\n"
"assert callable(completion) and callable(Router)\n"
)
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True)
assert result.returncode == 0, result.stderr
@pytest.mark.skipif(sys.platform != "linux", reason="reads /proc for RSS")
def test_import_litellm_stays_lightweight():
"""`import litellm` must not pull in the SDK/proxy heavyweights or blow up RSS (LIT-6607)."""
code = (
"import json, re, sys\n"
"import litellm\n"
"heavy = [m for m in ('litellm.main', 'litellm.utils', 'litellm.router', 'litellm.proxy.proxy_cli',\n"
" 'tiktoken', 'fastapi', 'grpc', 'boto3') if m in sys.modules]\n"
"with open('/proc/self/status') as f:\n"
" rss_kb = int(re.search(r'VmRSS:\\s+(\\d+) kB', f.read()).group(1))\n"
"print(json.dumps({'total': len(sys.modules), 'heavy': heavy, 'rss_mb': rss_kb / 1024}))\n"
)
result = subprocess.run([sys.executable, "-c", code], capture_output=True, text=True, check=True)
stats = json.loads(result.stdout)
assert stats["heavy"] == [], f"heavy modules imported eagerly: {stats['heavy']}"
assert stats["total"] < 800, f"import litellm loaded {stats['total']} modules"
assert stats["rss_mb"] < 75, f"import litellm used {stats['rss_mb']:.1f} MB RSS"

24
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.test.tsx
View file

@ -424,17 +424,14 @@ describe("UsagePage", () => {
// Check that key metrics are displayed
const totalRequestElements = screen.getAllByText("Total Requests");
expect(totalRequestElements.length).toBeGreaterThan(0);
expect(screen.getByText("1,500")).toBeInTheDocument();
const successfulRequestLabelElements = screen.getAllByText("Successful Requests");
expect(successfulRequestLabelElements.length).toBeGreaterThan(0);
// Successful and Failed Requests both read the gateway counter, not the
// spend-derived 1,450 / 50 that the same payload carries for the per-key and
// per-model breakdowns. They must share a source, or the tiles contradict the
// endpoint breakdown chart below them.
await waitFor(() => {
expect(screen.getAllByText("424,242").length).toBeGreaterThan(0);
});
expect(screen.getAllByText("909").length).toBeGreaterThan(0);
expect(screen.getByText("425,151")).toBeInTheDocument();
expect(screen.queryByText("1,500")).not.toBeInTheDocument();
expect(screen.queryByText("1,450")).not.toBeInTheDocument();
});
@ -454,7 +451,7 @@ describe("UsagePage", () => {
renderWithProviders(<UsagePage {...defaultProps} />);
await waitFor(() => {
expect(screen.getAllByText("1,500").length).toBeGreaterThan(0);
expect(screen.getAllByText("75,000").length).toBeGreaterThan(0);
});
await act(async () => {
@ -464,13 +461,13 @@ describe("UsagePage", () => {
await waitFor(() => {
expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(2);
});
expect(screen.queryByText("1,500")).not.toBeInTheDocument();
expect(screen.queryByText("75,000")).not.toBeInTheDocument();
await act(async () => {
releaseSecondFetch();
});
await waitFor(() => {
expect(screen.getAllByText("1,500").length).toBeGreaterThan(0);
expect(screen.getAllByText("75,000").length).toBeGreaterThan(0);
});
});
@ -485,8 +482,10 @@ describe("UsagePage", () => {
await waitFor(() => {
expect(screen.getAllByText("1,450").length).toBeGreaterThan(0);
});
expect(screen.getByText("1,500")).toBeInTheDocument();
expect(screen.queryByText("424,242")).not.toBeInTheDocument();
expect(screen.queryByText("909")).not.toBeInTheDocument();
expect(screen.queryByText("425,151")).not.toBeInTheDocument();
expect(screen.queryByTestId("gateway-requests-by-endpoint")).not.toBeInTheDocument();
});
@ -499,6 +498,7 @@ describe("UsagePage", () => {
expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalled();
});
expect(mockGatewayDailyActivityCall).not.toHaveBeenCalled();
expect(screen.getByText("1,500")).toBeInTheDocument();
expect(screen.queryByText("424,242")).not.toBeInTheDocument();
expect(screen.queryByTestId("gateway-requests-by-endpoint")).not.toBeInTheDocument();
});
@ -1045,7 +1045,7 @@ describe("UsagePage", () => {
});
// Should still render the data from the paginated fallback, which lands a render after the call
expect(await screen.findByText("1,500")).toBeInTheDocument();
expect(await screen.findByText("75,000")).toBeInTheDocument();
});
it("should stop showing the previous range's paginated pages while a new range is in flight", async () => {
@ -1069,7 +1069,7 @@ describe("UsagePage", () => {
renderWithProviders(<UsagePage {...defaultProps} />);
await waitFor(() => {
expect(screen.getAllByText("1,500").length).toBeGreaterThan(0);
expect(screen.getAllByText("75,000").length).toBeGreaterThan(0);
});
await act(async () => {
@ -1079,13 +1079,13 @@ describe("UsagePage", () => {
await waitFor(() => {
expect(mockUserDailyActivityAggregatedCall).toHaveBeenCalledTimes(2);
});
expect(screen.queryByText("1,500")).not.toBeInTheDocument();
expect(screen.queryByText("75,000")).not.toBeInTheDocument();
await act(async () => {
releaseSecondAggregated();
});
await waitFor(() => {
expect(screen.getAllByText("1,500").length).toBeGreaterThan(0);
expect(screen.getAllByText("75,000").length).toBeGreaterThan(0);
});
});

5
ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsagePageView.tsx
View file

@ -573,7 +573,10 @@ const UsagePage: React.FC<UsagePageProps> = ({ teams, organizations }) => {
<CardContent>
<h3 className="text-lg font-medium text-foreground">Total Requests</h3>
<p className="text-2xl font-bold mt-2">
{userSpendData.metadata?.total_api_requests?.toLocaleString() || 0}
{(gatewayActivity
? gatewayActivity.total_successful_requests + gatewayActivity.total_failed_requests
: userSpendData.metadata?.total_api_requests
)?.toLocaleString() || 0}
</p>
</CardContent>
</ShadcnCard>

29
ui/litellm-dashboard/src/components/AIHub/ModelHubTable.tsx
View file

@ -32,7 +32,6 @@ import { Dialog, DialogContent, DialogHeader, DialogTitle } from "@/components/u
import { InputGroup, InputGroupAddon, InputGroupButton, InputGroupInput } from "@/components/ui/input-group";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs";
import { Copy, Inbox, Search as SearchIcon, X } from "lucide-react";
import { useRouter } from "next/navigation";
import React, { useCallback, useEffect, useMemo, useState } from "react";
import { Prism as SyntaxHighlighter } from "react-syntax-highlighter";
import { prism } from "react-syntax-highlighter/dist/esm/styles/prism";
@ -72,7 +71,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const [modelHubData, setModelHubData] = useState<ModelHubData[] | null>(null);
const [loading, setLoading] = useState<boolean>(true);
const [isModalVisible, setIsModalVisible] = useState(false);
const [isPublicPageModalVisible, setIsPublicPageModalVisible] = useState(false);
const [selectedModel, setSelectedModel] = useState<null | ModelHubData>(null);
const [filteredData, setFilteredData] = useState<ModelHubData[]>([]);
const [isMakePublicModalVisible, setIsMakePublicModalVisible] = useState(false);
@ -93,7 +91,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const [skillHubData, setSkillHubData] = useState<Plugin[]>([]);
const [skillLoading, setSkillLoading] = useState<boolean>(false);
const [isMakeSkillPublicModalVisible, setIsMakeSkillPublicModalVisible] = useState(false);
const router = useRouter();
const { data: uiSettings, isLoading: isUISettingsLoading } = useUISettings();
// Check authentication requirement for public AI Hub
@ -256,10 +253,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
setIsMcpModalVisible(true);
}, []);
const goToPublicModelPage = () => {
router.replace(`/model_hub_table?key=${accessToken}`);
};
const handleMakePublicPage = () => {
if (!accessToken) {
return;
@ -289,7 +282,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const handleOk = () => {
setIsModalVisible(false);
setIsPublicPageModalVisible(false);
setSelectedModel(null);
setIsAgentModalVisible(false);
setSelectedAgent(null);
@ -299,7 +291,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
const handleCancel = () => {
setIsModalVisible(false);
setIsPublicPageModalVisible(false);
setSelectedModel(null);
setIsAgentModalVisible(false);
setSelectedAgent(null);
@ -639,26 +630,6 @@ const ModelHubTable: React.FC<ModelHubTableProps> = ({ accessToken, publicPage,
</Card>
)}
{/* Public Page Modal */}
<Dialog open={isPublicPageModalVisible} onOpenChange={(open) => !open && handleCancel()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[600px]">
<DialogHeader>
<DialogTitle>{"Public Model Hub"}</DialogTitle>
</DialogHeader>
<div className="pt-5 pb-5">
<div className="flex justify-between mb-4">
<p className="text-base mr-2">Shareable Link:</p>
<p className="max-w-sm ml-2 bg-border pr-2 pl-2 pt-1 pb-1 text-center rounded-sm">
{`${getProxyBaseUrl()}/ui/model_hub_table`}
</p>
</div>
<div className="flex justify-end">
<Button onClick={goToPublicModelPage}>See Page</Button>
</div>
</div>
</DialogContent>
</Dialog>
{/* Model Details Modal */}
<Dialog open={isModalVisible} onOpenChange={(open) => !open && handleCancel()}>
<DialogContent className="max-h-[calc(100dvh-2rem)] overflow-y-auto sm:max-w-[1000px]">