Merge remote-tracking branch 'origin/main' into HEAD

# Conflicts:
#	litellm/proxy/db/db_spend_update_writer.py
This commit is contained in:
mateo-berri 2026-10-23 19:41:24 -07:00
commit f7f6d76e62
241 changed files with 14810 additions and 5637 deletions

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

View file

@ -425,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

View file

@ -121,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \

View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -146,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()

View file

@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
runs, the middleware stack is frozen and middleware added later never runs, so
a late call is skipped without registering anything.
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
try:
if not is_otel_v2_enabled():
return
if app.middleware_stack is not None:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

View file

@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
import base64
import json
import logging
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
_logger: Final = logging.getLogger(__name__)
JSON_CONTENT_TYPE: Final = "application/json"
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
def __init__(
self,
endpoint: str | None,
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
headers: Mapping[str, str],
certificate_file: str | None = None,
session: "requests.Session | None" = None,
*,
max_request_size: int | None = None,
) -> None:
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
super().__init__(
endpoint=endpoint,
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
certificate_file=certificate_file,
session=session,
max_request_size=max_request_size,
)
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
return encode_spans_json(spans)
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
if self._shutdown:
_logger.warning("Exporter already shutdown, ignoring batch")
return SpanExportResult.FAILURE
with self._metrics.export_operation(len(spans)) as result:
try:
serialized_data: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
_logger.error("Failed to encode span batch: %s", error)
result.error = error
return SpanExportResult.FAILURE
if 0 < self._max_request_size < len(serialized_data):
result.error = RequestPayloadTooLargeError(
f"Serialized span request size {len(serialized_data)} bytes "
f"exceeds max_request_size {self._max_request_size} bytes."
)
_logger.warning("Dropping span batch: %s", result.error)
return SpanExportResult.FAILURE
export_result: Final = self._client.export(serialized_data)
if not export_result.success:
result.error = export_result.error
result.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
if export_result.status_code is not None
else None
)
return SpanExportResult.FAILURE
return SpanExportResult.SUCCESS

View file

@ -76,6 +76,7 @@ def agentops_preset(
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
import requests
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter,
)
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
"""
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
super().__init__(endpoint=endpoint)
self._agentops_session = requests.Session()
super().__init__(endpoint=endpoint, session=self._agentops_session)
self._agentops_api_key = api_key
self._auth_resolved = False
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
try:
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
if token:
# ``_session`` is the requests.Session the base exporter
# POSTs through; updating its Authorization header is how the
# minted JWT reaches every subsequent export.
self._session.headers["Authorization"] = f"Bearer {token}"
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
except Exception as e:
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)

View file

@ -19,6 +19,7 @@ import httpx
from pydantic import ConfigDict
from litellm import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.types.llms.base import LiteLLMBaseModel
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
@ -129,7 +130,7 @@ class GetBlogPosts:
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
Fetches from ``url`` otherwise, falling back to local backup on failure.
"""
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
now: Final = time.time()

View file

@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
if dynamic_api_key is None and api_key is not None:
dynamic_api_key = api_key
return model, custom_llm_provider, dynamic_api_key, api_base
def get_provider_affinity_header(api_base: str | None) -> str | None:
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)

View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
@ -458,7 +459,7 @@ async def refetch_model_cost_map(
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
@ -742,7 +743,7 @@ def get_model_cost_map(
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True

View file

@ -0,0 +1,7 @@
import os
def is_offline_mode() -> bool:
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"

View file

@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
return getattr(value, key, None)
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
if header is None:
return None
return default_header
if not isinstance(header, str):
raise TypeError("provider_affinity_header must be a string")
return validate_provider_affinity_header_name(header)
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
def add_provider_affinity_header(
headers: Mapping[str, object], litellm_params: object | None
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
header_name: Final = _get_provider_affinity_header_name(litellm_params)
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
return dict(headers)

View file

@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
else:
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
# model arrives without the "aws_polly/" prefix, e.g. "standard"
engine: Final = self._extract_engine_from_model(model)
mapped_params["engine"] = engine
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
"""
Extract engine from model name.
speech() strips the "aws_polly/" prefix before the config is called,
so the bare engine name is what normally arrives here.
Examples:
- aws_polly/neural -> neural
- aws_polly/standard -> standard
- standard -> standard
- aws_polly/long-form -> long-form
- aws_polly -> neural (default)
"""
if "/" in model:
parts: Final = model.split("/")
if len(parts) >= 2:
engine: Final = parts[1].lower()
if engine in self.VALID_ENGINES:
return engine
engine: Final = model.rsplit("/", 1)[-1].lower()
if engine in self.VALID_ENGINES:
return engine
return self.DEFAULT_ENGINE
def validate_environment(

View file

@ -3,6 +3,7 @@ from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from urllib.parse import urlsplit
import httpx
from openai.types.responses import ResponseReasoningItem
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
parsed_base: Final = urlsplit(resolved_api_base or "")
path: Final = parsed_base.path.rstrip("/")
if path.endswith(("/openai/v1", "/openai/v1/responses")):
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
return BaseAzureLLM.get_base_azure_url(
api_base=parsed_base._replace(path=responses_path).geturl(),
litellm_params=responses_params,
route="/openai/v1/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
return BaseAzureLLM.get_base_azure_url(
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
litellm_params=responses_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

View file

@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
audio_or_image_in_message_content,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(messages)
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:

View file

@ -27,6 +27,7 @@ from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
cached: Final = huggingface_hub.try_to_load_from_cache(
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
)
if not isinstance(cached, str):
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
return cached
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)

View file

@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.realtime_errors import (
close_after_upstream_handshake_refusal,
realtime_error_event,
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -2457,13 +2460,13 @@ class BaseLLMHTTPHandler:
# Also check for extra_headers in kwargs (from config or direct calls)
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
# Merge all header sources: forwarded < extra_headers < provider_specific
merged_headers: Final = {}
if forwarded_headers:
merged_headers.update(forwarded_headers)
if extra_headers_from_kwargs:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
{
**(forwarded_headers or {}),
**(extra_headers_from_kwargs or {}),
**(provider_specific_headers or {}),
}
)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
@ -2474,11 +2477,19 @@ class BaseLLMHTTPHandler:
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
(
headers,
api_base,
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
headers=merged_headers or {},
headers=(
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
if default_affinity_header is not None
else dict(merged_headers)
),
model=model,
messages=messages,
optional_params=request_params,

View file

@ -0,0 +1,10 @@
import re
def is_opencode_api_base(api_base: str | None) -> bool:
return bool(
api_base
and re.fullmatch(
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
)
)

View file

@ -0,0 +1,6 @@
from litellm.llms.opencode.common_utils import is_opencode_api_base
class OpenCodeProviderAdapter:
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
return "x-opencode-session" if is_opencode_api_base(api_base) else None

View file

@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
with_control_options,
)
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_provider_affinity_header,
invalid_provider_routing_hints_message,
)
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (
CompletionDispatchContext,
_CompletionDispatchResult,
@ -169,7 +172,6 @@ from litellm.utils import (
ProviderConfigManager,
TextChoices,
TextCompletionResponse,
TextCompletionStreamWrapper,
TranscriptionResponse,
Usage,
add_provider_specific_params_to_optional_params,
@ -5996,19 +5998,28 @@ def completion(
},
)
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
if litellm_params.get("provider_affinity_header") is not None:
affinity_api_base: Final = (
str(client.base_url)
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
else get_openai_credentials(api_base=api_base).api_base
if custom_llm_provider == "openai"
else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
try:
headers = add_provider_affinity_header(
headers=headers or litellm.headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params["provider_affinity_header"],
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
except ValueError as affinity_error:
raise litellm.BadRequestError(

View file

@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.MODIFY_PARAMS: detectors.modify_params,

View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -42337,6 +42433,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -48499,6 +48723,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -52906,12 +53141,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53163,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -73330,6 +73569,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,

Binary file not shown.

After

Width:  |  Height:  |  Size: 685 KiB

View file

@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
path_prefixes=("/vantage",),
),
LazyFeature(
name="usage_ai",
module_path="litellm.proxy.management_endpoints.usage_endpoints",
path_prefixes=("/usage/ai",),
),
LazyFeature(
name="prompts",
module_path="litellm.proxy.prompts.prompt_endpoints",

View file

@ -57898,163 +57898,6 @@
}
}
},
"usage_ai": {
"components": {
"schemas": {
"ChatMessage": {
"properties": {
"content": {
"title": "Content",
"type": "string"
},
"role": {
"enum": [
"user",
"assistant"
],
"title": "Role",
"type": "string"
}
},
"required": [
"role",
"content"
],
"title": "ChatMessage",
"type": "object"
},
"HTTPValidationError": {
"properties": {
"detail": {
"items": {
"$ref": "#/components/schemas/ValidationError"
},
"title": "Detail",
"type": "array"
}
},
"title": "HTTPValidationError",
"type": "object"
},
"UsageAIChatRequest": {
"properties": {
"messages": {
"description": "Chat messages (user/assistant history)",
"items": {
"$ref": "#/components/schemas/ChatMessage"
},
"title": "Messages",
"type": "array"
},
"model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Model to use for AI chat",
"title": "Model"
}
},
"required": [
"messages"
],
"title": "UsageAIChatRequest",
"type": "object"
},
"ValidationError": {
"properties": {
"ctx": {
"title": "Context",
"type": "object"
},
"input": {
"title": "Input"
},
"loc": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"title": "Location",
"type": "array"
},
"msg": {
"title": "Message",
"type": "string"
},
"type": {
"title": "Error Type",
"type": "string"
}
},
"required": [
"loc",
"msg",
"type"
],
"title": "ValidationError",
"type": "object"
}
}
},
"paths": {
"/usage/ai/chat": {
"post": {
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
"operationId": "usage_ai_chat_usage_ai_chat_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UsageAIChatRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Usage Ai Chat",
"tags": [
"usage_ai"
]
}
}
}
},
"vantage": {
"components": {
"schemas": {

View file

@ -21,6 +21,7 @@ from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm._version import version
from litellm.constants import HIBP_RANGE_API_BASE
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
return general_settings.get("password_policy_check_breached_passwords", True) is not False
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
async def is_password_breached(

View file

@ -12,7 +12,6 @@ import json
import os
import random
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
@ -91,6 +90,7 @@ from litellm.proxy.spend_tracking.savings import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
@ -329,6 +329,38 @@ def _merged_spend_restore(first: _SpendRestore, second: _SpendRestore) -> _Spend
return {**first, **second}
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
spend_log_error(
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
step_name,
payload.get("request_id"),
payload.get("api_key"),
exc=exc,
)
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
raw: Final = payload.get("metadata")
if not isinstance(raw, str):
return _EMPTY_SPEND_LOG_METADATA
try:
parsed: Final[SpendLogsMetadata] = json.loads(raw)
except json.JSONDecodeError as e:
spend_log_error(
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
"metadata-derived fields. request_id=%s api_key=%s",
payload.get("request_id"),
payload.get("api_key"),
exc=e,
)
return _EMPTY_SPEND_LOG_METADATA
return parsed
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
@ -663,8 +695,8 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
# Single task replaces 11 create_task() calls
asyncio.create_task(
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
user_id=user_id,
@ -677,7 +709,8 @@ class DBSpendUpdateWriter:
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(kwargs),
)
),
task_name="spend rollups",
)
self._enqueue_tool_registry_upsert(
@ -1102,91 +1135,52 @@ class DBSpendUpdateWriter:
project_id: str | None = None,
):
"""
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
Each helper is wrapped in try/except so one failure doesn't prevent the others.
The deepcopy runs here, off the awaited request path, so the daily spend
helpers get a payload isolated from the spend-log queue entry and the caller.
Runs every spend-update helper for one request. Each helper goes through
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
"""
payload_copy: Final = copy.deepcopy(payload)
request_tags: Final = payload_copy.get("request_tags")
try:
await self._update_user_db(
await self._run_spend_update_step(
"_update_user_db",
self._update_user_db(
response_cost=response_cost,
user_id=user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
end_user_id=end_user_id,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_user_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_key_db(
response_cost=response_cost,
hashed_token=hashed_token,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_key_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_team_db(
response_cost=response_cost,
team_id=team_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_team_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_org_db(
response_cost=response_cost,
org_id=org_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_org_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_project_db(
response_cost=response_cost,
project_id=project_id,
prisma_client=prisma_client,
)
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _update_project_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_tag_db(
response_cost=response_cost,
request_tags=request_tags,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_tag_db failed: %s",
traceback.format_exc(),
)
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_key_db",
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_team_db",
self._update_team_db(
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_org_db",
self._update_org_db(
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_project_db",
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_tag_db",
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
payload=payload_copy,
)
await self._update_model_access_group_db(
response_cost=response_cost,
request_model_access_groups=request_model_access_groups,
@ -1194,96 +1188,64 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
router=get_llm_router(),
)
_agent_id_for_spend: Final = payload_copy.get("agent_id")
try:
await self._update_agent_db(
response_cost=response_cost,
agent_id=_agent_id_for_spend,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_agent_db failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_team_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
traceback.format_exc(),
)
await self._run_spend_update_step(
"_update_agent_db",
self._update_agent_db(
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_user_transaction",
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_end_user_transaction",
self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_agent_transaction",
self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_team_transaction",
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
try:
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _record_request_error failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy,
org_id=org_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_tag_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
traceback.format_exc(),
)
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_org_transaction",
self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_tag_transaction",
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
@staticmethod
async def _run_spend_update_step(
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
) -> None:
try:
await step
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
async def _update_key_db(
self,
response_cost: float | None,
@ -2857,7 +2819,7 @@ class DBSpendUpdateWriter:
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
if not date:
return
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
metadata: Final = _parse_spend_log_metadata(payload)
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
request_error_accumulator.record(
payload=payload,
@ -2908,7 +2870,7 @@ class DBSpendUpdateWriter:
# spend, token and per-entity columns written here stay either way.
request_status: Final = prisma_client.get_request_status(payload)
verbose_proxy_logger.debug("Logged request status: %s", request_status)
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
_metadata: Final = _parse_spend_log_metadata(payload)
usage_obj: Final = _metadata.get("usage_object", {}) or {}
if isinstance(payload["startTime"], datetime):
start_time: Final = payload["startTime"].isoformat()

View file

@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
"/v1/messages",
"responses",
"files",
"interactions",
)
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
LlmProviders.VERTEX_AI.value,
)
)
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
)
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
),
ProviderSpecificHeader(
custom_llm_provider=LlmProviders.DATABRICKS.value,
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
extra_headers=anthropic_api_headers,
native_anthropic_messages_only=True,
),

View file

@ -2093,7 +2093,9 @@ async def get_user_key_counts(
return result
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
order_by: Final[dict[str, str]] = {}
if sort_by is None:
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
"spend",
"user_alias",
"user_role",
"max_budget",
"updated_at",
"sso_user_id",
"status",
"key_count",
]
if sort_by not in valid_columns:
raise HTTPException(
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
if sort_by in {"status", "key_count"}:
return None
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
order_by[sort_by] = sort_order.lower()
return order_by
@ -2338,16 +2350,40 @@ async def get_users(
# Build order_by conditions
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
where=where,
skip=skip,
take=page_size,
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
computed_sort: Final = sort_by in ("status", "key_count")
sorted_ids: Final = (
await sorted_user_ids(
prisma_client,
sort_by="status" if sort_by == "status" else "key_count",
descending=sort_order.lower() == "desc",
skip=skip,
take=page_size,
role=role,
user_ids=user_ids,
user_email=user_email,
team=team,
sso_user_ids=sso_user_ids,
organization_ids=organization_ids,
search=search,
)
if computed_sort
else ()
)
page_users: Final = await UserRepository(prisma_client).table.find_many(
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"},
)
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
# Get total count of user rows
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)

View file

@ -7415,20 +7415,26 @@ async def key_aliases(
)
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
order_by: Final[dict[str, str]] = {}
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
if sort_by is None:
return None
# Validate sort_by is a valid column
valid_columns: Final = [
valid_columns: Final = (
"spend",
"max_budget",
"created_at",
"updated_at",
"token",
"key_alias",
]
"expires",
"last_active",
"budget_reset_at",
"total_spend",
"tpm_limit",
"rpm_limit",
)
if sort_by not in valid_columns:
raise HTTPException(
status_code=400,
@ -7442,9 +7448,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
order_by[sort_by] = sort_order.lower()
return order_by
direction: Final = sort_order.lower()
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
if sort_by not in (*nullable_sort_columns, "total_spend"):
return {sort_by: direction}
return [
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
{"token": "asc"},
]
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
@ -7686,7 +7697,7 @@ async def list_key_helper(
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)

View file

@ -3,6 +3,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Final
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰0𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰"
_DECIMAL_DIGITS: Final = "".join(
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
)
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
_BUDGET_DURATION_SECONDS: Final = f"""(
SELECT EXTRACT(EPOCH FROM (
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
(parts[1] || ' ' || CASE parts[2]
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
)) FROM regexp_match(
CASE lower(trim(budget_duration))
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
'^([0-9]+)(mo|[smhdw])'
) AS duration(parts)
)"""
@dataclass(frozen=True, slots=True)
class PrismaBudgetListExecutor:
"""The database half of the budgets list. Every caller-supplied value is bound to a
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
clauses, params = where_sql(plan.where)
ordering: Final = ", ".join(
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
else order_by_sql((key,))
for key in plan.order
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
sql: Final = (
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
+ (f" WHERE {clauses}" if clauses else "")
+ f" ORDER BY {order_by_sql(plan.order)}"
+ f" ORDER BY {ordering}"
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
# budget_duration is deliberately absent from `sortable`: the column holds strings
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
sortable=frozenset(
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,8 +187,9 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`

View file

@ -28,6 +28,7 @@ from litellm.constants import (
MAX_COMPETITOR_NAMES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
@ -657,7 +658,7 @@ async def get_policy_templates(
Fetches from GitHub with automatic fallback to local backup on failure.
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
"""
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
"true",
"1",
"yes",

View file

@ -6117,13 +6117,52 @@ async def list_team_v2(
"total_pages": 0,
}
# Build order_by conditions
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
order_by = None
if sort_by and sort_by in valid_sort_columns:
if sort_order.lower() not in ["asc", "desc"]:
sort_order = "asc"
order_by = {sort_by: sort_order.lower()}
from litellm.proxy.management_helpers.team_list_sorting import (
COMPUTED_TEAM_SORTS,
UnsupportedTeamFilter,
sorted_team_ids,
)
valid_sort_columns: Final = (
"team_id",
"team_alias",
"created_at",
"spend",
"max_budget",
"tpm_limit",
"rpm_limit",
"updated_at",
)
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
order_by: Final = (
(
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
{"team_id": "asc"},
)
if nullable_sort
else ({sort_by: sort_direction}, {"team_id": "asc"})
if sort_by in ("spend", "updated_at")
else {sort_by: sort_direction}
if sort_by in valid_sort_columns
else None
)
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
sorted_ids: Final = (
await sorted_team_ids(
prisma_client,
where=where_conditions,
sort_by=sort_by or "",
descending=sort_direction == "desc",
skip=skip,
take=page_size,
)
if computed_sort
else ()
)
if isinstance(sorted_ids, UnsupportedTeamFilter):
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
# Get teams with pagination
if use_deleted_table:
@ -6138,26 +6177,32 @@ async def list_team_v2(
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
else:
teams = await _team_db(prisma_client).find_many(
where=where_conditions,
skip=skip,
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"}, # Default sort
order=order_by if order_by else {"created_at": "desc"},
include=_INCLUDE_MODEL_TABLE,
)
# Get total count for pagination
total_count = await _team_db(prisma_client).count(where=where_conditions)
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
# Calculate total pages
total_pages: Final = -(-total_count // page_size) # Ceiling division
# Aggregate virtual-key counts per team for the current page. The deleted
# table does not carry keys_count, so it is skipped.
keys_count_by_team: dict[str, int] = {}
if not use_deleted_table:
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
keys_count_by_team: Final = (
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
)
# Convert Prisma models to response models with members_count and keys_count
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
team_list: Final = _convert_teams_to_response_models(
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
)
# Resolve resources inherited from access groups (single batch query)
if not use_deleted_table:

View file

@ -152,7 +152,7 @@ from litellm.proxy.utils import (
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
@ -1085,25 +1085,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
)
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
if premium_user is True:
return
if prisma_client is None:
raise ProxyException(
message=CommonProxyErrors.db_not_connected_error.value,
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
if billable_users and billable_users > 5:
raise ProxyException(
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
class _SSOUserAdmissionError(ProxyException):
pass
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
@ -1123,8 +1106,6 @@ async def google_login(
from litellm.proxy.proxy_server import (
cli_sso_session_cache,
general_settings,
premium_user,
prisma_client,
saml_sso_cache,
user_custom_ui_sso_sign_in_handler,
)
@ -1139,15 +1120,6 @@ async def google_login(
if admin_ui_is_disabled and not is_cli_sso_login:
return admin_ui_disabled()
####### Check if user is a Enterprise / Premium User #######
if (
microsoft_client_id is not None
or google_client_id is not None
or generic_client_id is not None
or SAMLAuthHandler.is_saml_configured()
):
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
####### Detect DB + MASTER KEY in .env #######
missing_env_vars: Final = show_missing_vars_in_env()
if missing_env_vars is not None:
@ -1972,6 +1944,8 @@ async def get_user_info_from_db(
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
@ -2337,7 +2311,6 @@ async def saml_callback(request: Request):
general_settings,
jwt_handler,
master_key,
premium_user,
prisma_client,
saml_sso_cache,
)
@ -2361,8 +2334,6 @@ async def saml_callback(request: Request):
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
relay_state: Final = post_data.get("RelayState")
cp_return_to: Final[str | None] = (
@ -2826,12 +2797,20 @@ async def insert_sso_user(
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
response: Final = await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
from litellm.proxy.proxy_server import premium_user
return response
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
try:
return await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
except ProxyException as error:
if premium_user is True:
raise
raise _SSOUserAdmissionError(
message=error.message, type=error.type, param=error.param, code=error.code
) from error
DefaultTeamJoin: TypeAlias = Callable[
@ -3572,6 +3551,8 @@ class SSOAuthenticationHandler:
prisma_client=prisma_client,
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if (
user_info is None

View file

@ -1,9 +0,0 @@
"""
Usage endpoints package.
Re-exports the router from endpoints module.
"""
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
router,
)

View file

@ -1,627 +0,0 @@
"""
AI Usage Chat - uses LLM tool calling to answer questions about
usage/spend data by querying the aggregated daily activity endpoints.
"""
import json
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import date
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
from fastapi import HTTPException
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.types.utils import ChatCompletionMessageToolCall
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
USAGE_AI_TEMPERATURE: Final = 0.2
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
ENTITY_FIELD_USER: Final = "user_id"
ENTITY_FIELD_TEAM: Final = "team_id"
ENTITY_FIELD_TAG: Final = "tag"
PAGINATED_PAGE_SIZE: Final = 200
MAX_CHAT_MESSAGES: Final = 20
TOP_N_MODELS: Final = 15
TOP_N_PROVIDERS: Final = 10
TOP_N_KEYS: Final = 10
# ---------------------------------------------------------------------------
# Types
# ---------------------------------------------------------------------------
class SSEStatusEvent(TypedDict):
type: Literal["status"]
message: str
class SSEToolCallEvent(TypedDict, total=False):
type: Literal["tool_call"]
tool_name: str
tool_label: str
arguments: dict[str, str]
status: Literal["running", "complete", "error"]
error: str
class SSEChunkEvent(TypedDict):
type: Literal["chunk"]
content: str
class SSEDoneEvent(TypedDict):
type: Literal["done"]
class SSEErrorEvent(TypedDict):
type: Literal["error"]
message: str
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
class _EntityEntry(TypedDict, total=False):
metrics: ReadOnly[Mapping[str, float]]
metadata: ReadOnly[Mapping[str, str]]
class _DayDump(TypedDict, total=False):
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
class _EntityTotal(NamedTuple):
"""Running per-entity totals accumulated while summarising a usage dump."""
alias: str
spend: float
requests: float
tokens: float
class _UsageDump(Protocol):
@overload
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
@overload
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
class _ToolFunctionDef(TypedDict):
name: ReadOnly[str]
description: ReadOnly[str]
parameters: ReadOnly[Mapping[str, object]]
class _ToolDef(TypedDict):
type: ReadOnly[str]
function: ReadOnly[_ToolFunctionDef]
class ToolHandler(TypedDict):
fetch: Callable[..., Awaitable[_UsageDump]]
summarise: Callable[[_UsageDump], str]
label: str
# ---------------------------------------------------------------------------
# Tool definitions (OpenAI function-calling schema)
# ---------------------------------------------------------------------------
_DATE_PARAMS: Final = {
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
}
_TOOL_USAGE: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_usage_data",
"description": (
"Fetch aggregated global usage/spend data. Returns daily spend, "
"token counts, request counts, and breakdowns by model, provider, "
"and API key. Use for overall spend, top models, top providers."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"user_id": {
"type": "string",
"description": "Optional user ID filter. Omit for global view.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TEAM: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_team_usage_data",
"description": (
"Fetch usage/spend data broken down by team. Use for questions "
"like 'which team spends the most' or 'show me team X usage'."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"team_ids": {
"type": "string",
"description": "Optional comma-separated team IDs. Omit for all teams.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TAG: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_tag_usage_data",
"description": (
"Fetch usage/spend data broken down by tag. Tags are labels "
"attached to requests (features, environments, credentials)."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"tags": {
"type": "string",
"description": "Optional comma-separated tag names. Omit for all tags.",
},
},
"required": ["start_date", "end_date"],
},
},
}
TOOLS_BASE: Final = [_TOOL_USAGE]
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
"""Return the tool list appropriate for the user's role."""
return TOOLS_ADMIN if is_admin else TOOLS_BASE
_SYSTEM_PROMPT_BASE: Final = (
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
"You help users understand their LLM API spend and usage data.\n\n"
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
"You may call multiple tools if the question spans different dimensions.\n\n"
"Guidelines:\n"
"- Be concise and specific. Use exact numbers from the data.\n"
"- Format costs as dollar amounts (e.g. $12.34).\n"
"- When comparing entities, show a ranked list.\n"
"- If data is empty or no results found, say so clearly.\n"
"- Do not hallucinate data — only use what the tools return.\n"
"- Today's date will be provided below. Use it to interpret relative dates "
"like 'this week', 'this month', 'last 7 days', etc."
)
_TOOL_DESCRIPTIONS_ADMIN: Final = (
"You have access to these tools:\n"
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
"- `get_team_usage_data`: Team-level usage breakdown\n"
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
)
_TOOL_DESCRIPTIONS_BASE: Final = (
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
)
def _build_system_prompt(is_admin: bool) -> str:
"""Build role-appropriate system prompt with today's date."""
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
# keep a public reference for test assertions
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
# ---------------------------------------------------------------------------
# Data fetchers
# ---------------------------------------------------------------------------
def _parse_csv_ids(raw: str | None) -> list[str] | None:
if not raw:
return None
return [t.strip() for t in raw.split(",") if t.strip()]
async def _query_activity(
table_name: str,
entity_id_field: str,
entity_id: str | list[str] | None,
start_date: str,
end_date: str,
*,
use_aggregated: bool = False,
) -> SpendAnalyticsPaginatedResponse:
"""Shared helper that calls the daily activity query layer."""
from litellm.proxy.management_endpoints.common_daily_activity import (
daily_activity_repository,
daily_activity_scope,
get_daily_activity,
get_daily_activity_aggregated,
)
from litellm.proxy.proxy_server import prisma_client
if use_aggregated:
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
repository: Final = daily_activity_repository(prisma_client)
scope: Final = daily_activity_scope(
table_name,
entity_id_field,
entity_id,
None,
None,
start_date,
end_date,
None,
None,
)
return await get_daily_activity_aggregated(
repository,
scope,
)
return await get_daily_activity(
prisma_client=prisma_client,
table_name=table_name,
entity_id_field=entity_id_field,
entity_id=entity_id,
entity_metadata_field=None,
start_date=start_date,
end_date=end_date,
model=None,
api_key=None,
page=1,
page_size=PAGINATED_PAGE_SIZE,
)
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_USER_SPEND,
ENTITY_FIELD_USER,
user_id,
start_date,
end_date,
use_aggregated=True,
)
return resp.model_dump(mode="json")
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TEAM_SPEND,
ENTITY_FIELD_TEAM,
_parse_csv_ids(team_ids),
start_date,
end_date,
)
return resp.model_dump(mode="json")
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TAG_SPEND,
ENTITY_FIELD_TAG,
_parse_csv_ids(tags),
start_date,
end_date,
)
return resp.model_dump(mode="json")
# ---------------------------------------------------------------------------
# Summarisers — convert raw JSON to concise text the LLM can reason over
# ---------------------------------------------------------------------------
def _accumulate_breakdown(
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
) -> dict[str, dict[str, float]]:
"""Aggregate a single breakdown dimension across days."""
totals: Final[dict[str, dict[str, float]]] = {}
for day in results:
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
if key not in totals:
totals[key] = {f: 0.0 for f in fields}
m = entry.get("metrics", {})
for f in fields:
totals[key][f] += m.get(f, 0)
return totals
def _ranked_lines(
totals: dict[str, dict[str, float]],
fmt: Callable[[str, dict[str, float]], str],
limit: int,
) -> list[str]:
"""Sort by spend descending, format each entry, and truncate."""
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
def _summarise_usage_data(data: _UsageDump) -> str:
meta: Final = data.get("metadata", {})
results: Final = data.get("results", [])
header: Final = (
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
f"Successful: {meta.get('total_successful_requests', 0)} | "
f"Failed: {meta.get('total_failed_requests', 0)}\n"
f"Total Tokens: {meta.get('total_tokens', 0)}"
)
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
model_lines: Final = _ranked_lines(
models,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
TOP_N_MODELS,
)
provider_lines: Final = _ranked_lines(
providers,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
TOP_N_PROVIDERS,
)
sections = [header, ""]
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
return "\n".join(sections)
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
"""Summarise team/tag entity usage data."""
results: Final = data.get("results", [])
if not results:
return f"No {entity_label} usage data found for the given date range."
totals: Final[dict[str, _EntityTotal]] = {}
for day in results:
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
previous = totals.get(eid)
m = entry.get("metrics", {})
totals[eid] = _EntityTotal(
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
)
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
label = d.alias if d.alias != eid else eid
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Tool dispatch registry
# ---------------------------------------------------------------------------
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
"get_usage_data": ToolHandler(
fetch=_fetch_usage_data,
summarise=_summarise_usage_data,
label="global usage data",
),
"get_team_usage_data": ToolHandler(
fetch=_fetch_team_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Team"),
label="team usage data",
),
"get_tag_usage_data": ToolHandler(
fetch=_fetch_tag_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Tag"),
label="tag usage data",
),
}
# ---------------------------------------------------------------------------
# SSE streaming
# ---------------------------------------------------------------------------
def _sse(event: SSEEvent) -> str:
return f"data: {json.dumps(event)}\n\n"
def _resolve_fetch_kwargs(
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> dict[str, str]:
"""Build keyword arguments for a tool's fetch function."""
start_date: Final = fn_args.get("start_date", "")
end_date: Final = fn_args.get("end_date", "")
if not start_date or not end_date:
raise ValueError("Missing required start_date or end_date from tool arguments")
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
if fn_name == "get_usage_data":
if not is_admin:
if user_id is None:
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
# should have already rejected this. If we ever reach here it means
# a future caller invoked the helper without scoping — fail loudly
# rather than issuing an unfiltered global query.
raise ValueError(
"Non-admin caller has user_id=None; refusing to issue an "
"unscoped query. Endpoint-level guard missing."
)
kwargs["user_id"] = user_id
elif fn_args.get("user_id"):
kwargs["user_id"] = fn_args["user_id"]
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
kwargs["team_ids"] = fn_args["team_ids"]
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
kwargs["tags"] = fn_args["tags"]
return kwargs
async def _execute_tool_call(
handler: ToolHandler,
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> str:
"""Run a single tool and return the summarised result text, or the reason the query
layer rejected its dates so the model can correct them."""
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
try:
raw_data: Final = await handler["fetch"](**kwargs)
except HTTPException as e:
reason: Final = _invalid_date_reason(e)
if reason is None:
raise
return f"Invalid date range: {reason}"
return handler["summarise"](raw_data)
def _invalid_date_reason(error: HTTPException) -> str | None:
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
match detail:
case {"error": str() as reason} if error.status_code == 400:
return reason
case _:
return None
async def _process_tool_call(
tc: ChatCompletionMessageToolCall,
chat_messages: list[Mapping[str, object]],
user_id: str | None,
is_admin: bool,
) -> AsyncIterator[str]:
"""Execute a single tool call, yielding SSE events for status."""
fn_name: Final = tc.function.name
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
if fn_name is None or fn_name not in allowed_names or not handler:
chat_messages.append(
{
"role": "tool",
"tool_call_id": tc.id,
"content": f"Tool not available: {fn_name}",
}
)
return
tool_event_base: Final = {
"type": "tool_call",
"tool_name": fn_name,
"tool_label": handler["label"],
"arguments": fn_args,
}
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
try:
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
except Exception as e:
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
tool_result = f"Error fetching {handler['label']}. Please try again."
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
response: Final = await litellm.acompletion(
model=model,
messages=chat_messages,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
if delta:
yield _sse({"type": "chunk", "content": delta})
async def stream_usage_ai_chat(
messages: list[dict[str, str]],
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
chat_messages: Final[list[Mapping[str, object]]] = [
{"role": "system", "content": _build_system_prompt(is_admin)},
*truncated,
]
try:
yield _sse({"type": "status", "message": "Thinking..."})
tools: Final = get_tools_for_role(is_admin)
response: Final = await litellm.acompletion(
model=resolved_model,
messages=chat_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
)
choice: Final = response.choices[0]
if not choice.message.tool_calls:
if choice.message.content:
yield _sse({"type": "chunk", "content": choice.message.content})
yield _sse({"type": "done"})
return
chat_messages.append(choice.message.model_dump())
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
yield event
yield _sse({"type": "done"})
except Exception as e:
verbose_proxy_logger.error("AI usage chat failed: %s", e)
yield _sse(
{
"type": "error",
"message": "An internal error occurred. Please try again.",
}
)

View file

@ -1,77 +0,0 @@
"""
USAGE AI CHAT ENDPOINTS
/usage/ai/chat - Stream AI chat responses about usage data
"""
from typing import Final, Literal
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from pydantic import Field
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
class ChatMessage(LiteLLMBaseModel):
role: Literal["user", "assistant"]
content: str
class UsageAIChatRequest(LiteLLMBaseModel):
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
model: str | None = Field(default=None, description="Model to use for AI chat")
@router.post(
"/usage/ai/chat",
tags=["Budget & Spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
)
async def usage_ai_chat(
data: UsageAIChatRequest,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
AI chat about usage data. Streams SSE events with the AI response.
The AI agent has access to tools that query aggregated daily activity data.
"""
from litellm.proxy.management_endpoints.common_utils import (
require_caller_user_id_for_non_admin,
user_api_key_has_admin_view,
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
stream_usage_ai_chat,
)
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
if is_admin:
user_id = user_api_key_dict.user_id
else:
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)

View file

@ -0,0 +1,109 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm.proxy.utils import PrismaClient
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
{
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
"NULLIF(t.organization_id, ''))",
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
"models": "cardinality(t.models)",
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
}
)
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
@dataclass(frozen=True, slots=True)
class UnsupportedTeamFilter:
field: str
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
if field not in ("team_id", "team_alias", "organization_id"):
return UnsupportedTeamFilter(field)
column: Final = f't."{field}"'
match value:
case str(text):
return f"{column} = ${index}", (text,)
case {"in": list(values)}:
return f"{column} = ANY(${index}::text[])", (values,)
case {"contains": str(text), "mode": "insensitive"}:
return f"{column} ILIKE ${index}", (f"%{text}%",)
case {"startsWith": str(text)}:
return f"{column} LIKE ${index}", (f"{text}%",)
case _:
return UnsupportedTeamFilter(field)
def _group_tokens(
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
) -> tuple[str | tuple[str, JsonValue], ...]:
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
if not children:
return ("FALSE" if field == "OR" else "TRUE",)
return (
"(",
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
")",
)
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
while pending:
match pending.pop():
case str(fragment):
fragments.append(fragment)
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
pending.extend(reversed(_group_tokens(field, value)))
case (field, value):
match _field_sql(field, value, len(params) + 1):
case UnsupportedTeamFilter() as failure:
return failure
case (fragment, values):
fragments.append(fragment)
params.extend(values)
return "".join(fragments), tuple(params)
class _TeamId(BaseModel):
model_config = ConfigDict(frozen=True)
team_id: str
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
async def sorted_team_ids(
prisma_client: PrismaClient,
*,
where: Mapping[str, object],
sort_by: str,
descending: bool,
skip: int,
take: int,
) -> Sequence[str] | UnsupportedTeamFilter:
result: Final = _where_sql(_WHERE.validate_python(where))
if isinstance(result, UnsupportedTeamFilter):
return result
predicate, params = result
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
direction: Final = "DESC" if descending else "ASC"
sql: Final = (
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
return tuple(row.team_id for row in rows)

View file

@ -0,0 +1,77 @@
from collections.abc import Sequence
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.utils import PrismaClient
class _UserId(BaseModel):
model_config = ConfigDict(frozen=True)
user_id: str
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
async def sorted_user_ids(
prisma_client: PrismaClient,
*,
sort_by: Literal["status", "key_count"],
descending: bool,
skip: int,
take: int,
role: str | None,
user_ids: str | None,
user_email: str | None,
team: str | None,
sso_user_ids: str | None,
organization_ids: str | None,
search: str | None,
) -> Sequence[str]:
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
expression: Final = (
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
if sort_by == "status"
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
)
direction: Final = "DESC" if descending else "ASC"
sql: Final = f"""
SELECT u.user_id FROM "LiteLLM_UserTable" u
WHERE ($1::text IS NULL OR u.user_role = $1)
AND ($2::text[] IS NULL
OR CASE WHEN cardinality($2::text[]) = 1
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
ELSE u.user_id = ANY($2::text[]) END)
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
AND ($4::text IS NULL OR $4 = ANY(u.teams))
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
AND (cardinality($7::text[]) = 0 OR EXISTS (
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
OR u.user_email ILIKE '%' || $8 || '%')
ORDER BY {expression} {direction}, u.user_id ASC
LIMIT $9 OFFSET $10
"""
rows: Final = _USER_IDS.validate_python(
await prisma_client.db.query_raw(
sql,
role or None,
ids if user_ids else None,
user_email,
team,
sso_user_ids is None,
sso_ids,
org_ids,
search or None,
take,
skip,
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
)
)
return tuple(row.user_id for row in rows)

View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
get_blog_posts,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy._types import (
CommonProxyErrors,
)
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
url: str,
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
) -> Mapping[str, AutoRouterPresetRecord]:
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
return _load_bundled_autorouter_presets()
try:
return await fetch(url)

View file

@ -1,4 +1,5 @@
import asyncio
import os
import re
import xml.etree.ElementTree as ET
from collections import Counter
@ -13,6 +14,7 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.llms.base import LiteLLMBaseModel
@ -88,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
return _latest_release_fetch_lock
def _default_release_check_enabled() -> bool:
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
def _bucket_for(line: str) -> _Bucket | None:
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
return None
@ -224,12 +230,16 @@ async def latest_release_info(
client: Annotated[_AsyncGetClient, Depends(_default_client)],
cache: Annotated[InMemoryCache, Depends(_default_cache)],
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
) -> LatestReleaseInfo | None:
"""
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
"""
if not release_check_enabled:
return None
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)

View file

@ -6071,13 +6071,12 @@ class PrismaClient:
elif table_name == "user":
db_data = self.jsonify_object(data=data)
try:
new_user_row: Final = await UserRepository(self).table.upsert(
where={"user_id": data["user_id"]},
data={
"create": {**db_data},
"update": {}, # don't do anything if it already exists
},
)
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
if new_user_row is None:
raise HTTPException(
status_code=403,
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
)
except Exception as e:
if (
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"

View file

@ -2,8 +2,11 @@
User repository for database operations on LiteLLM_UserTable.
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Generator, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, contextmanager
from contextvars import ContextVar
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol
@ -35,6 +38,16 @@ ORDER BY p.user_id
"""
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
@contextmanager
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
try:
yield
finally:
_FREE_SSO_CREATION.reset(token)
class _UserDb(Protocol):
@ -42,6 +55,12 @@ class _UserDb(Protocol):
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
class _UserTransaction(_UserDb, Protocol):
async def execute_raw(self, query: str) -> int: ...
async def query_raw(self, query: str) -> object: ...
class _PrismaClientView(Protocol):
@property
def db(self) -> _UserDb: ...
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
@property
def writer_db(self) -> _UserDb: ...
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
from prisma import Json
total: Final = await table.count()
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
null, or true all count, so seats are counted as total users minus
the deactivated ones.
"""
from prisma import Json # pyright: ignore[reportUnknownVariableType]
return await _count_billable_users(self.table)
total: Final = await self.count()
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
"""Return the user row, or None when a new free SSO user would exceed capacity."""
scope: Final = _FREE_SSO_CREATION.get()
if (
scope is None
or (scope[0] is not None and scope[0] != data["user_id"])
or scope[1] is not asyncio.current_task()
):
return await self.table.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
client: Final[_PrismaClientView] = self.prisma_client
async with client.tx() as transaction:
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
if existing is not None:
return existing
if await _count_billable_users(transaction.litellm_usertable) >= 5:
return None
return await transaction.litellm_usertable.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
async def create_user(
self,

View file

@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
@ -1467,6 +1469,12 @@ def responses(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
try:
effective_extra_headers: Final = (
add_provider_affinity_header(
@ -1480,8 +1488,9 @@ def responses(
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
if litellm_params.provider_affinity_header is not None
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
else extra_headers
)
except ValueError as affinity_error:

View file

@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
}
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.PROVIDER_AFFINITY,
Feature.SAFEGUARDS_CLASSIFIER,
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,

View file

@ -5,10 +5,15 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.rust_bridge.detectors import overridden, present, streaming
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
def provider_affinity(request: NativeCall) -> bool:
return is_opencode_api_base(api_base(request))
def anthropic_workload_identity(request: NativeCall) -> bool:
"""Python mints an Anthropic federation token when no static credential is configured."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

View file

@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay
from pydantic import TypeAdapter
import litellm
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool:
def provider_affinity(request: NativeCall) -> bool:
"""Python validates and fills the affinity header from the session ID."""
return present(request, "provider_affinity_header")
return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request))
def text_format(request: NativeCall) -> bool:

View file

View file

@ -0,0 +1,88 @@
from typing import Final
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response.usage = chunk.get("usage", None)
else:
response.usage = None
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration

View file

@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
)
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
@ -218,6 +219,7 @@ def _get_cached_audio_utils():
return _audio_utils_module
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper
from litellm.types.llms.openai import (
AllMessageValues,
AllPromptValues,
@ -262,13 +264,17 @@ from litellm.types.utils import (
SearchProviders,
SelectTokenizerResponse,
StreamingChoices,
TextChoices,
TextCompletionResponse,
TranscriptionResponse,
Usage,
credential_api_key_secret_reference,
is_litellm_owned_kwarg,
)
from litellm.types.utils import (
TextChoices as _TextChoices,
)
from litellm.types.utils import (
TextCompletionResponse as _TextCompletionResponse,
)
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError):
claude_json_str = json.dumps(json_data)
import importlib.metadata
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
from typing_extensions import assert_never
TextChoices: TypeAlias = _TextChoices
TextCompletionResponse: TypeAlias = _TextCompletionResponse
TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper
from litellm import utils as litellm_utils
# These are lazy loaded via __getattr__
@ -340,7 +350,8 @@ if TYPE_CHECKING:
# Type stubs for lazy-loaded functions to help mypy understand their types
# These imports allow mypy to understand the types when these are accessed via __getattr__
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
exception_type: Callable[..., Exception]
from litellm.litellm_core_utils.get_litellm_params import (
get_base_model_from_litellm_call_metadata,
get_litellm_params,
@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
kind: Final = huggingface_tokenizer_kind(model)
if kind is None:
if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()):
return None
backend: Final = _huggingface_tokenizer_backend()
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
@ -7945,88 +7956,6 @@ def register_prompt_template(
return litellm.custom_prompt_dict
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response["usage"] = chunk.get("usage", None)
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
exception_type: Final = litellm_utils.exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration
def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream:
return ModelResponseStream(
id=model_response.id,

View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -42337,6 +42433,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -48499,6 +48723,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -52906,12 +53141,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53163,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -73330,6 +73569,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,

View file

@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v"
ignoreUntil = 2026-11-01
reason = "diskcache has no fixed release published; remove this entry once one exists"
[[IgnoredVulns]]
id = "GHSA-v4x8-3wr5-3wq8"
ignoreUntil = 2026-11-20
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
[[IgnoredVulns]]
id = "GHSA-vfj7-8cjw-p6xm"
ignoreUntil = 2026-11-03

View file

@ -137,7 +137,7 @@ semantic-router = [
"semantic-router>=0.1.15,<1.0; python_version < '3.14'",
"aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'",
]
mlflow = ["mlflow>=3.11.1,<4.0"]
mlflow = ["mlflow>=3.17.2,<4.0"]
grpc = [
# First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff
"grpcio==1.84.1",
@ -177,16 +177,16 @@ proxy-runtime = [
"grpcio==1.84.1",
"prometheus-client>=0.20.0,<1.0",
"langfuse>=4.7,<5.0",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"ddtrace>=4.8.2,<5.0",
"sentry-sdk>=2.21.0,<3.0",
"mangum>=0.17.0,<1.0",
"azure-ai-contentsafety>=1.0.0,<2.0",
"azure-storage-file-datalake>=12.20.0,<13.0",
"pypdf>=6.16.1,<7.0",
"pypdf>=6.21.2,<7.0",
"llm-sandbox>=0.3.39,<1.0",
"detect-secrets>=1.5.0,<2.0",
]
@ -223,10 +223,10 @@ dev = [
"types-PyYAML==6.0.12.20250915",
"botocore-stubs==1.43.14",
"types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"langfuse>=4.7,<5.0",
"fastapi-offline==1.7.6",
"fakeredis==2.34.1",
@ -255,10 +255,10 @@ proxy-dev = [
"hypercorn==0.17.3",
"prometheus-client==0.20.0",
"sentry-sdk==2.21.0",
"opentelemetry-api==1.33.1",
"opentelemetry-sdk==1.33.1",
"opentelemetry-exporter-otlp==1.33.1",
"opentelemetry-instrumentation-fastapi==0.54b1",
"opentelemetry-api==1.45.1",
"opentelemetry-sdk==1.45.1",
"opentelemetry-exporter-otlp==1.45.1",
"opentelemetry-instrumentation-fastapi==0.66b1",
"azure-identity==1.25.2",
"a2a-sdk==1.1.0",
"backports-zstd==1.8.0; python_version < '3.14'",
@ -280,7 +280,7 @@ ci = [
"langchain==1.3.9",
"lunary==1.4.36; python_version == '3.10'",
"lunary==1.4.37; python_version >= '3.11'",
"logfire==4.6.0",
"logfire==5.2.0",
"traceloop-sdk==0.34.0",
"detect-secrets==1.5.0",
"PyGithub==2.8.1",
@ -359,7 +359,7 @@ constraint-dependencies = [
"packaging>=24.0",
"soupsieve>=2.8.4",
"httplib2>=0.32.0",
"setuptools>=83.0.0",
"setuptools>=84.1.0",
]
override-dependencies = [
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
@ -370,10 +370,23 @@ override-dependencies = [
default-groups = ["dev"]
required-version = ">=0.10.9"
exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window
certifi = "2026-10-20T03:00:00Z"
# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window
grpcio = "2026-10-22T15:00:00Z"
# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window
exclude-newer-package = { certifi = "2026-10-20T03:00:00Z", grpcio = "2026-10-22T15:00:00Z", boto3 = "2026-10-21T19:31:00Z", botocore = "2026-10-21T19:31:00Z" }
boto3 = "2026-10-21T19:31:00Z"
botocore = "2026-10-21T19:31:00Z"
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window
mlflow = "2026-10-21T03:00:00Z"
mlflow-skinny = "2026-10-21T03:00:00Z"
mlflow-tracing = "2026-10-21T03:00:00Z"
# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window
pypdf = "2026-10-21T17:00:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

View file

@ -1,125 +0,0 @@
"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check
the daily activity routes answer 400 with, instead of a generic fetch error."""
from __future__ import annotations
import json
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Final
import pytest
from integration._support.client import JSON_OBJECT, Gateway, object_value
from integration._support.upstream import delete_scenario, register_scenario
from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway
from pydantic import JsonValue
pytestmark: Final = pytest.mark.timeout(180)
_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data")
@dataclass(frozen=True, slots=True)
class _UsageChat:
gateway: Gateway
identity: str
@pytest.fixture(scope="module")
def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]:
"""A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream."""
identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}"
handle: Final = register_scenario(identity, json_response({}))
try:
with owned_gateway(
tmp_path_factory.mktemp("audit-usage-chat"),
{"model_list": [], "router_settings": {"num_retries": 0}},
environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity},
) as owned:
yield _UsageChat(owned, identity)
finally:
delete_scenario(handle)
def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]:
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
return {
"id": "chatcmpl-usage",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}}
for tool in _TOOLS
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]:
"""Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up
model call carried, keyed by tool, and the tool statuses the SSE stream reported."""
register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date)))
Upstream(usage_chat.gateway.upstream_url).reset()
with usage_chat.gateway.client.stream(
"POST",
"/usage/ai/chat",
json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]},
headers={"Authorization": f"Bearer {usage_chat.gateway.key}"},
timeout=60,
) as response:
assert response.status_code == 200
events: Final = tuple(
object_value(JSON_OBJECT.validate_json(line.removeprefix("data: ")))
for line in response.iter_lines()
if line.startswith("data: ")
)
calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity)
assert len(calls) == 2, calls
messages: Final = object_value(calls[1]["body"]).get("messages")
assert isinstance(messages, list), calls[1]
tool_messages: Final = {
str(message["tool_call_id"]).removeprefix("call-"): str(message["content"])
for message in messages
if isinstance(message, dict) and message.get("role") == "tool"
}
statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call")
return tool_messages, statuses
_REVERSED: Final = "Invalid date range: end_date must be on or after start_date"
_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates"
_NO_TEAM_DATA: Final = "No Team usage data found for the given date range."
_NO_TAG_DATA: Final = "No Tag usage data found for the given date range."
@pytest.mark.parametrize(
("start_date", "end_date", "team_message", "tag_message"),
(
pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"),
pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"),
pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"),
pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"),
),
)
def test_usage_chat_tools_answer_with_the_query_layers_date_check(
usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str
) -> None:
"""The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query
takes these dates without that check, still answers with its usage summary."""
tool_messages, statuses = _tool_results(usage_chat, start_date, end_date)
assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages
assert tool_messages["get_team_usage_data"] == team_message, tool_messages
assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages
assert statuses == ("running", "complete") * len(_TOOLS), statuses

View file

@ -0,0 +1,94 @@
from datetime import datetime, timezone
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
from tests.integration._support.client import Gateway, Scenario
@pytest.mark.parametrize("descending", (False, True))
def test_budget_reset_sort_orders_durations_before_pagination(
descending: bool,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "5d", "𝟝d", "5d", None)
ids: Final = tuple(
scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10)
for index, duration in enumerate(durations)
)
now: Final = datetime.now(timezone.utc)
expected: Final = tuple(
budget_id
for _, budget_id in sorted(
zip(durations, ids, strict=True),
key=lambda item: (
item[0] is None,
duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0,
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "-budget_duration" if descending else "budget_duration",
"page_size": "3",
"page": str(page),
},
)
for page in range(1, -(-len(ids) // 3) + 1)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
filtered: Final = gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "budget_duration",
"filter[budget_duration][in]": "7d,30d",
},
)
assert filtered.status_code == 200, filtered.text
assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"]
@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit"))
@pytest.mark.parametrize("descending", (False, True))
def test_budget_limits_sort_unlimited_as_infinity(
field: str, descending: bool, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-unlimited-{uuid4().hex}"
values: Final = (20, None, 0, 3)
ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values))
expected: Final = tuple(
identity
for value, identity in sorted(
zip(values, ids, strict=True),
key=lambda item: float("inf") if item[0] is None else item[0],
reverse=descending,
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": ("-" if descending else "") + field,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected

View file

@ -1,11 +1,16 @@
from typing import FrozenSet
from datetime import datetime, timedelta, timezone
from itertools import chain
from typing import Final, FrozenSet
import pytest
from litellm.proxy.utils import hash_token
from tests.integration._support.actors import TEAM_ALPHA, Actor
from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources
from tests.integration._support.actors import create_scratch_key
from tests.integration._support.client import Gateway, Scenario
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
pytestmark = pytest.mark.asyncio(loop_scope="session")
@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor
},
)
assert resp.status_code == 403, resp.text
@pytest.mark.parametrize(
"field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted"))
async def test_key_list_sorts_across_pages_with_nulls_last(
field: str,
direction: str,
deleted: bool,
gateway: Gateway,
scenario: Scenario,
db: ManagementDatabase,
resources: ScratchResources,
) -> None:
team_admin: Final = scenario.user(user_role="internal_user")
owner: Final = scenario.user(user_role="internal_user")
team_id: Final = scenario.team(
members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}]
)
team_admin_key: Final = scenario.key(user_id=team_admin)
table: Final = (
DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table
)
values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None)
tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values)))
rows: Final = [
{
"token": token,
"key_alias": resources.prefix,
"team_id": team_id,
"user_id": owner,
field: (
datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value)
if field in ("expires", "last_active", "budget_reset_at") and value is not None
else value
),
}
for token, value in zip(tokens, values, strict=True)
]
await table.create_many(rows)
try:
valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None)
expected: Final = [
token
for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1]))
] + [token for value, token in zip(values, tokens, strict=True) if value is None]
for caller in (gateway.key, team_admin_key):
pages: Final = [
gateway.client.get(
"/key/list",
headers={"Authorization": f"Bearer {caller}"},
params={
"key_alias": resources.prefix,
"team_id": team_id,
"include_team_keys": "true",
"sort_by": field,
"sort_order": direction,
"size": "2",
"page": str(page),
**({"status": "deleted"} if deleted else {}),
},
)
for page in (1, 2, 3)
]
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total_count"] == len(tokens) for page in pages)
assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected
finally:
await table.delete_many(where={"token": {"in": tokens}})

View file

@ -3,7 +3,9 @@ import contextlib
import json
import uuid
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Final
from urllib.parse import parse_qs, urlencode, urlparse
@ -15,7 +17,7 @@ from pydantic import JsonValue
from tests.integration._support.client import Gateway, gateway_from_environment
from tests.integration._support.database import read_rows
from tests.integration._support.process import owned_proxy
from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy
from tests.integration._support.wire import Reply, Request, wire_server
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role(
"metadata": {"auth_provider": IDP_PROVIDER},
}
]
def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value)
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
with _sso_gateway(tmp_path, {}) as gateway:
for index in range(4):
signed_in: Final = _sign_in(gateway, f"free-sso-{index}")
assert signed_in.status_code == 303, signed_in.text
contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8))
with ThreadPoolExecutor(max_workers=len(contenders)) as pool:
responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders))
assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403]
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}]
for subject, response in zip(contenders, responses):
if response.status_code == 403:
assert "requires a LiteLLM Enterprise license" in response.text
assert not response.cookies
assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == []
existing: Final = _sign_in(gateway, "free-sso-0")
assert existing.status_code == 303, existing.text
admin_created: Final = gateway.request(
"POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False}
)
assert admin_created.status_code == 200, admin_created.text
above_capacity: Final = _sign_in(gateway, "free-sso-0")
refused: Final = _sign_in(gateway, "free-sso-extra")
assert (above_capacity.status_code, refused.status_code) == (303, 403)
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}]

View file

@ -0,0 +1,244 @@
import json
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize(
"field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_across_pages_and_respects_membership(
field: str,
direction: str,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
caller: Final = scenario.key(user_id=owner)
values: Final = (20, 3, 3, 100, 0)
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{index}",
team_alias=f"{prefix}-{index}",
members_with_roles=[{"user_id": owner, "role": "admin"}],
)
for index in range(len(values))
)
outsider: Final = scenario.team(team_alias=f"{prefix}-outside")
for team_id, value in zip(teams, values, strict=True):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, '
"tpm_limit = %s::bigint, rpm_limit = %s::bigint, "
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', "
"members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) "
"WHERE team_id = %s",
(
str(value),
str(value),
str(value),
str(value),
str(value),
json.dumps(
[{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)]
),
str(value),
team_id,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team_id, team_id, str(value)),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s',
(teams[-1],),
)
try:
expected: Final = tuple(
team_id
for value, team_id in sorted(
zip(values, teams, strict=True),
key=lambda item: (
(item[0] != 0 if direction == "desc" else item[0] == 0)
if field in ("max_budget", "tpm_limit", "rpm_limit")
else False,
item[0] * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"search": prefix,
"user_id": owner,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages))
assert returned == expected
assert outsider not in returned
assert all(page.json()["total"] == len(teams) for page in pages)
finally:
for team_id in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_organization_names_and_missing_values(
direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-{uuid4().hex}"
budget: Final = scenario.budget(max_budget=10)
organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", ""))
teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4))
try:
for (organization_id, alias), team_id in zip(organizations, teams):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(organization_id, alias, budget),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s',
(organization_id, team_id),
)
ascending: Final = (teams[0], teams[2], teams[1])
ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending))
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
params={
"search": prefix,
"sort_by": "organization_alias",
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1])
finally:
for team_id in teams:
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,))
for organization_id, _ in organizations:
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,))
@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_team_sorts_preserve_org_and_membership_union(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-scope-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{i}",
team_alias=f"{prefix}-{i}",
members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [],
)
for i in range(4)
)
try:
for org, alias in zip(orgs, ("alpha", "zulu"), strict=True):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, alias, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, team in enumerate(teams):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, '
"models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s",
(
orgs[0] if index < 2 else orgs[1],
json.dumps(
[
{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"}
for i in range(index + 1)
]
),
str(index + 1),
team,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team, team, str(index + 1)),
)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3)
expected: Final = tuple(
team
for value, team in sorted(
zip(values, teams[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"user_id": owner,
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected
denied: Final = gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"organization_id": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for team in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,))
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

View file

@ -0,0 +1,179 @@
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None:
prefix: Final = f"sort-{uuid4().hex}"
ids: Final = tuple(
scenario.user(
user_id=f"{prefix}-{index}",
user_role="internal_user",
user_alias=f"Alias {value if index != 2 else 4:03}",
user_email=f"{prefix}-{index}@example.com",
max_budget=value,
metadata={"scim_active": value != 3},
)
for index, value in enumerate((20, 3, 3, 100, 0))
)
for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True):
write_rows(
'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, '
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s",
(f"{prefix}-{value:03}", str(value), user_id),
) if value != 3 or user_id == ids[1] else write_rows(
"UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s",
(f"{prefix}-004", user_id),
)
for index in range(value if value < 4 else 1):
scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}")
write_rows(
'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s',
(ids[-1],),
)
scenario.key(user_id=ids[-1])
write_rows(
'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1])
)
values: Final = (
(0, 1, 1, 0, 0)
if field == "status"
else (1, 3, 3, 1, 0)
if field == "key_count"
else (20, 3, 4, 100, None)
if field == "sso_user_id"
else (20, 3, 4, 100, 0)
if field == "user_alias"
else (20, 3, 3, 100, None)
)
expected: Final = tuple(
user_id
for _, user_id in sorted(
((value, user_id) for value, user_id in zip(values, ids, strict=True)),
key=lambda item: (
(item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None),
(item[0] or 0) * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == len(ids) for page in pages)
assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected
@pytest.mark.parametrize("field", ("status", "key_count"))
def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None:
team: Final = scenario.team()
member: Final = scenario.member(team)
other: Final = scenario.user(user_role="internal_user")
for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}):
baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"})
response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field})
assert response["total"] == baseline["total"]
assert sorted(user["user_id"] for user in response["users"]) == sorted(
user["user_id"] for user in baseline["users"]
)
ordinary_key: Final = scenario.key(user_id=other)
denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field})
assert denied.status_code == 403, denied.text
@pytest.mark.parametrize("field", ("status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_user_sorts_preserve_org_admin_scope(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-users-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
members: Final = tuple(
scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1})
for index in range(4)
)
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
try:
for org in orgs:
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, org, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, member in enumerate(members):
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'internal_user')",
(member, orgs[0] if index < 3 else orgs[1]),
)
for _ in range(index):
scenario.key(user_id=member)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 1, 0) if field == "status" else (0, 1, 2)
expected: Final = tuple(
member
for value, member in sorted(
zip(values, members[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
key=caller,
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages))
assert returned == expected
denied: Final = gateway.request(
"GET",
"/user/list",
key=caller,
params={
"organization_ids": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

View file

@ -0,0 +1,180 @@
from __future__ import annotations
import json
import uuid
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway, object_value
from integration._support.forward_proxy import refusing_forward_proxy
from integration._support.process import owned_proxy
from pydantic import JsonValue
_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm"
_CONTROL_HOST: Final = "provider.offline-mode.example"
_REMOTE_URLS: Final = {
"LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json",
"LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json",
"LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json",
"LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml",
}
_SELF_INITIATED_HOSTS: Final = frozenset(
{
"cost-map.offline-mode.example:443",
"beta-headers.offline-mode.example:443",
"autorouter.offline-mode.example:443",
"blog.offline-mode.example:443",
"raw.githubusercontent.com:443",
"docs.litellm.ai:443",
"api.pwnedpasswords.com:443",
"huggingface.co:443",
}
)
_PER_FEATURE_FLAGS: Final = (
"LITELLM_OFFLINE_MODE",
"LITELLM_LOCAL_MODEL_COST_MAP",
"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS",
"LITELLM_LOCAL_AUTOROUTER_PRESETS",
"LITELLM_LOCAL_POLICY_TEMPLATES",
"LITELLM_LOCAL_BLOG_POSTS",
"LITELLM_DISABLE_LATEST_RELEASE_CHECK",
"LITELLM_LICENSE",
)
def _bundled(name: str) -> JsonValue:
return json.loads((_PACKAGE / name).read_text(encoding="utf-8"))
def _write_config(directory: Path) -> Path:
config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "offline-mode-control",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_base": f"https://{_CONTROL_HOST}/v1",
"api_key": "synthetic-provider-key",
},
},
{
"model_name": "offline-mode-kimi",
"litellm_params": {
"model": "bedrock/us.moonshotai.kimi-k3",
"aws_region_name": "us-east-1",
"aws_access_key_id": "synthetic-access-key",
"aws_secret_access_key": "synthetic-secret-key",
},
},
],
"router_settings": {"num_retries": 0},
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue:
response: Final = proxy.request(method, path, body)
assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}"
return json.loads(response.content)
@pytest.mark.timeout(300)
@pytest.mark.parametrize(
("offline_mode", "expected_hosts", "reload_status"),
[
pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"),
pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"),
],
)
def test_offline_mode_stops_every_self_initiated_request(
gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int
) -> None:
with refusing_forward_proxy() as egress:
environment: Final = {
"HTTPS_PROXY": egress.url,
"HTTP_PROXY": egress.url,
"ALL_PROXY": egress.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
**_REMOTE_URLS,
**({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}),
}
with owned_proxy(
gateway,
tmp_path,
environment,
config=_write_config(tmp_path),
remove_environment=_PER_FEATURE_FLAGS,
workers=2,
) as isolated:
source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source"))
reloaded: Final = isolated.request("POST", "/reload/model_cost_map")
beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers"))
release: Final = _json(isolated, "GET", "/get/latest_release_info")
presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets"))
templates: Final = _json(isolated, "GET", "/policy/templates")
blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts"))
user: Final = object_value(
_json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"})
)
password_set: Final = object_value(
_json(
isolated,
"POST",
"/user/update",
{"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"},
)
)
tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]},
)
)
kimi_tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]},
)
)
control: Final = isolated.request(
"POST",
"/v1/chat/completions",
{"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]},
)
reached: Final = frozenset(egress.targets())
assert reached == expected_hosts
assert control.status_code != 200, control.text
assert source["source"] == "local", source
assert source["is_env_forced"] is (offline_mode is not None), source
assert reloaded.status_code == reload_status, reloaded.text
assert beta_headers["status"] == "success", beta_headers
assert release is None
assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json")))
assert templates == _bundled("policy_templates_backup.json")
assert [object_value(post)["url"] for post in _list(blog["posts"])] == [
object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"])
]
assert password_set["user_id"] == user["user_id"], password_set
assert tokens["tokenizer_type"] == "openai_tokenizer", tokens
assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens
def _list(value: JsonValue) -> list[JsonValue]:
assert isinstance(value, list), value
return value

View file

@ -99,16 +99,9 @@ class _Activity(BaseModel):
results: tuple[_Day, ...]
class _UpstreamMessage(BaseModel):
model_config = ConfigDict(frozen=True)
role: str | None = None
class _UpstreamRequest(BaseModel):
model_config = ConfigDict(frozen=True)
stream: bool | None = None
tools: tuple[JsonValue, ...] | None = None
messages: tuple[_UpstreamMessage, ...] = ()
@dataclass(frozen=True, slots=True)
@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]:
return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n")
def _usage_tool_call() -> Mapping[str, JsonValue]:
arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)})
return {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}}
],
}
def _response_object() -> bytes:
return json.dumps(
{
@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply:
body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}")
if body.stream:
return Reply(content_type="text/event-stream", chunks=_stream_frames())
roles: Final = frozenset(message.role for message in body.messages)
if body.tools and "tool" not in roles:
return Reply(body=_completion(_usage_tool_call(), "tool_calls"))
return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop"))
@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None:
)
def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]:
return tuple(
json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ")
)
@dataclass(frozen=True, slots=True)
class Tenant:
label: str
@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]:
@pytest.mark.timeout(360)
def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
gateway: Gateway, tmp_path: Path
) -> None:
with _rig(gateway, tmp_path) as (proxy, database_url):
spender: Final = _spender(proxy, database_url, "ai-chat")
spender: Final = _spender(proxy, database_url, "repeated-timeout")
with _pinned(proxy) as pinned:
with _locked_spend_logs(database_url):
with _recording(database_url) as during_chat:
chat: Final = pinned.request(
"POST",
"/usage/ai/chat",
body={
"messages": [{"role": "user", "content": "What did we spend?"}],
"model": "openai/gpt-4o-mini",
},
)
assert chat.status_code == 200, chat.text
assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
}
assert _events(chat) == (
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": REPLY_TEXT},
{"type": "done"},
), chat.text
assert len(_lookups(during_chat)) == 1
with _recording(database_url) as during_timeout:
timed_out: Final = _read(pinned, spender.digest)
assert timed_out.status_code == 200, timed_out.text
assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed
assert _aliases(timed_out, spender.digest) == (None,), timed_out.text
assert len(_lookups(during_timeout)) == 1
cached_miss: Final = _read(pinned, spender.digest)
assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed
assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text
@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked
assert gateway_activity == 200, gateway_activity
chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}")
assert object_value(chat["usage"]) == dict(USAGE), chat
@pytest.mark.timeout(300)
def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery(
gateway: Gateway, tmp_path: Path
) -> None:
with (
scratch_database() as database_url,
database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url),
wire_server(_respond) as wire,
_proxy(gateway, tmp_path, relayed_url, wire.url) as proxy,
):
spender: Final = _spender(proxy, database_url, "dropped-ai-chat")
with _pinned(proxy) as pinned:
relay.arm()
chat: Final = pinned.request(
"POST",
"/usage/ai/chat",
body={
"messages": [{"role": "user", "content": "What did we spend?"}],
"model": "openai/gpt-4o-mini",
},
)
assert chat.status_code == 200, chat.text
assert relay.tripped.is_set(), chat.text
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
}
assert _events(chat) == (
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": REPLY_TEXT},
{"type": "done"},
), chat.text
recovered: Final = _named_once_reconnected(pinned, spender)
assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text

View file

@ -6,7 +6,6 @@ from collections.abc import Iterable
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from pathlib import Path
from queue import SimpleQueue
from typing import Final
@ -31,8 +30,7 @@ from integration._support.daily_activity import (
user_row,
user_with_an_email,
)
from integration._support.database import read_rows, scratch_database
from integration._support.process import owned_proxy
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons
REQUESTS_OF_A_BURST: Final = 21
READS_DURING_A_BURST: Final = 30
TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models")
TOOL_CALL: Final = "call_integration_usage"
ANSWER: Final = "One request cost $0.25"
SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join(
(
"Total Spend: $0.2500",
"Total Requests: 1",
"Successful: 1 | Failed: 0",
"Total Tokens: 15",
"",
"Top Models by Spend:",
" - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)",
"",
"Top Providers by Spend:",
" - openai: $0.2500 (1 reqs)",
)
)
def _chat_completion() -> dict[str, JsonValue]:
@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply:
return Reply(body=json.dumps(body).encode())
def _usage_tool_call() -> dict[str, JsonValue]:
call: Final[dict[str, JsonValue]] = {
"id": TOOL_CALL,
"type": "function",
"function": {
"name": "get_usage_data",
"arguments": json.dumps({"start_date": DAY, "end_date": DAY}),
},
}
return {
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": None, "tool_calls": [call]},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes:
chunk: Final = {
"id": "chatcmpl-integration-usage",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
}
return f"data: {json.dumps(chunk)}\n\n".encode()
def _usage_analyst(request: Request) -> Reply:
if json.loads(request.body).get("stream"):
return Reply(
chunks=(
_streamed_chunk({"role": "assistant", "content": ANSWER}, None),
_streamed_chunk({}, "stop"),
b"data: [DONE]\n\n",
),
content_type="text/event-stream",
)
return Reply(body=json.dumps(_usage_tool_call()).encode())
def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]:
return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY)
@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session(
)
@pytest.mark.timeout(300)
def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner(
gateway: Gateway, tmp_path: Path
) -> None:
question: Final = f"what did we spend {uuid.uuid4().hex}"
owner: Final = f"integration-{uuid.uuid4().hex}"
ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}"
with (
scratch_database() as scratch_url,
wire_server(_usage_analyst) as wire,
owned_proxy(
gateway,
tmp_path,
{
"DATABASE_URL": scratch_url,
"OPENAI_API_BASE": f"{wire.url}/v1",
"OPENAI_BASE_URL": f"{wire.url}/v1",
"OPENAI_API_KEY": "integration-provider-key",
},
remove_environment=("DATABASE_URL_READ_REPLICA",),
) as candidate,
):
candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False})
with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url):
answer: Final = candidate.request(
"POST",
"/usage/ai/chat",
{"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"},
)
assert answer.status_code == 200, answer.text
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": DAY, "end_date": DAY},
}
events: Final = [
json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ")
]
assert events == [
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": ANSWER},
{"type": "done"},
], answer.text
asked, analysed = wire.drain()
assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"]
assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question}
assert json.loads(analysed.body)["messages"][-1] == {
"role": "tool",
"tool_call_id": TOOL_CALL,
"content": SUMMARY_OF_ONE_SEEDED_ROW,
}
assert owner not in analysed.body.decode()
assert ownerless_key not in analysed.body.decode()
@pytest.mark.timeout(300)
def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None:
released: Final = threading.Event()

View file

@ -14,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
import pytest
import requests
from requests_mock import Adapter
pytest.importorskip("opentelemetry")
@ -22,16 +23,18 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa:
)
from opentelemetry import baggage # noqa: E402
from opentelemetry.context import attach, detach # noqa: E402
from opentelemetry._logs.severity import SeverityNumber # noqa: E402
from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402
from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402
from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above
from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above
from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.metrics import MeterProvider # noqa: E402
from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.trace.export import ( # noqa: E402
BatchSpanProcessor,
ConsoleSpanExporter,
SimpleSpanProcessor,
SpanExportResult,
)
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
InMemorySpanExporter,
@ -46,6 +49,7 @@ import litellm # noqa: E402
from tests.unit.integrations.conftest import TlsSink # noqa: E402
from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402
from litellm.integrations.otel.plumbing import providers # noqa: E402
from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above
from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402
from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402
from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402
@ -733,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict]
class Collector(BaseHTTPRequestHandler):
def do_POST(self):
body = self.rfile.read(int(self.headers["Content-Length"]))
received.append({"path": self.path, "headers": dict(self.headers), "body": body})
received.append(
{"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body}
)
self.send_response(200)
self.end_headers()
@ -793,6 +799,95 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint():
assert child["parentSpanId"] == format(parent_id, "016x")
@pytest.mark.parametrize(
("statuses", "expected"),
[
((200,), SpanExportResult.SUCCESS),
((400,), SpanExportResult.FAILURE),
((503, 200), SpanExportResult.SUCCESS),
],
)
def test_http_json_exporter_preserves_http_failure_and_retry_results(
statuses: tuple[int, ...], expected: SpanExportResult
) -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri(
"POST",
endpoint,
[{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses],
)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(
endpoint=endpoint,
headers={
"CONTENT-TYPE": "application/x-protobuf",
"x-collector-token": "secret",
},
session=session,
)
try:
assert exporter.export(()) is expected
finally:
exporter.shutdown()
assert len(adapter.request_history) == len(statuses)
for request in adapter.request_history:
assert request.headers["Content-Type"] == "application/json"
assert request.headers["x-collector-token"] == "secret"
assert request.json() == {"resourceSpans": []}
@pytest.mark.parametrize(
("size_limit", "expected", "request_count"),
[
(0, SpanExportResult.SUCCESS, 1),
(len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1),
(len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0),
],
)
def test_http_json_exporter_checks_json_size_before_sending(
size_limit: int, expected: SpanExportResult, request_count: int
) -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit)
try:
assert exporter.export(()) is expected
finally:
exporter.shutdown()
assert len(adapter.request_history) == request_count
def test_http_json_exporter_rejects_export_after_shutdown() -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
exporter.shutdown()
assert exporter.export(()) is SpanExportResult.FAILURE
assert adapter.request_history == []
def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
try:
assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE
finally:
exporter.shutdown()
assert adapter.request_history == []
def test_http_protobuf_exporter_still_posts_protobuf():
(_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector(
"http/protobuf"
@ -1630,9 +1725,13 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey
severity_number=SeverityNumber.INFO,
body="v2-tls-test",
)
log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test"))
log_data: Final = ReadableLogRecord(
log_record=record,
resource=Resource({}),
instrumentation_scope=InstrumentationScope("v2-tls-test"),
)
result = exporter.export([log_data])
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/logs"
finally:
exporter.shutdown()

View file

@ -382,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch):
assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n"
for path, response in responses.items():
context = server_spans[path].context
assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-01"]
assert response.headers.get_list("traceresponse") == [
f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"
]
@pytest.mark.parametrize("path", ["/relayed", "/relayed/health"])
@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo
headers = contextvars.Context().run(trace_response_headers, span)
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-01"
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"

View file

@ -2,13 +2,19 @@
event loop: the preset does no network I/O, and a custom exporter mints the JWT
lazily on its first export (in the BatchSpanProcessor worker thread)."""
import json
import httpx
import pytest
import requests_mock
import respx
from opentelemetry.sdk.trace.export import SpanExportResult
from litellm.integrations.otel.plumbing import providers
from litellm.integrations.otel.model.config import ExporterSpec
from litellm.integrations.otel.presets import agentops as agentops_mod
from litellm.integrations.otel.presets.agentops import (
_AGENTOPS_AUTH_ENDPOINT,
_AGENTOPS_ENDPOINT,
_AGENTOPS_EXPORTER_KIND,
_build_agentops_exporter,
@ -75,50 +81,44 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch):
)
def test_agentops_exporter_mints_jwt_lazily(monkeypatch):
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
monkeypatch.setattr(
agentops_mod, "_fetch_agentops_jwt", lambda _k: {"token": "jwt-xyz"}
)
spec = ExporterSpec(
kind=_AGENTOPS_EXPORTER_KIND,
endpoint=_AGENTOPS_ENDPOINT,
options={"api_key": "ak"},
)
exporter = _build_agentops_exporter(spec)
# No auth header until the first export triggers the (off-loop) fetch.
assert "Authorization" not in exporter._session.headers
exporter._ensure_authenticated()
assert exporter._session.headers["Authorization"] == "Bearer jwt-xyz"
# Cached: a second resolution does not re-fetch.
calls = []
monkeypatch.setattr(
agentops_mod,
"_fetch_agentops_jwt",
lambda k: calls.append(k) or {"token": "again"},
)
exporter._ensure_authenticated()
assert calls == []
def test_agentops_exporter_tolerates_fetch_failure(monkeypatch):
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
def _raise(_k):
raise RuntimeError("auth down")
monkeypatch.setattr(agentops_mod, "_fetch_agentops_jwt", _raise)
exporter = _build_agentops_exporter(
def _agentops_exporter():
return _build_agentops_exporter(
ExporterSpec(
kind=_AGENTOPS_EXPORTER_KIND,
endpoint=_AGENTOPS_ENDPOINT,
options={"api_key": "ak"},
)
)
exporter._ensure_authenticated() # must not raise
assert "Authorization" not in exporter._session.headers
def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch():
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
with respx.mock() as auth, requests_mock.Mocker() as collector:
token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"})
collector.post(_AGENTOPS_ENDPOINT, status_code=200)
exporter = _agentops_exporter()
assert token_route.call_count == 0, "building the exporter must not fetch the JWT"
results = [exporter.export(()), exporter.export(())]
assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS]
assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}]
assert [request.headers.get("Authorization") for request in collector.request_history] == [
"Bearer jwt-xyz",
"Bearer jwt-xyz",
]
def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails():
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
with respx.mock() as auth, requests_mock.Mocker() as collector:
auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down"))
collector.post(_AGENTOPS_ENDPOINT, status_code=200)
result = _agentops_exporter().export(())
assert result is SpanExportResult.SUCCESS
assert [request.headers.get("Authorization") for request in collector.request_history] == [None]
def test_fetch_jwt_uses_owned_client_not_shared_pool(monkeypatch):

View file

@ -513,6 +513,22 @@ class TestLangfuseOtelIntegration:
# Endpoint assertion removed as side effect is gone
def _exported_auth_headers(exporter):
import requests_mock
from opentelemetry.sdk.trace.export import SpanExportResult
with requests_mock.Mocker() as collector:
collector.post(requests_mock.ANY, status_code=200)
assert exporter.export(()) is SpanExportResult.SUCCESS
return [
(
request.url,
{name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")},
)
for request in collector.request_history
]
class TestLangfuseOtelKeyDynamicConfig:
"""Key/team-scoped Langfuse credentials must define the full export target
(OTLP endpoint + auth), not just auth headers on the init-time exporter."""
@ -617,15 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig:
exporter = span_processors[0].span_exporter
assert isinstance(exporter, OTLPSpanExporter)
assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces"
import base64
expected_auth = base64.b64encode(b"key_public:key_secret").decode()
assert exporter._headers == {
"Authorization": f"Basic {expected_auth}",
"x-langfuse-ingestion-version": "4",
}
assert _exported_auth_headers(exporter) == [
(
"https://langfuse.example.com/api/public/otel/v1/traces",
{"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"},
)
]
def test_key_dynamic_params_reuse_cached_provider(self):
with self._clean_env():
@ -678,10 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig:
provider = next(iter(logger._tracer_provider_cache.values())).provider
exporter = provider._active_span_processor._span_processors[0].span_exporter
assert isinstance(exporter, OTLPSpanExporter)
assert exporter._headers == {
"Authorization": f"Basic {secret}",
"x-langfuse-ingestion-version": "4",
}
assert _exported_auth_headers(exporter) == [
(
"https://langfuse.example.com/api/public/otel/v1/traces",
{"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"},
)
]
class TestLangfuseOtelResponsesAPI:

View file

@ -19,14 +19,16 @@ import pytest
# Adds the grandparent directory to sys.path to allow importing project modules
from opentelemetry import trace
from opentelemetry._logs import LogRecord
from opentelemetry._logs.severity import SeverityNumber
from opentelemetry.sdk._logs import LogData, LogRecord
from opentelemetry.sdk._logs import ReadableLogRecord
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogRecordExportResult, SimpleLogRecordProcessor
from opentelemetry.sdk.metrics import MeterProvider
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExportResult
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
from parameterized import parameterized
@ -2321,9 +2323,13 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat
severity_number=SeverityNumber.INFO,
body="tls-export-test",
)
log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test"))
log_data: Final = ReadableLogRecord(
log_record=record,
resource=Resource({}),
instrumentation_scope=InstrumentationScope("tls-export-test"),
)
result: Final = exporter.export([log_data])
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"
assert tls_sink.received.get(timeout=5) == "/v1/logs"
finally:
exporter.shutdown()
@ -6639,7 +6645,7 @@ class TestOpenTelemetryProviderlessCallAttributes(unittest.TestCase):
time.sleep(self.POLL_INTERVAL)
return None
def _emitted_log_records(self, semconv_opt_in: str) -> tuple[LogData, ...]:
def _emitted_log_records(self, semconv_opt_in: str) -> tuple[ReadableLogRecord, ...]:
log_exporter = InMemoryLogExporter()
logger_provider = OTLoggerProvider()
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
@ -6713,7 +6719,12 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated
until pods were OOMKilled)."""
BSP_THREAD_NAME = "OtelBatchSpanProcessor"
def setUp(self) -> None:
processor: Final = BatchSpanProcessor(InMemorySpanExporter())
try:
self.bsp_thread_name = processor._batch_processor._worker_thread.name
finally:
processor.shutdown()
def _logger(self, cap=3, exporter="console"):
logger = OpenTelemetry(
@ -6730,7 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
logger._tracer_provider_cache.clear()
def _live_exporter_threads(self):
return [t for t in threading.enumerate() if t.name == self.BSP_THREAD_NAME]
return [t for t in threading.enumerate() if t.name == self.bsp_thread_name]
def _wait_for_exporter_threads(self, expected, timeout=10.0):
"""Dropped providers are shut down off-thread, so poll instead of sleeping."""
@ -6751,8 +6762,6 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"})
self.assertEqual(len(logger._tracer_provider_cache), 3)
# Guards the thread-name constant: a rename upstream would make this read 0 and the
# bound assertion below would pass while measuring nothing.
self.assertGreaterEqual(len(self._live_exporter_threads()), 1)
self.assertLessEqual(self._wait_for_exporter_threads(before + 3) - before, 3)

View file

@ -8,6 +8,7 @@ from litellm import CustomLLM
from litellm.litellm_core_utils import get_llm_provider_logic
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
get_provider_affinity_header,
inferred_provider,
is_registered_custom_provider,
)
@ -706,3 +707,11 @@ class TestClaudeModelPatternMatching:
set_fallback_generalizations([])
with pytest.raises(litellm.BadRequestError):
litellm.get_llm_provider(model="claude-opus-4-9")
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
api_base: Final = "https://opencode.ai/zen/go/v1"
model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base)
assert (model, provider, destination) == ("test-model", prefix, api_base)
assert get_provider_affinity_header(destination) == "x-opencode-session"

View file

@ -399,9 +399,10 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in():
@pytest.mark.asyncio
async def test_refetch_respects_local_env_override(monkeypatch):
"""LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
async def test_refetch_respects_local_env_override(monkeypatch, flag):
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP."""
monkeypatch.setenv(flag, "True")
def _fail(request):
raise AssertionError("no HTTP request should be made when local map is forced")
@ -708,9 +709,10 @@ def test_boot_load_falls_back_when_urlsplit_rejects_the_url():
assert "invalid URL" in (source["fallback_reason"] or "")
def test_boot_load_respects_local_env_override(monkeypatch):
"""LITELLM_LOCAL_MODEL_COST_MAP=True still short-circuits to the backup with zero HTTP."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
def test_boot_load_respects_local_env_override(monkeypatch, flag):
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP."""
monkeypatch.setenv(flag, "True")
def _fail(request):
raise AssertionError("no HTTP request should be made when local map is forced")

View file

@ -2047,7 +2047,9 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
) -> None:
"""The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key,
without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS."""
import requests_mock
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import SpanExportResult
from litellm.integrations.opentelemetry import OpenTelemetry
from litellm.litellm_core_utils import litellm_logging as logging_module
@ -2069,8 +2071,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace"
exporter: Final = logger._get_span_processor().span_exporter
assert isinstance(exporter, OTLPSpanExporter)
assert exporter._endpoint == expected_endpoint
assert exporter._headers == {"x-api-key": api_key}
with requests_mock.Mocker() as collector:
collector.post(expected_endpoint, status_code=200)
assert exporter.export(()) is SpanExportResult.SUCCESS
assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [
(expected_endpoint, api_key)
]
assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ
finally:
logging_module._in_memory_loggers.clear()

View file

@ -0,0 +1,17 @@
import pytest
from litellm.litellm_core_utils.offline_mode import is_offline_mode
@pytest.mark.parametrize(
("value", "expected"),
[("true", True), ("True", True), (" TRUE ", True), ("false", False), ("", False), ("1", False), ("yes", False)],
)
def test_is_offline_mode_reads_true_case_insensitively(monkeypatch, value, expected):
monkeypatch.setenv("LITELLM_OFFLINE_MODE", value)
assert is_offline_mode() is expected
def test_is_offline_mode_is_off_when_unset(monkeypatch):
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
assert is_offline_mode() is False

View file

@ -1173,6 +1173,27 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch:
assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE)
@pytest.mark.parametrize("model", ["llama-2-7b", "meta-llama/llama-3-8b-instruct"])
def test_offline_mode_counts_downloaded_tokenizer_models_with_tiktoken(monkeypatch: pytest.MonkeyPatch, model: str):
_load_huggingface_tokenizer.cache_clear()
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
with patch("litellm.utils.tokenizer_dispatch.from_pretrained") as from_pretrained:
result: Final = _select_tokenizer_helper(model)
from_pretrained.assert_not_called()
assert result["type"] == "openai_tokenizer"
def test_offline_mode_keeps_deepseek_v4_on_its_bundled_tokenizer(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
assert litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) == _deepseek_token_count(
_CODE_SAMPLE
)
def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer():
_load_huggingface_tokenizer.cache_clear()
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()

View file

@ -1,6 +1,10 @@
import litellm
import json
from unittest.mock import MagicMock, patch
import httpx
import pytest
from unittest.mock import MagicMock
import litellm
@pytest.mark.asyncio
@ -134,6 +138,34 @@ async def test_aws_polly_tts_with_native_voice():
assert request_body.get("TextType", "text") == "text"
@pytest.mark.asyncio
@pytest.mark.usefixtures("fake_provider_credentials")
@pytest.mark.parametrize("engine", ["standard", "neural", "long-form", "generative"])
async def test_aws_polly_tts_sends_engine_from_model(engine: str) -> None:
"""
The engine in the model name must reach Polly. speech() strips the
provider prefix before the config reads it, which used to turn every
engine into the neural default.
"""
mock_httpx_response = MagicMock(spec=httpx.Response)
mock_httpx_response.content = b"fake_audio_data"
mock_httpx_response.status_code = 200
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post:
mock_post.return_value = mock_httpx_response
await litellm.aspeech(
model=f"aws_polly/{engine}",
voice="Joanna",
input="Hello",
aws_region_name="us-east-1",
)
request_body = json.loads(mock_post.call_args.kwargs["data"])
assert request_body["Engine"] == engine
@pytest.mark.asyncio
@pytest.mark.usefixtures("fake_provider_credentials")
async def test_aws_polly_tts_with_openai_voice_mapping():

View file

@ -1,6 +1,8 @@
import json
from typing import Final
import pytest
from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig
@ -17,3 +19,17 @@ def test_installed_botocore_signs_the_speech_request() -> None:
authorization: Final = headers["Authorization"]
assert "/us-west-2/polly/aws4_request" in authorization
assert json.loads(body) == {"Text": "ping", "VoiceId": "Joanna"}
@pytest.mark.parametrize(
("model", "expected"),
[
("standard", "standard"),
("Long-Form", "long-form"),
("aws_polly/generative", "generative"),
("aws_polly", "neural"),
("unknown", "neural"),
],
)
def test_extract_engine_from_model(model: str, expected: str) -> None:
assert AWSPollyTextToSpeechConfig()._extract_engine_from_model(model) == expected

View file

@ -21,6 +21,7 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import LlmProviders
from litellm.utils import ProviderConfigManager
from tests.unit.proxy.conftest import httpx_transport
@pytest.mark.serial
@ -100,6 +101,69 @@ def test_get_complete_url():
assert result == expected
@pytest.mark.parametrize(
("base_path", "expected_path"),
[
("/foundry/openai/openai/v1", "/foundry/openai/openai/v1/responses?api-version=v1"),
("/foundry/openai/openai/v1/", "/foundry/openai/openai/v1/responses?api-version=v1"),
("/foundry/openai/openai/v1/responses", "/foundry/openai/openai/v1/responses?api-version=v1"),
("/foundry/openai/openai/v1/responses/", "/foundry/openai/openai/v1/responses?api-version=v1"),
(
"/openai/v1?redirect=/openai/v1/responses",
"/openai/v1/responses?redirect=%2Fopenai%2Fv1%2Fresponses&api-version=v1",
),
(
"/openai/v1?tenant=example&api-version=preview",
"/openai/v1/responses?tenant=example&api-version=preview",
),
(
"/openai/v1/responses?api-version=2025-04-01-preview",
"/openai/v1/responses?api-version=2025-04-01-preview",
),
(
"/openai/v1/responses?redirect=/openai/responses",
"/openai/v1/responses?redirect=%2Fopenai%2Fresponses&api-version=v1",
),
("/foo%2Fbar/openai/v1?tenant=example", "/foo%2Fbar/openai/v1/responses?tenant=example&api-version=v1"),
(
"/foo%3Fbar/openai/v1/responses?tenant=example",
"/foo%3Fbar/openai/v1/responses?tenant=example&api-version=v1",
),
("/openai", "/openai/v1/responses?api-version=v1"),
("/foundry/openai", "/foundry/openai/v1/responses?api-version=v1"),
],
)
def test_explicit_v1_api_base_preserves_the_gateway_prefix_and_query(base_path: str, expected_path: str) -> None:
result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url(
api_base=f"https://gateway.example.com{base_path}", litellm_params={"api_version": "v1"}
)
assert result == f"https://gateway.example.com{expected_path}"
@pytest.mark.parametrize(
("litellm_params", "expected_version"),
[
({}, AZURE_DEFAULT_RESPONSES_API_VERSION),
({"api_version": "v1"}, "v1"),
({"api_version": "latest"}, "latest"),
({"api_version": "preview"}, "preview"),
({"api_version": "2025-04-01-preview"}, "2025-04-01-preview"),
({"api_version": "2024-10-21"}, AZURE_DEFAULT_RESPONSES_API_VERSION),
({"api_version": "2024-10-21", "responses_api_version": "preview"}, "preview"),
],
)
def test_explicit_v1_api_base_keeps_responses_api_version_selection(
litellm_params: dict[str, str], expected_version: str
) -> None:
result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url(
api_base="https://gateway.example.com/foundry/openai/openai/v1",
litellm_params=litellm_params,
)
assert result == f"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version={expected_version}"
@pytest.mark.serial
def test_response_id_path_requests_encode_response_id():
config = AzureOpenAIResponsesAPIConfig()
@ -947,6 +1011,119 @@ def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url(
assert route.calls[0].request.url == _unset_api_version_responses_url()
_COMPLETED_RESPONSE_SSE: Final = "".join(
f"data: {json.dumps(event)}\n\n"
for event in (
{
"type": "response.output_text.delta",
"sequence_number": 0,
"item_id": "msg_1",
"output_index": 0,
"content_index": 0,
"delta": "Done.",
},
{"type": "response.completed", "sequence_number": 1, "response": _COMPLETED_RESPONSE},
)
)
@pytest.fixture
def gateway_responses(respx_mock: respx.MockRouter, stream: bool) -> respx.Route:
body: Final = _COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE)
content_type: Final = "text/event-stream" if stream else "application/json"
return respx_mock.post(
"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version=v1"
).mock(return_value=httpx.Response(200, text=body, headers={"content-type": content_type}))
@pytest.mark.parametrize("stream", [False, True])
def test_explicit_v1_base_sends_responses_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None:
response: Final = litellm.responses(
model="azure/gpt-6.1-sol",
input="hi",
api_base="https://gateway.example.com/foundry/openai/openai/v1",
api_key="azure-test-key",
api_version="v1",
stream=stream,
)
text: Final = (
"".join(event.delta for event in response if event.type == "response.output_text.delta")
if stream
else response.output[0].content[0].text
)
assert text == "Done."
assert gateway_responses.call_count == 1
@pytest.mark.parametrize("stream", [False, True])
def test_explicit_v1_base_sends_bridged_chat_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None:
response: Final = litellm.completion(
model="azure/responses/gpt-6.1-sol",
messages=[{"role": "user", "content": "hi"}],
api_base="https://gateway.example.com/foundry/openai/openai/v1",
api_key="azure-test-key",
api_version="v1",
stream=stream,
)
text: Final = (
"".join(chunk.choices[0].delta.content or "" for chunk in response)
if stream
else response.choices[0].message.content
)
assert text == "Done."
assert gateway_responses.call_count == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
@pytest.mark.parametrize("stream", [False, True])
async def test_explicit_v1_base_sends_async_responses_to_the_gateway(
gateway_responses: respx.Route, stream: bool
) -> None:
response: Final = await litellm.aresponses(
model="azure/gpt-6.1-sol",
input="hi",
api_base="https://gateway.example.com/foundry/openai/openai/v1",
api_key="azure-test-key",
api_version="v1",
stream=stream,
)
text: Final = (
"".join([event.delta async for event in response if event.type == "response.output_text.delta"])
if stream
else response.output[0].content[0].text
)
assert text == "Done."
assert gateway_responses.call_count == 1
@pytest.mark.asyncio
@pytest.mark.usefixtures(httpx_transport.__name__)
@pytest.mark.parametrize("stream", [False, True])
async def test_explicit_v1_base_sends_async_bridged_chat_to_the_gateway(
gateway_responses: respx.Route, stream: bool
) -> None:
response: Final = await litellm.acompletion(
model="azure/responses/gpt-6.1-sol",
messages=[{"role": "user", "content": "hi"}],
api_base="https://gateway.example.com/foundry/openai/openai/v1",
api_key="azure-test-key",
api_version="v1",
stream=stream,
)
text: Final = (
"".join([chunk.choices[0].delta.content or "" async for chunk in response])
if stream
else response.choices[0].message.content
)
assert text == "Done."
assert gateway_responses.call_count == 1
_LIFECYCLE_BODIES: Final = {
"get": _COMPLETED_RESPONSE,
"delete": {"id": "resp_dated", "object": "response", "deleted": True},

View file

@ -879,6 +879,32 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages():
}
def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text():
messages: Final = [
{"role": "user", "content": "load the WebFetch tool"},
{
"role": "assistant",
"content": None,
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}],
},
{
"role": "tool",
"tool_call_id": "call_1",
"content": [{"type": "tool_reference", "tool_name": "WebFetch"}],
},
]
request: Final = AzureAIStudioConfig().transform_request(
model="fw-glm-5.2",
messages=messages,
optional_params={},
litellm_params={},
headers={},
)
assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""}
@pytest.mark.parametrize(
"api_base, expected_url",
[

View file

@ -1,5 +1,6 @@
import asyncio
import base64
import contextlib
import hashlib
import json
import logging
@ -18,6 +19,9 @@ import tiktoken
import litellm
from litellm._logging import verbose_logger
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
HUGGING_FACE_SOURCE,
KIMI_K3_RANKS_FILE,
KIMI_K3_REVISION,
KimiK3Encodings,
KimiK3TokenizerSource,
kimi_k3_anthropic_tokens,
@ -522,3 +526,40 @@ async def test_tokenizer_load_still_running_after_the_wait_is_logged(
assert loaded is None
assert any("Kimi K3 tokenizer still loading" in record.getMessage() for record in caplog.records)
@pytest.mark.parametrize(
("offline_mode", "disable_download", "asks_hub"),
[("true", None, False), ("false", True, False), ("false", None, True)],
)
def test_offline_mode_and_disabled_downloads_never_ask_hugging_face_for_the_tokenizer(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, offline_mode: str, disable_download: bool | None, asks_hub: bool
):
import huggingface_hub.constants
import huggingface_hub.utils._detect_agent
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path / "hub"))
monkeypatch.setattr(huggingface_hub.constants, "AGENT_HARNESSES_PATH", str(tmp_path / "agent_harnesses.json"))
monkeypatch.setattr(huggingface_hub.utils._detect_agent, "_registry", None)
monkeypatch.setenv("LITELLM_OFFLINE_MODE", offline_mode)
monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", disable_download)
with respx.mock(assert_all_called=False) as router:
hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503))
with contextlib.suppress(Exception):
HUGGING_FACE_SOURCE.ranks_path()
assert hub.called is asks_hub
def test_offline_mode_reads_the_tokenizer_from_the_hugging_face_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
import huggingface_hub.constants
cached: Final = tmp_path / "models--moonshotai--Kimi-K3" / "snapshots" / KIMI_K3_REVISION / KIMI_K3_RANKS_FILE
cached.parent.mkdir(parents=True)
cached.write_bytes(b"cached ranks")
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path))
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
with respx.mock(assert_all_called=False) as router:
hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503))
path: Final = HUGGING_FACE_SOURCE.ranks_path()
assert Path(path).read_bytes() == b"cached ranks"
assert not hub.called

View file

@ -5542,3 +5542,85 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_
sent: Final = received[0]
assert "safeguards" not in json.loads(sent.content)
assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers)
@pytest.mark.asyncio
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
@pytest.mark.parametrize(
("api_base", "explicit_header", "expected_session"),
[
("https://opencode.ai/inference/go/anthropic", None, "conversation-123"),
("https://opencode.ai/inference/go/anthropic", "caller-override", "caller-override"),
("https://other.example", None, None),
],
)
async def test_messages_opencode_session_header(
stream: bool,
environment_base: str | None,
api_base: str,
explicit_header: str | None,
expected_session: str | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
if environment_base:
monkeypatch.setenv(environment_base, "https://opencode.ai/inference/go/anthropic")
received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
payload: Final = {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "claude-haiku-5-5",
"content": [{"type": "text", "text": "OK"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 1},
}
def respond(request: httpx.Request) -> httpx.Response:
received.put_nowait(request)
return (
httpx.Response(
200,
headers={"content-type": "text/event-stream"},
text=f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': payload})}\n\n"
'event: message_stop\ndata: {"type":"message_stop"}\n\n',
)
if stream
else httpx.Response(200, json=payload)
)
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
try:
result: Final = await litellm.anthropic.messages.acreate(
model="anthropic/claude-haiku-5-5",
messages=[{"role": "user", "content": "hello"}],
max_tokens=128000,
api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base,
api_key="test-key",
stream=stream,
litellm_session_id="conversation-123",
client=upstream,
extra_headers={"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"},
)
if stream:
chunks: Final = b"".join([chunk async for chunk in result])
assert b'"text": "OK"' in chunks
assert b"message_stop" in chunks
else:
assert result["content"] == [{"type": "text", "text": "OK"}]
finally:
await upstream.close()
sent: Final = received.get_nowait()
assert str(sent.url) == f"{api_base}/v1/messages"
assert sent.headers.get("x-opencode-session") == expected_session
assert sent.headers["x-api-key"] == "test-key"
assert json.loads(sent.content) == {
"model": "claude-haiku-5-5",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 128000,
"stream": stream,
}
assert received.empty()

View file

@ -0,0 +1,90 @@
from typing import Final
import pytest
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
@pytest.mark.parametrize(
"api_base",
[
"https://opencode.ai/zen/v1",
"https://opencode.ai/zen/go/v1/",
"https://opencode.ai/inference/openai/v1",
"https://opencode.ai/inference/go/openai/v1",
"https://opencode.ai/inference/go/anthropic/v1",
"https://opencode.ai/inference/anthropic/v1",
],
)
def test_opencode_defaults_affinity_header_to_the_callers_session(api_base: str) -> None:
headers: Final = {"Authorization": "Bearer test-key"}
params: Final = {"metadata": {"session_id": "conversation-123"}}
assert add_provider_affinity_header(
headers, params, OpenCodeProviderAdapter().get_provider_affinity_header(api_base)
) == {
**headers,
"x-opencode-session": "conversation-123",
}
assert headers == {"Authorization": "Bearer test-key"}
assert params == {"metadata": {"session_id": "conversation-123"}}
@pytest.mark.parametrize(
"api_base",
[
None,
"https://api.openai.com/v1",
"https://opencode.ai.example/zen/go/v1",
"https://opencode.ai@example.com/zen/go/v1",
"https://example.com/opencode.ai/zen/go/v1",
"https://example.com/v1?upstream=https://opencode.ai/zen/go/v1",
"https://opencode.ai/console/v1",
"http://opencode.ai/zen/go/v1",
],
)
def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: str | None) -> None:
headers: Final = {"Authorization": "Bearer test-key", "x-existing": "unchanged"}
assert (
add_provider_affinity_header(
headers,
{"litellm_session_id": "conversation-123"},
OpenCodeProviderAdapter().get_provider_affinity_header(api_base),
)
== headers
)
@pytest.mark.parametrize(
"params",
[
{},
{"litellm_trace_id": "request-123"},
{"metadata": {"trace_id": "request-123"}},
{
"litellm_session_id": "request-123",
"litellm_metadata": {"session_id": "request-123", SESSION_ID_GENERATED_METADATA_KEY: True},
},
],
)
def test_opencode_does_not_invent_a_conversation_id(params: dict[str, object]) -> None:
assert (
add_provider_affinity_header(
{}, params, OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1")
)
== {}
)
def test_opencode_preserves_explicit_header_and_affinity_configuration() -> None:
assert add_provider_affinity_header(
{"X-OpenCode-Session": "explicit"},
{"litellm_session_id": "conversation-123"},
OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"),
) == {"X-OpenCode-Session": "explicit"}
assert add_provider_affinity_header(
{},
{"litellm_session_id": "conversation-123", "provider_affinity_header": "x-custom-session"},
OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"),
) == {"x-custom-session": "conversation-123"}

View file

@ -180,6 +180,18 @@ async def test_breach_check_skipped_when_disabled():
assert result is None
@pytest.mark.asyncio
async def test_breach_check_skipped_in_offline_mode(monkeypatch):
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
password = "correct horse battery staple"
result = await validate_password_not_breached(
password=password,
general_settings={},
client=_client_returning(f"{_sha1_upper(password)[5:]}:42"),
)
assert result is None
@pytest.mark.asyncio
async def test_rejects_breached_password():
password = "correct horse battery staple"

View file

@ -11688,6 +11688,124 @@ def _alias_request(route: str, data: dict, content_type: str = "application/json
return request
def _viewer_inference_key(monkeypatch: pytest.MonkeyPatch, team_id: str | None, *, spend: float = 0.25) -> str:
from litellm.proxy import proxy_server
from litellm.proxy.utils import ProxyLogging, hash_token
key: Final = "sk-viewer-inference"
hashed_key: Final = hash_token(key)
cache: Final = UserApiKeyCache()
cache.set_cache(
key=hashed_key,
value=UserAPIKeyAuth(
token=hashed_key,
user_id="viewer",
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
team_id=team_id,
models=["gpt-6.1-sol"],
max_budget=1.0,
spend=spend,
),
)
cache.set_cache(
key="viewer",
value=LiteLLM_UserTable(
user_id="viewer",
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
user_email="viewer@example.com",
models=["gpt-6.1-sol"],
),
)
if team_id is not None:
cache.set_cache(
key=team_membership_reservation_cache_key(user_id="viewer", team_id=team_id),
value=NO_TEAM_MEMBERSHIP_SENTINEL,
)
for name, value in {
**_proxy_attrs_for_centralized_checks(),
"prisma_client": MagicMock(),
"user_api_key_cache": cache,
"spend_counter_cache": DualCache(),
"proxy_logging_obj": ProxyLogging(user_api_key_cache=cache),
"llm_model_list": [],
}.items():
monkeypatch.setattr(proxy_server, name, value)
return key
@pytest.mark.asyncio
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
@pytest.mark.parametrize("route", ["/chat/completions", "/v1/chat/completions"])
async def test_viewer_inference_uses_the_normal_key_authorization_path(
monkeypatch: pytest.MonkeyPatch, team_id: str | None, route: str
) -> None:
key: Final = _viewer_inference_key(monkeypatch, team_id)
request: Final = _alias_request(route, {"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "usage"}]})
result: Final = await user_api_key_auth(request=request, api_key=f"Bearer {key}")
assert (result.user_id, result.user_role, result.team_id, result.models, result.max_budget) == (
"viewer",
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
team_id,
["gpt-6.1-sol"],
1.0,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
async def test_viewer_inference_rejects_a_model_outside_the_key_allowlist(
monkeypatch: pytest.MonkeyPatch, team_id: str | None
) -> None:
key: Final = _viewer_inference_key(monkeypatch, team_id)
request: Final = _alias_request("/chat/completions", {"model": "blocked-model"})
with pytest.raises(ProxyException) as failure:
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
assert (failure.value.code, failure.value.type, failure.value.param) == (
"403",
ProxyErrorTypes.key_model_access_denied,
"model",
)
assert failure.value.message == (
"The requested model 'blocked-model' is not available for this API key, or the model name is invalid. "
"Check the models available to you and try again."
)
@pytest.mark.asyncio
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
@pytest.mark.parametrize("spend", [1.0, 1.25], ids=["at-limit", "over-limit"])
async def test_viewer_inference_rejects_an_exhausted_key_budget(
monkeypatch: pytest.MonkeyPatch, team_id: str | None, spend: float
) -> None:
key: Final = _viewer_inference_key(monkeypatch, team_id, spend=spend)
request: Final = _alias_request("/chat/completions", {"model": "gpt-6.1-sol"})
with pytest.raises(ProxyException) as failure:
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
assert failure.value.type == ProxyErrorTypes.budget_exceeded
assert f"Current cost: {spend}, Max budget: 1.0" in failure.value.message
@pytest.mark.asyncio
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
async def test_viewer_inference_access_does_not_allow_management_writes(
monkeypatch: pytest.MonkeyPatch, team_id: str | None
) -> None:
key: Final = _viewer_inference_key(monkeypatch, team_id)
request: Final = _alias_request("/team/new", {"team_alias": "forbidden"})
with pytest.raises(ProxyException) as failure:
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
assert failure.value.code == "403"
assert "proxy_admin_viewer" in failure.value.message
async def _enforce_alias_access(token: UserAPIKeyAuth, data: dict, route: str, request, router: litellm.Router):
from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access
@ -12480,11 +12598,14 @@ async def test_lens_oauth2_validates_identity_without_inference_user_field(
patch("litellm.proxy.proxy_server.master_key", "sk-fixture-master"),
patch("litellm.proxy.proxy_server.prisma_client", None),
patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()),
patch.dict(os.environ, {
"OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect",
"OAUTH_CLIENT_ID": "fixture-client",
"OAUTH_CLIENT_SECRET": "fixture-client-secret",
}),
patch.dict(
os.environ,
{
"OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect",
"OAUTH_CLIENT_ID": "fixture-client",
"OAUTH_CLIENT_SECRET": "fixture-client-secret",
},
),
patch("httpx.AsyncClient.send", side_effect=token_info) as network,
):
if active:
@ -12611,6 +12732,4 @@ def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery(
) -> None:
from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks
assert (
_should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected
)
assert _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected

View file

@ -36,6 +36,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
build_window_spend_transaction,
)
from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
from litellm.responses.streaming_iterator import drain_logging_tasks
from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
from tests.unit.proxy.db.fake_prisma_engine import engine_call
@ -2158,46 +2159,122 @@ async def test_commit_spend_updates_to_db_increments_key_total_spend_alongside_s
@pytest.mark.asyncio
async def test_update_database_creates_single_task():
async def test_shutdown_drain_finishes_the_rollups_update_database_started():
"""
Test that update_database() fires exactly 1 asyncio.create_task() call
(the batched task) instead of the previous 11.
Regression for LIT-6027. The key, team and daily rollups ran in an unreferenced asyncio task,
so the shutdown sequence (flush, drain_logging_tasks, flush) never waited for them and spend
logs landed without matching daily rows. Once update_database returns, drain_logging_tasks
must leave every rollup for that request in its queue.
"""
db_writer = DBSpendUpdateWriter()
# Mock all helpers so nothing real runs
db_writer: Final = DBSpendUpdateWriter()
db_writer._insert_spend_log_to_db = AsyncMock()
db_writer._batch_database_updates = AsyncMock()
prisma: Final = MagicMock()
prisma.get_request_status = MagicMock(return_value="success")
fake_payload: Final = {
"request_id": "req-lit-6027",
"api_key": "hashed-key",
"user": "test-user",
"team_id": "test-team",
"agent_id": None,
"startTime": "2024-01-01T00:00:00",
"endTime": "2024-01-01T00:00:01",
"model": "gpt-4o-mini",
"custom_llm_provider": "openai",
"model_group": "gpt-4o-mini",
"prompt_tokens": 10,
"completion_tokens": 5,
"spend": 0.0,
"metadata": '{"usage_object": {}}',
}
with (
patch("litellm.proxy.proxy_server.disable_spend_logs", False),
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch("litellm.proxy.proxy_server.prisma_client", prisma),
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
patch("litellm.proxy.db.db_spend_update_writer.asyncio.create_task") as mock_create_task,
patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload),
):
await db_writer.update_database(
token="test-token",
user_id="test-user",
end_user_id="test-end-user",
end_user_id=None,
team_id="test-team",
org_id=None,
kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"},
completion_response=MagicMock(),
start_time=datetime.now(),
end_time=datetime.now(),
team_id="test-team",
org_id="test-org",
completion_response=MagicMock(),
response_cost=0.1,
kwargs={"model": "gpt-4", "custom_llm_provider": "openai"},
)
await drain_logging_tasks()
# Exactly 1 create_task call (the batch), not 11
assert mock_create_task.call_count == 1
spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
assert spend["key_list_transactions"] == {"test-token": 0.1}
assert spend["team_list_transactions"] == {"test-team": 0.1}
daily_user: Final = (
await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
)
daily_team: Final = (
await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
)
assert daily_user is not None and daily_team is not None
(user_row,) = daily_user.values()
(team_row,) = daily_team.values()
assert (user_row["api_requests"], user_row["spend"]) == (1, 0.1)
assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1)
@pytest.mark.asyncio
async def test_batch_database_updates_isolation_on_failure():
async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue():
"""
Regression for LIT-6027. A rollup enqueue that is still in flight when the shutdown drain
starts (here: blocked on a full spend queue) must be finished by the time drain_logging_tasks
returns, so the second shutdown flush commits it.
"""
enqueue_started: Final = asyncio.Event()
release_enqueue: Final = asyncio.Event()
class _BlockedSpendQueue(SpendUpdateQueue):
async def add_update(self, update: SpendUpdateQueueItem):
enqueue_started.set()
await release_enqueue.wait()
await super().add_update(update)
db_writer: Final = DBSpendUpdateWriter()
db_writer.spend_update_queue = _BlockedSpendQueue()
db_writer._insert_spend_log_to_db = AsyncMock()
with (
patch("litellm.proxy.proxy_server.disable_spend_logs", True),
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
):
await db_writer.update_database(
token="t1",
user_id=None,
end_user_id=None,
team_id="team-1",
org_id=None,
kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"},
completion_response=MagicMock(),
start_time=datetime.now(),
end_time=datetime.now(),
response_cost=0.1,
)
await enqueue_started.wait()
drain: Final = asyncio.create_task(drain_logging_tasks())
release_enqueue.set()
await drain
spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
assert spend["key_list_transactions"] == {"t1": 0.1}
assert spend["team_list_transactions"] == {"team-1": 0.1}
@pytest.mark.asyncio
async def test_batch_database_updates_isolation_on_failure(caplog: pytest.LogCaptureFixture):
"""
Test that if one helper inside _batch_database_updates raises,
all other helpers still execute.
all other helpers still execute and the failure is reported at ERROR with the request id.
"""
db_writer = DBSpendUpdateWriter()
@ -2217,6 +2294,7 @@ async def test_batch_database_updates_isolation_on_failure():
db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock()
db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock()
caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name)
await db_writer._batch_database_updates(
response_cost=0.1,
user_id="u1",
@ -2226,7 +2304,7 @@ async def test_batch_database_updates_isolation_on_failure():
end_user_id="eu1",
prisma_client=MagicMock(),
litellm_proxy_budget_name="budget",
payload={"key": "value"},
payload={"request_id": "req-iso", "key": "value"},
)
# _update_key_db raised, but all others should still have been called
@ -2242,6 +2320,48 @@ async def test_batch_database_updates_isolation_on_failure():
db_writer.add_spend_log_transaction_to_daily_team_transaction.assert_awaited_once()
db_writer.add_spend_log_transaction_to_daily_org_transaction.assert_awaited_once()
db_writer.add_spend_log_transaction_to_daily_tag_transaction.assert_awaited_once()
assert any(
"_update_key_db" in record.getMessage() and "req-iso" in record.getMessage()
for record in caplog.records
if record.levelno >= logging.ERROR
)
@pytest.mark.asyncio
async def test_malformed_metadata_keeps_the_daily_rollup_row_and_is_reported(caplog: pytest.LogCaptureFixture):
"""
Regression for LIT-6027. A spend log whose metadata is not JSON used to vanish from every
daily rollup while the spend log itself persisted, with nothing above debug in the logs.
"""
writer: Final = DBSpendUpdateWriter()
prisma: Final = MagicMock()
prisma.get_request_status = MagicMock(return_value="success")
payload: Final = {
"request_id": "req-bad-metadata",
"user": "test-user",
"startTime": "2024-01-01T00:00:00",
"endTime": "2024-01-01T00:00:01",
"api_key": "test-key",
"model": "gpt-4o-mini",
"custom_llm_provider": "openai",
"model_group": "gpt-4o-mini",
"prompt_tokens": 100,
"completion_tokens": 50,
"spend": 0.05,
"metadata": "{not-json",
}
caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name)
await writer.add_spend_log_transaction_to_daily_user_transaction( # pyright: ignore[reportUnknownMemberType] # prisma_client param is untyped on the writer
payload=payload, prisma_client=prisma
)
daily: Final = await writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
(row,) = daily.values()
assert (row["api_requests"], row["spend"], row["prompt_tokens"]) == (1, 0.05, 100)
assert any(
"req-bad-metadata" in record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR
)
@pytest.mark.asyncio
@ -2775,12 +2895,12 @@ async def test_update_database_does_not_deepcopy_on_request_path():
"""
Regression for LIT-4088: copy.deepcopy must not run while the caller awaits
update_database(). The deepcopy used to isolate the daily-spend helpers is
relocated into the _batch_database_updates background task, and the spend-log
relocated into the _batch_database_updates logging task, and the spend-log
insert receives the payload directly (all consumers are read-only).
Asserts:
- zero copy.deepcopy calls happen on the awaited request path
- the batch background task still hands the daily helpers an isolated copy
- the tracked rollup task still hands the daily helpers an isolated copy
(mutating the original after the task ran does not bleed into it)
- the spend-log insert receives the payload on the request path with the
correct content
@ -2856,16 +2976,13 @@ async def test_update_database_does_not_deepcopy_on_request_path():
response_cost=0.1,
)
# Request path is clean: nothing was deepcopied while the caller awaited.
assert len(deepcopy_calls) == 0
# The spend-log insert ran inline on the request path with the real payload.
assert captured_spend_log["ref"] is fake_payload
assert captured_spend_log["model_at_call"] == "gpt-4"
assert fake_payload["spend"] == 0.1
# Now let the batch background task run; the deepcopy happens here.
await asyncio.sleep(0)
await drain_logging_tasks()
assert len(deepcopy_calls) >= 1
assert len(captured_batch_payloads) == 1

View file

@ -52,7 +52,7 @@ app.include_router(router)
client = TestClient(app)
BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets"
SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"]
SORTABLE = ["budget_duration", "budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"]
def _row(budget_id: str, **overrides: Any) -> dict[str, Any]:
@ -172,22 +172,12 @@ def test_appends_the_tiebreaker_to_an_explicit_sort(query_raw, as_proxy_admin):
_get("sort=-max_budget")
assert 'ORDER BY "max_budget" DESC NULLS LAST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0]
assert 'ORDER BY "max_budget" DESC NULLS FIRST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0]
def test_refuses_to_sort_on_budget_duration(query_raw, as_proxy_admin):
"""The column holds "7d"/"30d", so a lexicographic ORDER BY would put "30d"
before "7d" and silently mis-order the page."""
_serve(query_raw, [])
def test_accepts_duration_sorting(query_raw, as_proxy_admin):
response = _get("sort=budget_duration")
assert response.status_code == 400
assert response.headers["content-type"].startswith("application/problem+json")
body = response.json()
assert "budget_duration" in body["detail"]
assert body["allowed"] == SORTABLE
query_raw.assert_not_called()
assert response.status_code == 200
def test_the_advertised_sort_fields_are_the_ones_that_work(query_raw, as_proxy_admin):
@ -346,7 +336,7 @@ def test_q_does_not_search_any_other_column(query_raw, as_proxy_admin):
def test_is_null_selects_the_unlimited_budgets(query_raw, as_proxy_admin):
""""Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap."""
""" "Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap."""
_serve(query_raw, [_row("b-unlimited", max_budget=None)])
body = _get("filter[max_budget][is_null]=true").json()

View file

@ -5,14 +5,20 @@ Tests _test_guardrail_definitions and _compute_overall_action directly
without needing a running proxy.
"""
import json
from pathlib import Path
from unittest.mock import patch
import pytest
from fastapi import Request
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.management_endpoints.policy_endpoints.endpoints import (
GuardrailTestResultEntry,
_compute_overall_action,
_test_guardrail_definitions,
get_policy_templates,
list_policies,
)
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
@ -368,3 +374,19 @@ async def test_list_policies_reports_each_loaded_policy_with_its_resolved_guardr
assert response == PolicyListResponse(policies=expected_policies, total_count=0)
assert list(response.policies) == list(expected_policies)
@pytest.mark.parametrize("flag", ["LITELLM_OFFLINE_MODE", "LITELLM_LOCAL_POLICY_TEMPLATES"])
@pytest.mark.asyncio
async def test_policy_templates_serve_the_bundled_copy_without_fetching(monkeypatch: pytest.MonkeyPatch, flag: str):
monkeypatch.setenv(flag, "true")
bundled = json.loads((Path(litellm.__file__).parent / "policy_templates_backup.json").read_text(encoding="utf-8"))
with patch(
"litellm.llms.custom_httpx.http_handler.get_async_httpx_client",
side_effect=AssertionError("policy templates were fetched"),
) as get_client:
response = await get_policy_templates(request=_policy_list_request(), user_api_key_dict=UserAPIKeyAuth())
get_client.assert_not_called()
assert response == bundled

View file

@ -9198,58 +9198,6 @@ async def test_saml_callback_blocked_when_admin_ui_disabled():
assert "Admin UI is Disabled" in response.body.decode()
@pytest.mark.asyncio
async def test_saml_callback_enforces_free_sso_user_limit_after_validation():
"""An IdP-initiated assertion must not bypass the >5 free-SSO-user Enterprise gate
that /sso/key/generate enforces; the ACS re-checks it after validating the assertion,
so the entitlement DB query never runs on unvalidated input."""
from litellm.proxy._types import ProxyException
from litellm.proxy.management_endpoints.types import CustomOpenID
from litellm.proxy.management_endpoints.ui_sso import saml_callback
call_order: list[str] = []
async def _fake_handle_acs(**kwargs):
call_order.append("validate")
return (CustomOpenID(
id="dana@litellm.ai",
email="dana@litellm.ai",
first_name=None,
last_name=None,
display_name="dana",
picture=None,
provider="saml",
team_ids=[],
user_role=None,
), None)
async def _fake_count_billable_users():
call_order.append("count")
return 6
async def _stream():
yield b"SAMLResponse=signed-response"
request_double = SimpleNamespace(cookies={}, headers={}, stream=_stream)
with patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}), patch(
"litellm.proxy.proxy_server.premium_user", False
), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch(
"litellm.proxy.proxy_server.master_key", MASTER_KEY
), patch(
"litellm.proxy.management_endpoints.sso.saml_sso.SAMLAuthHandler.handle_acs",
new=_fake_handle_acs,
), patch(
"litellm.repositories.user_repository.UserRepository.count_billable_users",
new=AsyncMock(side_effect=_fake_count_billable_users),
):
with pytest.raises(ProxyException) as exc:
await saml_callback(request_double)
assert str(exc.value.code) == "403"
assert call_order == ["validate", "count"]
@pytest.mark.asyncio
async def test_cli_poll_key_tolerates_missing_user_row():
"""The CLI poll must still mint the JWT when the user lookup raises,

View file

@ -1,657 +0,0 @@
"""
Tests for AI Usage Chat module.
"""
import json
from collections.abc import AsyncIterator
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
TOOL_HANDLERS,
TOOLS_ADMIN,
TOOLS_BASE,
_build_system_prompt,
_summarise_entity_data,
_summarise_usage_data,
stream_usage_ai_chat,
)
SAMPLE_AGGREGATED_RESPONSE = {
"results": [
{
"date": "2025-01-15",
"metrics": {
"spend": 50.25,
"prompt_tokens": 20000,
"completion_tokens": 10000,
"total_tokens": 30000,
"api_requests": 500,
"successful_requests": 480,
"failed_requests": 20,
"cache_read_input_tokens": 0,
"cache_creation_input_tokens": 0,
},
"breakdown": {
"models": {
"gpt-4": {
"metrics": {
"spend": 40.0,
"api_requests": 300,
"total_tokens": 25000,
},
"metadata": {},
"api_key_breakdown": {},
},
},
"providers": {
"openai": {
"metrics": {"spend": 50.25, "api_requests": 500},
"metadata": {},
"api_key_breakdown": {},
},
},
"api_keys": {
"sk-test123": {
"metrics": {"spend": 50.25},
"metadata": {"key_alias": "Production Key"},
},
},
"model_groups": {},
"mcp_servers": {},
"entities": {},
},
},
],
"metadata": {
"total_spend": 50.25,
"total_api_requests": 500,
"total_successful_requests": 480,
"total_failed_requests": 20,
"total_tokens": 30000,
},
}
SAMPLE_TEAM_RESPONSE = {
"results": [
{
"date": "2025-01-15",
"metrics": {"spend": 100.0, "api_requests": 1000, "total_tokens": 50000},
"breakdown": {
"entities": {
"team-1": {
"metrics": {
"spend": 60.0,
"api_requests": 600,
"total_tokens": 30000,
},
"metadata": {"alias": "Engineering"},
"api_key_breakdown": {},
},
"team-2": {
"metrics": {
"spend": 40.0,
"api_requests": 400,
"total_tokens": 20000,
},
"metadata": {"alias": "Marketing"},
"api_key_breakdown": {},
},
},
"models": {},
"providers": {},
"api_keys": {},
"model_groups": {},
"mcp_servers": {},
},
},
],
"metadata": {"total_spend": 100.0, "total_api_requests": 1000},
}
class TestToolSchemas:
def test_admin_tools_include_all(self):
assert len(TOOLS_ADMIN) == 3
names = {t["function"]["name"] for t in TOOLS_ADMIN}
assert "get_usage_data" in names
assert "get_team_usage_data" in names
assert "get_tag_usage_data" in names
def test_base_tools_restricted_to_usage_only(self):
assert len(TOOLS_BASE) == 1
assert TOOLS_BASE[0]["function"]["name"] == "get_usage_data"
def test_admin_prompt_mentions_all_tools(self):
prompt = _build_system_prompt(is_admin=True)
assert "get_usage_data" in prompt
assert "get_team_usage_data" in prompt
assert "get_tag_usage_data" in prompt
def test_non_admin_prompt_only_mentions_usage_tool(self):
prompt = _build_system_prompt(is_admin=False)
assert "get_usage_data" in prompt
assert "get_team_usage_data" not in prompt
assert "get_tag_usage_data" not in prompt
def test_system_prompt_includes_todays_date(self):
from datetime import date
prompt = _build_system_prompt(is_admin=True)
assert date.today().isoformat() in prompt
class TestSummariseUsageData:
def test_summarise_includes_totals(self):
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
assert "$50.25" in summary
assert "500" in summary
def test_summarise_includes_models(self):
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
assert "gpt-4" in summary
def test_summarise_includes_providers(self):
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
assert "openai" in summary
def test_summarise_handles_empty_data(self):
empty = {"results": [], "metadata": {}}
summary = _summarise_usage_data(empty)
assert "no data" in summary.lower()
class TestSummariseEntityData:
def test_team_summary_includes_teams(self):
summary = _summarise_entity_data(SAMPLE_TEAM_RESPONSE, "Team")
assert "Engineering" in summary
assert "Marketing" in summary
assert "$60.0" in summary
assert "$40.0" in summary
def test_team_summary_empty(self):
empty = {"results": [], "metadata": {}}
summary = _summarise_entity_data(empty, "Team")
assert "No Team usage data" in summary
class TestStreamUsageAiChat:
@pytest.mark.asyncio
async def test_stream_emits_status_events(self):
mock_tool_call = MagicMock()
mock_tool_call.id = "call_123"
mock_tool_call.function.name = "get_usage_data"
mock_tool_call.function.arguments = json.dumps(
{
"start_date": "2025-01-01",
"end_date": "2025-01-31",
}
)
mock_first_response = MagicMock()
mock_first_response.choices = [MagicMock()]
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
mock_first_response.choices[0].message.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_123",
"type": "function",
"function": {
"name": "get_usage_data",
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}',
},
}
],
}
async def mock_stream():
chunk = MagicMock()
chunk.choices = [MagicMock()]
chunk.choices[0].delta.content = "Total spend is $50.25"
yield chunk
with (
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
) as mock_litellm,
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_usage_data",
new_callable=AsyncMock,
) as mock_fetch,
):
mock_litellm.acompletion = AsyncMock(
side_effect=[
mock_first_response,
mock_stream(),
]
)
mock_fetch.return_value = SAMPLE_AGGREGATED_RESPONSE
events = []
async for event in stream_usage_ai_chat(
messages=[{"role": "user", "content": "What is my total spend?"}],
model="gpt-4o-mini",
user_id="user-123",
is_admin=True,
):
events.append(json.loads(event.replace("data: ", "").strip()))
status_events = [e for e in events if e["type"] == "status"]
tool_call_events = [e for e in events if e["type"] == "tool_call"]
chunk_events = [e for e in events if e["type"] == "chunk"]
done_events = [e for e in events if e["type"] == "done"]
assert len(status_events) >= 1
assert "Thinking" in status_events[0]["message"]
assert len(tool_call_events) >= 1
assert tool_call_events[0]["tool_name"] == "get_usage_data"
assert tool_call_events[0]["status"] in ("running", "complete")
assert len(chunk_events) >= 1
assert len(done_events) == 1
@pytest.mark.asyncio
async def test_stream_handles_team_tool(self):
mock_tool_call = MagicMock()
mock_tool_call.id = "call_team"
mock_tool_call.function.name = "get_team_usage_data"
mock_tool_call.function.arguments = json.dumps(
{
"start_date": "2025-01-01",
"end_date": "2025-01-31",
}
)
mock_first_response = MagicMock()
mock_first_response.choices = [MagicMock()]
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
mock_first_response.choices[0].message.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_team",
"type": "function",
"function": {
"name": "get_team_usage_data",
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}',
},
}
],
}
async def mock_stream():
chunk = MagicMock()
chunk.choices = [MagicMock()]
chunk.choices[0].delta.content = "Engineering is the top team."
yield chunk
with (
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
) as mock_litellm,
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_team_usage_data",
new_callable=AsyncMock,
) as mock_fetch,
):
mock_litellm.acompletion = AsyncMock(
side_effect=[
mock_first_response,
mock_stream(),
]
)
mock_fetch.return_value = SAMPLE_TEAM_RESPONSE
events = []
async for event in stream_usage_ai_chat(
messages=[{"role": "user", "content": "Which team spends the most?"}],
model="gpt-4o-mini",
is_admin=True,
):
events.append(json.loads(event.replace("data: ", "").strip()))
chunk_events = [e for e in events if e["type"] == "chunk"]
assert len(chunk_events) >= 1
assert "Engineering" in chunk_events[0]["content"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"reason",
[
"end_date must be on or after start_date",
"start_date and end_date must be valid YYYY-MM-DD dates",
],
ids=["reversed", "invalid-format"],
)
@pytest.mark.parametrize(
"tool_name", ["get_usage_data", "get_team_usage_data", "get_tag_usage_data"]
)
async def test_invalid_tool_dates_are_reported_to_the_model(
self, tool_name: str, reason: str
) -> None:
fetch: Final = AsyncMock(
side_effect=HTTPException(status_code=400, detail={"error": reason})
)
tool_message, statuses = await _run_one_tool(
tool_name, "2025-01-31", "2025-01-01", fetch
)
fetch.assert_awaited_once()
assert tool_message == f"Invalid date range: {reason}"
assert statuses == ["running", "complete"]
@pytest.mark.asyncio
async def test_tool_dates_the_query_layer_accepts_are_queried_as_sent(
self,
) -> None:
fetch: Final = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE)
tool_message, statuses = await _run_one_tool(
"get_usage_data", "2025/01/01", "2025-01-31T00:00:00", fetch
)
fetch.assert_awaited_once_with(
start_date="2025/01/01", end_date="2025-01-31T00:00:00"
)
assert tool_message.startswith("Total Spend: $")
assert statuses == ["running", "complete"]
@pytest.mark.asyncio
@pytest.mark.parametrize(
"error",
[
HTTPException(status_code=500, detail={"error": "db down"}),
HTTPException(status_code=400, detail="page and page_size must be >= 1"),
],
ids=["server-error", "other-400"],
)
async def test_other_tool_errors_stay_a_generic_fetch_error(
self, error: HTTPException
) -> None:
tool_message, statuses = await _run_one_tool(
"get_team_usage_data",
"2025-01-01",
"2025-01-31",
AsyncMock(side_effect=error),
)
assert tool_message == "Error fetching team usage data. Please try again."
assert statuses == ["running", "error"]
@pytest.mark.asyncio
async def test_stream_handles_error(self):
with patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
) as mock_litellm:
mock_litellm.acompletion = AsyncMock(side_effect=Exception("LLM error"))
events = []
async for event in stream_usage_ai_chat(
messages=[{"role": "user", "content": "test"}],
):
events.append(json.loads(event.replace("data: ", "").strip()))
error_events = [e for e in events if e["type"] == "error"]
assert len(error_events) == 1
assert "internal error" in error_events[0]["message"].lower()
@pytest.mark.asyncio
async def test_non_admin_enforces_user_id(self):
mock_tool_call = MagicMock()
mock_tool_call.id = "call_456"
mock_tool_call.function.name = "get_usage_data"
mock_tool_call.function.arguments = json.dumps(
{
"start_date": "2025-01-01",
"end_date": "2025-01-31",
"user_id": "other-user",
}
)
mock_first_response = MagicMock()
mock_first_response.choices = [MagicMock()]
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
mock_first_response.choices[0].message.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_456",
"type": "function",
"function": {
"name": "get_usage_data",
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31","user_id":"other-user"}',
},
}
],
}
async def mock_stream():
chunk = MagicMock()
chunk.choices = [MagicMock()]
chunk.choices[0].delta.content = "Data."
yield chunk
mock_fetch = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE)
with (
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
) as mock_litellm,
patch.dict(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS",
{
"get_usage_data": {
"fetch": mock_fetch,
"summarise": _summarise_usage_data,
"label": "global usage data",
}
},
),
):
mock_litellm.acompletion = AsyncMock(
side_effect=[
mock_first_response,
mock_stream(),
]
)
events = []
async for event in stream_usage_ai_chat(
messages=[{"role": "user", "content": "Show data"}],
model="gpt-4o-mini",
user_id="my-user-id",
is_admin=False,
):
events.append(event)
mock_fetch.assert_called_once_with(
start_date="2025-01-01",
end_date="2025-01-31",
user_id="my-user-id",
)
class TestUsageAiChatServiceAccountGuard:
"""
Security regression: a non-admin caller with user_id=None (service-account
key) must be rejected at the endpoint boundary, before any tool dispatch.
"""
@pytest.mark.asyncio
async def test_non_admin_with_user_id_none_is_rejected(self):
from fastapi import HTTPException
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
ChatMessage,
UsageAIChatRequest,
usage_ai_chat,
)
service_account_key = UserAPIKeyAuth(
user_id=None,
user_role=LitellmUserRoles.INTERNAL_USER,
)
request = MagicMock()
body = UsageAIChatRequest(
messages=[ChatMessage(role="user", content="hi")],
model="gpt-4o-mini",
)
with pytest.raises(HTTPException) as exc_info:
await usage_ai_chat(
data=body,
request=request,
user_api_key_dict=service_account_key,
)
assert exc_info.value.status_code == 403
assert "Service-account keys" in str(exc_info.value.detail)
def test_resolve_fetch_kwargs_tripwire_fires_on_none_user_id(self):
"""
Defense-in-depth: if a future endpoint forgets the entry guard and
a non-admin caller with user_id=None reaches _resolve_fetch_kwargs,
the tripwire must fire rather than issuing an unscoped query.
"""
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
_resolve_fetch_kwargs,
)
with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info:
_resolve_fetch_kwargs(
fn_name="get_usage_data",
fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"},
user_id=None,
is_admin=False,
)
assert "Endpoint-level guard missing" in str(exc_info.value)
class TestUsageAiChatKeepalive:
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
import asyncio
import litellm
from fastapi.responses import StreamingResponse
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
ChatMessage,
UsageAIChatRequest,
usage_ai_chat,
)
monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval)
async def slow_acompletion(**kwargs):
await asyncio.sleep(delay)
response = MagicMock()
response.choices = [MagicMock()]
response.choices[0].message.tool_calls = None
response.choices[0].message.content = "Total spend is $50.25"
return response
with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
new=AsyncMock(side_effect=slow_acompletion),
):
response = await usage_ai_chat(
data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"),
request=MagicMock(),
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
)
assert isinstance(response, StreamingResponse)
chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator]
return chunks, dict(response.headers)
@pytest.mark.asyncio
async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch):
chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05)
assert headers["content-type"].startswith("text/event-stream")
assert headers["cache-control"] == "no-cache"
assert headers["x-accel-buffering"] == "no"
assert chunks[0].startswith(b'data: {"type": "status"')
assert chunks[1] == b": ping\n\n"
assert chunks.count(b": ping\n\n") >= 3
assert b'"content": "Total spend is $50.25"' in b"".join(chunks)
assert chunks[-1] == b'data: {"type": "done"}\n\n'
@pytest.mark.asyncio
async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch):
chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15)
assert b": ping\n\n" not in chunks
assert chunks[0].startswith(b'data: {"type": "status"')
assert chunks[-1] == b'data: {"type": "done"}\n\n'
async def _run_one_tool(
tool_name: str, start_date: str, end_date: str, fetch: AsyncMock
) -> tuple[str, list[object]]:
"""Runs one admin chat turn whose model calls `tool_name` with these dates; returns the
tool message the follow-up model call carried and the tool statuses streamed."""
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
mock_tool_call: Final = MagicMock()
mock_tool_call.id = "call_dates"
mock_tool_call.function.name = tool_name
mock_tool_call.function.arguments = arguments
mock_first_response: Final = MagicMock()
mock_first_response.choices = [MagicMock()]
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
mock_first_response.choices[0].message.model_dump.return_value = {
"role": "assistant",
"content": None,
"tool_calls": [
{
"id": "call_dates",
"type": "function",
"function": {"name": tool_name, "arguments": arguments},
}
],
}
async def mock_stream() -> AsyncIterator[MagicMock]:
chunk: Final = MagicMock()
chunk.choices = [MagicMock()]
chunk.choices[0].delta.content = "Done."
yield chunk
with (
patch(
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
) as mock_litellm,
patch.dict(TOOL_HANDLERS[tool_name], {"fetch": fetch}),
):
mock_litellm.acompletion = AsyncMock(
side_effect=[mock_first_response, mock_stream()]
)
events: Final[tuple[dict[str, object], ...]] = tuple(
[
json.loads(event.replace("data: ", "").strip())
async for event in stream_usage_ai_chat(
messages=[{"role": "user", "content": "How much did we spend?"}],
model="gpt-4o-mini",
is_admin=True,
)
]
)
final_messages: Final = mock_litellm.acompletion.call_args_list[1].kwargs[
"messages"
]
tool_messages: Final = [m for m in final_messages if m["role"] == "tool"]
assert len(tool_messages) == 1, tool_messages
assert tool_messages[0]["tool_call_id"] == "call_dates"
return str(tool_messages[0]["content"]), [
e["status"] for e in events if e["type"] == "tool_call"
]

View file

@ -0,0 +1,85 @@
import json
import re
import sqlite3
from typing import Final
from unittest.mock import AsyncMock, MagicMock
import pytest
from pydantic import JsonValue
from litellm.proxy.management_helpers.team_list_sorting import UnsupportedTeamFilter, _where_sql, sorted_team_ids
@pytest.mark.parametrize(
("where", "expected"),
(
({}, ("a", "b", "c")),
({"OR": []}, ()),
({"AND": []}, ("a", "b", "c")),
({"team_id": "a"}, ("a",)),
({"team_alias": {"contains": "ALPHA", "mode": "insensitive"}}, ("a", "c")),
({"team_id": {"startsWith": "b"}}, ("b",)),
({"organization_id": {"in": ["org-1"]}}, ("a", "b")),
(
{
"OR": [{"team_id": "b"}, {"team_alias": {"contains": "alpha", "mode": "insensitive"}}],
"AND": [{"OR": [{"organization_id": "org-1"}, {"team_id": "c"}]}],
"team_id": {"in": ["a", "c"]},
},
("a", "c"),
),
({"team_id": "a' OR 1=1 --"}, ()),
),
)
def test_team_filters_preserve_search_and_authorization_intersection(
where: dict[str, JsonValue], expected: tuple[str, ...]
) -> None:
result: Final = _where_sql(where)
assert not isinstance(result, UnsupportedTeamFilter)
predicate, params = result
sqlite_predicate: Final = re.sub(
r"= ANY\((\$\d+)::text\[\]\)", r"IN (SELECT value FROM json_each(\1))", predicate
).replace(" ILIKE ", " LIKE ")
with sqlite3.connect(":memory:") as db:
db.execute("CREATE TABLE teams (team_id TEXT, team_alias TEXT, organization_id TEXT)")
db.executemany(
"INSERT INTO teams VALUES (?, ?, ?)",
(("a", "Alpha", "org-1"), ("b", "Beta", "org-1"), ("c", "Alpha", "org-2")),
)
rows: Final = db.execute(
f"SELECT t.team_id FROM teams t WHERE {sqlite_predicate} ORDER BY t.team_id",
{
str(index): json.dumps(value) if isinstance(value, list) else value
for index, value in enumerate(params, 1)
},
).fetchall()
assert tuple(row[0] for row in rows) == expected
@pytest.mark.parametrize("where", ({"unknown": "x"}, {"team_id": {"not": "x"}}, {"AND": [False]}))
@pytest.mark.asyncio
async def test_unsupported_filter_fails_without_reading_unscoped_teams(where: dict[str, JsonValue]) -> None:
prisma: Final = MagicMock()
prisma.db.query_raw = AsyncMock()
result: Final = await sorted_team_ids(prisma, where=where, sort_by="keys_count", descending=False, skip=0, take=25)
assert isinstance(result, UnsupportedTeamFilter)
prisma.db.query_raw.assert_not_awaited()
@pytest.mark.parametrize("sort_by", ("keys_count", "members", "models", "organization_alias"))
@pytest.mark.parametrize("descending", (False, True))
@pytest.mark.asyncio
async def test_sorted_ids_bind_search_and_pagination_without_interpolating_input(
sort_by: str, descending: bool
) -> None:
prisma: Final = MagicMock()
prisma.db.query_raw = AsyncMock(return_value=[])
search: Final = "x' OR TRUE --"
result: Final = await sorted_team_ids(
prisma, where={"team_id": search}, sort_by=sort_by, descending=descending, skip=25, take=25
)
assert result == ()
sql, *bindings = prisma.db.query_raw.await_args.args
assert search not in sql
assert bindings == [search, 25, 25]
assert ("DESC" if descending else "ASC") + " NULLS LAST, t.team_id ASC" in sql

View file

@ -0,0 +1,71 @@
from typing import Final, Literal
from unittest.mock import AsyncMock, MagicMock
import pytest
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
@pytest.mark.parametrize("sort_by", ("status", "key_count"))
@pytest.mark.parametrize("descending", (False, True))
@pytest.mark.asyncio
async def test_user_sort_binds_scope_search_and_page_and_excludes_ui_sessions(
sort_by: Literal["status", "key_count"], descending: bool
) -> None:
prisma: Final = MagicMock()
prisma.db.query_raw = AsyncMock(return_value=[])
search: Final = "x' OR TRUE --"
result: Final = await sorted_user_ids(
prisma,
sort_by=sort_by,
descending=descending,
skip=25,
take=25,
role="internal_user",
user_ids=" a, b ",
user_email=search,
team="team-1",
sso_user_ids="sso-1, sso-2",
organization_ids="org-1, org-2",
search=search,
)
assert result == ()
sql, *bindings = prisma.db.query_raw.await_args.args
assert search not in sql
assert bindings == [
"internal_user",
("a", "b"),
search,
"team-1",
False,
("sso-1", "sso-2"),
("org-1", "org-2"),
search,
25,
25,
] + ([UI_SESSION_TOKEN_TEAM_ID] if sort_by == "key_count" else [])
assert ("DESC" if descending else "ASC") + ", u.user_id ASC" in sql
assert ("k.team_id IS DISTINCT FROM $11::text" in sql) == (sort_by == "key_count")
@pytest.mark.asyncio
async def test_missing_user_filters_preserve_unfiltered_query() -> None:
prisma: Final = MagicMock()
prisma.db.query_raw = AsyncMock(return_value=[])
result: Final = await sorted_user_ids(
prisma,
sort_by="status",
descending=False,
skip=0,
take=25,
role=None,
user_ids=None,
user_email=None,
team=None,
sso_user_ids=None,
organization_ids=None,
search=None,
)
assert result == ()
assert prisma.db.query_raw.await_args.args[1:] == (None, None, None, None, True, (), (), None, 25, 0)

View file

@ -1430,6 +1430,30 @@ async def test_get_autorouter_presets_fetches_once_per_process(monkeypatch, rese
assert calls == ["https://example.test/presets.json"]
@pytest.mark.asyncio
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_AUTOROUTER_PRESETS", "LITELLM_OFFLINE_MODE"])
async def test_get_autorouter_presets_local_flags_serve_bundled_without_fetching(
monkeypatch, reset_autorouter_presets_cache, flag
):
from litellm.proxy.public_endpoints.public_endpoints import (
_load_bundled_autorouter_presets,
get_autorouter_presets,
)
monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False)
monkeypatch.setenv(flag, "True")
calls = []
async def fake_fetch(url):
calls.append(url)
raise AssertionError("no fetch when the bundled catalog is forced")
presets = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch)
assert presets == _load_bundled_autorouter_presets()
assert calls == []
@pytest.mark.asyncio
async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start(
monkeypatch, reset_autorouter_presets_cache

View file

@ -1,13 +1,12 @@
from dataclasses import dataclass, field
from collections.abc import Mapping
from dataclasses import dataclass, field
from typing import Final
from typing_extensions import LiteralString
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from httpx import ASGITransport, AsyncClient, Response
from typing_extensions import LiteralString
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth

View file

@ -1,7 +1,6 @@
from typing import Final
import pytest
from pydantic import ValidationError
from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings

View file

@ -93,6 +93,20 @@ def test_retired_roi_calculator_cannot_be_loaded_or_called(
assert response.status_code == 404, f"{method} {path}: {response.text}"
@pytest.mark.parametrize("disable_lazy_routes", ("false", "true"))
@pytest.mark.parametrize("path", ("/lazy/warm/usage_ai", "/usage/ai/chat"))
def test_retired_usage_chat_cannot_be_loaded_or_called(
monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str, path: str
) -> None:
monkeypatch.setenv(FLAG, disable_lazy_routes)
app: Final = FastAPI()
attach_lazy_features(app)
with TestClient(app) as client:
response: Final = client.post(path)
assert response.status_code == 404, f"{path}: {response.text}"
@pytest.mark.parametrize("value", ("1", "true", "TRUE", "yes", "on"))
def test_flag_registers_every_feature_at_startup(monkeypatch: pytest.MonkeyPatch, value: str) -> None:
monkeypatch.setenv(FLAG, value)

View file

@ -106,6 +106,13 @@ class TestGetMetadataVariableName:
request = self._make_request("/v1/files")
assert get_metadata_variable_name(request) == "litellm_metadata"
@pytest.mark.parametrize(
"path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"]
)
def test_returns_litellm_metadata_for_interactions_routes(self, path):
request = self._make_request(path)
assert get_metadata_variable_name(request) == "litellm_metadata"
def test_returns_metadata_for_chat_completions(self):
request = self._make_request("/chat/completions")
assert get_metadata_variable_name(request) == "metadata"
@ -708,6 +715,46 @@ async def test_add_litellm_data_to_request_parses_json_object_string_litellm_met
assert updated["litellm_metadata"]["cost_centre"] == "research"
@pytest.mark.asyncio
async def test_interactions_spend_logs_metadata_survives_the_router_metadata_bucket():
"""GH#47841: on /v1beta/interactions the router plants its deployment fields in
`litellm_metadata`. The proxy must put the request's metadata there too: otherwise the
spend log reads only the router's bucket and `spend_logs_metadata` from the
`x-litellm-spend-logs-metadata` header never reaches LiteLLM_SpendLogs."""
request_mock = MagicMock(spec=Request)
request_mock.url = MagicMock()
request_mock.url.__str__.return_value = "http://localhost/v1beta/interactions"
request_mock.url.path = "/v1beta/interactions"
request_mock.method = "POST"
request_mock.query_params = {}
request_mock.headers = {
"Content-Type": "application/json",
"x-litellm-spend-logs-metadata": '{"cost_centre": "research"}',
}
request_mock.client = MagicMock()
request_mock.client.host = "127.0.0.1"
request_mock.state.parent_otel_span = None
data = await add_litellm_data_to_request(
data={"model": "gemini-3.1-flash-lite", "input": "Reply OK.", "store": True},
request=request_mock,
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
proxy_config=MagicMock(),
general_settings={},
version="test-version",
)
assert data["litellm_metadata"]["spend_logs_metadata"] == {"cost_centre": "research"}
# The router then adds its deployment fields to the same bucket, and the spend log
# resolves that bucket.
data["litellm_metadata"].update({"model_group": "gemini-3.1-flash-lite", "deployment": "gemini/gemini-3.1-flash-lite"})
resolved = get_litellm_metadata_from_kwargs(
{"litellm_params": {"metadata": data.get("metadata") or {}, "litellm_metadata": data["litellm_metadata"]}}
)
assert resolved["spend_logs_metadata"] == {"cost_centre": "research"}
assert resolved["model_group"] == "gemini-3.1-flash-lite"
@pytest.mark.asyncio
async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_strip():
"""Regression: proxy_server_request['body'] used to be snapshotted before
@ -8064,6 +8111,67 @@ def test_client_anthropic_api_headers_stay_off_openai_compatible_providers():
assert forwarded == {}
def test_client_anthropic_api_headers_stay_off_azure_ai_outside_native_messages():
forwarded = _headers_forwarded_to({"anthropic-beta": "thinking-binding-controls-2026-08-01"}, "azure_ai")
assert forwarded == {}
@pytest.mark.asyncio
@pytest.mark.usefixtures("local_beta_headers_config")
async def test_client_thinking_binding_beta_reaches_azure_ai_foundry_claude():
import httpx
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
data: Final[dict] = {"model": "claude-opus-4-8"} # mutable-ok: the function under test writes into it
add_provider_specific_headers_to_request(
data=data,
headers={
"content-type": "application/json",
"anthropic-beta": "thinking-binding-controls-2026-08-01",
"anthropic-version": "2026-08-01",
},
)
received: Final[list[httpx.Request]] = [] # mutable-ok: the fake Foundry endpoint records what the proxy sent
def foundry_answers(request: httpx.Request) -> httpx.Response:
received.append(request)
return httpx.Response(
status_code=200,
json={
"id": "msg_01",
"type": "message",
"role": "assistant",
"model": "claude-opus-4-8",
"content": [{"type": "text", "text": "hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 12, "output_tokens": 6},
},
request=request,
)
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers))
try:
await litellm.anthropic.messages.acreate(
max_tokens=16,
messages=[{"role": "user", "content": "hi"}],
thinking={"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}},
model="azure_ai/claude-opus-4-8",
api_key="foundry-key",
api_base="https://my-resource.services.ai.azure.com/anthropic",
client=upstream,
provider_specific_header=data["provider_specific_header"],
)
finally:
await upstream.close()
assert received[0].headers["anthropic-beta"] == "thinking-binding-controls-2026-08-01"
assert received[0].headers["anthropic-version"] == "2026-08-01"
assert json.loads(received[0].content)["thinking"]["block_binding"] == {"prefix_mismatch_behavior": "drop_block"}
TYPESAFE_DANGER_ZONE_HEADERS = {
"TypeSafe-Danger-Zone-Images": "true",
"typesafe-danger-zone-long-context": "true",

View file

@ -401,6 +401,26 @@ class TestLatestReleaseInfoEndpoint:
assert response.status_code == 200
assert response.json() is None
@pytest.mark.parametrize("name", ["LITELLM_DISABLE_LATEST_RELEASE_CHECK", "LITELLM_OFFLINE_MODE"])
@pytest.mark.parametrize(
("flag", "expected_body", "expected_urls"),
[
("true", None, []),
(" TRUE ", None, []),
("false", EXPECTED_INFO, [CHANGELOG_FEED_URL, _tag_url("v1.102.0")]),
],
)
def test_disable_flag_skips_every_outbound_request(
self, http_client, monkeypatch, name, flag, expected_body, expected_urls
):
monkeypatch.setenv(name, flag)
client = _minor_release()
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)
response = http_client.get("/get/latest_release_info")
assert response.status_code == 200
assert response.json() == expected_body
assert client.urls == expected_urls
def test_repeated_requests_reuse_cache(self, http_client):
client = _minor_release()
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)

View file

@ -12,15 +12,118 @@ import asyncio
import hashlib
import json
import logging
from collections.abc import Mapping
from contextlib import AbstractAsyncContextManager
from types import SimpleNamespace
from typing import Final
from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from fastapi import HTTPException
from litellm._service_logger import ServiceTypes
from litellm.models.user import LiteLLM_UserTable
from litellm.proxy.db.log_db_metrics import record_db_io
from litellm.proxy.utils import PrismaClient
from litellm.repositories.user_repository import free_sso_user_creation
class _SSOAdmissionUsers:
def __init__(self, count: int) -> None:
self.rows: Mapping[str, LiteLLM_UserTable] = {
f"user-{index}": LiteLLM_UserTable(user_id=f"user-{index}")
for index in range(count)
}
async def find_unique(
self, where: Mapping[str, object]
) -> LiteLLM_UserTable | None:
return self.rows.get(str(where["user_id"]))
async def count(self, where: Mapping[str, object] | None = None) -> int:
if where is None:
return len(self.rows)
return sum((row.metadata or {}).get("scim_active") is False for row in self.rows.values())
async def upsert(
self, where: Mapping[str, object], data: Mapping[str, object]
) -> LiteLLM_UserTable:
existing: Final = await self.find_unique(where)
if existing is not None:
return existing
created: Final = LiteLLM_UserTable.model_validate(data["create"])
self.rows = {**self.rows, created.user_id: created}
return created
class _SSOAdmissionTransaction(AbstractAsyncContextManager["_SSOAdmissionTransaction"]):
def __init__(self, users: _SSOAdmissionUsers) -> None:
self.litellm_usertable: Final = users
async def __aenter__(self) -> _SSOAdmissionTransaction:
return self
async def __aexit__(self, *exc_info: object) -> None:
return None
async def execute_raw(self, query: str) -> int:
return 0
async def query_raw(self, query: str) -> object:
return ({"locked": 1},)
@pytest.mark.asyncio
@pytest.mark.parametrize(
("existing_count", "scope_id", "user_id", "enabled", "child_task", "refused"),
[
(4, "new-user", "new-user", True, False, False),
(5, "new-user", "new-user", True, False, True),
(6, "user-0", "user-0", True, False, False),
(5, "sso-user", "new-user", True, False, False),
(5, "new-user", "new-user", True, True, False),
(5, None, "new-user", True, False, True),
(5, None, "new-user", False, False, False),
],
)
async def test_user_insert_enforces_free_sso_admission_only_for_its_owner(
prisma_client: PrismaClient,
monkeypatch: pytest.MonkeyPatch,
existing_count: int,
scope_id: str | None,
user_id: str,
enabled: bool,
child_task: bool,
refused: bool,
) -> None:
users: Final = _SSOAdmissionUsers(existing_count)
prisma_client.db.litellm_usertable = users
prisma_client.db.tx.return_value = _SSOAdmissionTransaction(users)
monkeypatch.setattr("asyncio.sleep", AsyncMock(return_value=None))
original: Final = users.rows.get(user_id)
async def attempt() -> None:
if refused:
with pytest.raises(HTTPException) as error:
await prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
assert error.value.status_code == 403
assert user_id not in users.rows
elif child_task:
result: Final = await asyncio.create_task(
prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
)
assert result is users.rows[user_id]
else:
inserted: Final = await prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
assert inserted is users.rows[user_id]
with free_sso_user_creation(scope_id, enabled=enabled):
for _ in range(2):
await attempt()
assert len(users.rows) == existing_count + int(not refused and original is None)
if original is not None:
assert users.rows[user_id] is original
@pytest.mark.asyncio

View file

@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st
in expected_responses_api_request/.
"""
import asyncio
import copy
import json
import logging
@ -1231,3 +1232,110 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) ->
assert [event.response.output[0].content[0].text for event in response if event.type == "response.completed"] == [
"Done."
]
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("stream", [False, True])
@pytest.mark.parametrize(
("api_base", "explicit_header", "expected_session"),
[
("https://opencode.ai/inference/go/openai/v1", None, "conversation-123"),
("https://opencode.ai/inference/go/openai/v1", "caller-override", "caller-override"),
("https://other.example/v1", None, None),
],
)
@pytest.mark.asyncio
async def test_responses_opencode_session_header(
asynchronous: bool, stream: bool, api_base: str, explicit_header: str | None, expected_session: str | None
) -> None:
received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
payload: Final = _minimal_responses_api_payload("resp_test", "gpt-6-luna")
def respond(request: httpx.Request) -> httpx.Response:
received.put_nowait(request)
return (
httpx.Response(
200,
headers={"content-type": "text/event-stream"},
text="".join(
f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': payload})}\n\n"
for kind in ("response.created", "response.completed")
),
)
if stream
else httpx.Response(200, json=payload)
)
args: Final = {
"model": "openai/gpt-6-luna",
"input": "hello",
"api_base": api_base,
"api_key": "test-key",
"litellm_session_id": "conversation-123",
"stream": stream,
"extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"},
}
if asynchronous:
async_client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
try:
response: Final = await litellm.aresponses(**args, client=async_client)
if stream:
events: Final = tuple([event.type async for event in response])
assert "response.completed" in events
else:
assert response.output[0].content[0].text == "Done."
finally:
await async_client.close()
else:
sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
try:
result: Final = litellm.responses(**args, client=sync_client)
if stream:
assert "response.completed" in tuple(event.type for event in result)
else:
assert result.output[0].content[0].text == "Done."
finally:
sync_client.close()
sent: Final = received.get_nowait()
assert str(sent.url) == f"{api_base}/responses"
assert sent.headers.get("x-opencode-session") == expected_session
assert sent.headers["authorization"] == "Bearer test-key"
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello", "stream": stream}
assert received.empty()
@pytest.mark.parametrize("asynchronous", [False, True])
@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"])
@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"])
@pytest.mark.asyncio
async def test_responses_opencode_session_uses_effective_destination(
monkeypatch: pytest.MonkeyPatch,
respx_mock: respx.MockRouter,
asynchronous: bool,
source: str,
explicit_base: str | None,
) -> None:
default_base: Final = "https://opencode.ai/inference/go/openai/v1"
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None)
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
if source != "litellm.api_base":
monkeypatch.setenv(source, default_base)
endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/responses").respond(
json=_minimal_responses_api_payload("resp_test", "gpt-6-luna")
)
args: Final = {
"model": "openai/gpt-6-luna",
"input": "hello",
"api_base": explicit_base,
"api_key": "test-key",
"litellm_session_id": "conversation-123",
}
response: Final = await litellm.aresponses(**args) if asynchronous else litellm.responses(**args)
assert response.output[0].content[0].text == "Done."
assert endpoint.call_count == 1
sent: Final = endpoint.calls[0].request
assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123")
assert sent.headers["authorization"] == "Bearer test-key"
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"}

Some files were not shown because too many files have changed in this diff Show more