mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
Merge remote-tracking branch 'origin/main' into HEAD
# Conflicts: # litellm/proxy/db/db_spend_update_writer.py
This commit is contained in:
commit
f7f6d76e62
241 changed files with 14810 additions and 5637 deletions
4
.github/CODEOWNERS
vendored
4
.github/CODEOWNERS
vendored
|
|
@ -1,4 +1,4 @@
|
|||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
|
||||
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
|
||||
/helm/ @yassin-berriai
|
||||
/terraform/ @yassin-berriai
|
||||
|
|
|
|||
1
.github/workflows/test-unit.yml
vendored
1
.github/workflows/test-unit.yml
vendored
|
|
@ -425,6 +425,7 @@ jobs:
|
|||
tests/unit/realtime_api
|
||||
tests/unit/rerank_api
|
||||
tests/unit/search
|
||||
tests/unit/text_completions
|
||||
tests/unit/vector_stores
|
||||
tests/unit/videos
|
||||
python-version: "3.12"
|
||||
|
|
|
|||
|
|
@ -121,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
|
|||
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
|
||||
COPY --from=uvbin /uv /usr/local/bin/uv
|
||||
RUN apk add --no-cache python-3.13
|
||||
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
|
||||
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
|
||||
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
|
||||
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
|
||||
uv venv /opt/liteadmin --python python3.13 && \
|
||||
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \
|
||||
|
|
|
|||
|
|
@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
|
|||
|
||||
### Verify Docker Image Signatures
|
||||
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
|
||||
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
|
||||
|
||||
**Verify using the pinned commit hash (recommended):**
|
||||
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
|
||||
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
|
||||
|
||||
```bash
|
||||
cosign verify \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
|
||||
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
|
||||
ghcr.io/berriai/litellm:<release-tag>
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
-----BEGIN PUBLIC KEY-----
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
|
||||
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
|
||||
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
|
||||
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
|
||||
-----END PUBLIC KEY-----
|
||||
|
|
|
|||
|
|
@ -235,7 +235,10 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
|
||||
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
|
||||
Self::ComputerUse20250124
|
||||
| Self::ComputerUse20251124
|
||||
| Self::Context1m20250807
|
||||
| Self::ThinkingBindingControls20260801 => &[],
|
||||
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
|
||||
Self::Effort20251124 => &[BetaProvider::VertexAi],
|
||||
Self::FastMode20260201 | Self::Oauth20250420 => &[
|
||||
|
|
@ -272,7 +275,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
|
||||
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
|
||||
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
|
||||
Self::ToolSearchTool20251019 => &[
|
||||
BetaProvider::Anthropic,
|
||||
|
|
|
|||
|
|
@ -67,7 +67,7 @@
|
|||
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
|
||||
"text_editor_20241022": null,
|
||||
"text_editor_20250124": null,
|
||||
"thinking-binding-controls-2026-08-01": null,
|
||||
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
|
||||
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
|
||||
"token-efficient-tools-2025-02-19": null,
|
||||
"web-fetch-2025-09-10": "web-fetch-2025-09-10",
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ import httpx
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
# Cache for the loaded configuration
|
||||
_BETA_HEADERS_CONFIG: dict | None = None
|
||||
|
|
@ -146,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict:
|
|||
Dict containing the beta headers configuration
|
||||
"""
|
||||
# Check if local-only mode is enabled
|
||||
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
|
||||
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
|
||||
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()
|
||||
|
||||
|
|
|
|||
|
|
@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
|
||||
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
|
||||
is unavailable. This MUST be called at app-creation time — once the lifespan
|
||||
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
|
||||
middleware after an application has started".
|
||||
runs, the middleware stack is frozen and middleware added later never runs, so
|
||||
a late call is skipped without registering anything.
|
||||
|
||||
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
|
||||
``ProxyTracerProvider``; the proxy publishes the real provider as the global
|
||||
|
|
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
|
|||
try:
|
||||
if not is_otel_v2_enabled():
|
||||
return
|
||||
if app.middleware_stack is not None:
|
||||
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
|
||||
return
|
||||
|
||||
# Lazy: only the V2-enabled path needs the optional
|
||||
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
|
|||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from types import MappingProxyType
|
||||
from typing import Final, TypeAlias
|
||||
|
|
@ -13,9 +14,13 @@ from typing import Final, TypeAlias
|
|||
import requests
|
||||
from google.protobuf.json_format import MessageToDict
|
||||
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
|
||||
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk.trace import ReadableSpan
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
|
||||
|
||||
_logger: Final = logging.getLogger(__name__)
|
||||
JSON_CONTENT_TYPE: Final = "application/json"
|
||||
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
|
||||
|
||||
|
|
@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
|
|||
def __init__(
|
||||
self,
|
||||
endpoint: str | None,
|
||||
headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict
|
||||
headers: Mapping[str, str],
|
||||
certificate_file: str | None = None,
|
||||
session: "requests.Session | None" = None,
|
||||
*,
|
||||
max_request_size: int | None = None,
|
||||
) -> None:
|
||||
super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session)
|
||||
self._session.headers["Content-Type"] = JSON_CONTENT_TYPE
|
||||
super().__init__(
|
||||
endpoint=endpoint,
|
||||
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
|
||||
certificate_file=certificate_file,
|
||||
session=session,
|
||||
max_request_size=max_request_size,
|
||||
)
|
||||
|
||||
def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes:
|
||||
return encode_spans_json(spans)
|
||||
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
|
||||
if self._shutdown:
|
||||
_logger.warning("Exporter already shutdown, ignoring batch")
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
with self._metrics.export_operation(len(spans)) as result:
|
||||
try:
|
||||
serialized_data: Final = encode_spans_json(spans)
|
||||
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
|
||||
_logger.error("Failed to encode span batch: %s", error)
|
||||
result.error = error
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
if 0 < self._max_request_size < len(serialized_data):
|
||||
result.error = RequestPayloadTooLargeError(
|
||||
f"Serialized span request size {len(serialized_data)} bytes "
|
||||
f"exceeds max_request_size {self._max_request_size} bytes."
|
||||
)
|
||||
_logger.warning("Dropping span batch: %s", result.error)
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
export_result: Final = self._client.export(serialized_data)
|
||||
if not export_result.success:
|
||||
result.error = export_result.error
|
||||
result.error_attrs = (
|
||||
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
|
||||
if export_result.status_code is not None
|
||||
else None
|
||||
)
|
||||
return SpanExportResult.FAILURE
|
||||
|
||||
return SpanExportResult.SUCCESS
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ def agentops_preset(
|
|||
|
||||
def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
||||
"""Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter."""
|
||||
import requests
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
|
||||
OTLPSpanExporter,
|
||||
)
|
||||
|
|
@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
|||
"""
|
||||
|
||||
def __init__(self, *, endpoint: str | None, api_key: str | None) -> None:
|
||||
super().__init__(endpoint=endpoint)
|
||||
self._agentops_session = requests.Session()
|
||||
super().__init__(endpoint=endpoint, session=self._agentops_session)
|
||||
self._agentops_api_key = api_key
|
||||
self._auth_resolved = False
|
||||
|
||||
|
|
@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter:
|
|||
try:
|
||||
token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token")
|
||||
if token:
|
||||
# ``_session`` is the requests.Session the base exporter
|
||||
# POSTs through; updating its Authorization header is how the
|
||||
# minted JWT reaches every subsequent export.
|
||||
self._session.headers["Authorization"] = f"Bearer {token}"
|
||||
self._agentops_session.headers["Authorization"] = f"Bearer {token}"
|
||||
except Exception as e:
|
||||
verbose_logger.debug("AgentOps JWT fetch failed: %s", e)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ import httpx
|
|||
from pydantic import ConfigDict
|
||||
|
||||
from litellm import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
|
||||
|
|
@ -129,7 +130,7 @@ class GetBlogPosts:
|
|||
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
|
||||
Fetches from ``url`` otherwise, falling back to local backup on failure.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
|
||||
return cls.load_local_blog_posts()
|
||||
|
||||
now: Final = time.time()
|
||||
|
|
|
|||
|
|
@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
|
|||
if dynamic_api_key is None and api_key is not None:
|
||||
dynamic_api_key = api_key
|
||||
return model, custom_llm_provider, dynamic_api_key, api_base
|
||||
|
||||
|
||||
def get_provider_affinity_header(api_base: str | None) -> str | None:
|
||||
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
|
||||
|
||||
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)
|
||||
|
|
|
|||
|
|
@ -39,6 +39,7 @@ from litellm.constants import (
|
|||
from litellm.litellm_core_utils.fallback_generalizations import (
|
||||
set_fallback_generalizations,
|
||||
)
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
|
||||
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
|
||||
|
|
@ -458,7 +459,7 @@ async def refetch_model_cost_map(
|
|||
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
|
||||
map they already have.
|
||||
"""
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
|
||||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
|
|
@ -742,7 +743,7 @@ def get_model_cost_map(
|
|||
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
|
||||
# Note: can't use get_secret_bool here — this runs during litellm.__init__
|
||||
# before litellm._key_management_settings is set.
|
||||
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
|
||||
_cost_map_source_info.source = "local"
|
||||
_cost_map_source_info.url = None
|
||||
_cost_map_source_info.is_env_forced = True
|
||||
|
|
|
|||
7
litellm/litellm_core_utils/offline_mode.py
Normal file
7
litellm/litellm_core_utils/offline_mode.py
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
import os
|
||||
|
||||
|
||||
def is_offline_mode() -> bool:
|
||||
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
|
||||
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
|
||||
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"
|
||||
|
|
@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
|
|||
return getattr(value, key, None)
|
||||
|
||||
|
||||
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
|
||||
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
|
||||
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
|
||||
if header is None:
|
||||
return None
|
||||
return default_header
|
||||
if not isinstance(header, str):
|
||||
raise TypeError("provider_affinity_header must be a string")
|
||||
return validate_provider_affinity_header_name(header)
|
||||
|
|
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
|
|||
|
||||
|
||||
def add_provider_affinity_header(
|
||||
headers: Mapping[str, object], litellm_params: object | None
|
||||
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
|
||||
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
|
||||
header_name: Final = _get_provider_affinity_header_name(litellm_params)
|
||||
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
|
||||
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
|
||||
return dict(headers)
|
||||
|
||||
|
|
|
|||
|
|
@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
|
|||
else:
|
||||
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
|
||||
|
||||
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
|
||||
# model arrives without the "aws_polly/" prefix, e.g. "standard"
|
||||
engine: Final = self._extract_engine_from_model(model)
|
||||
mapped_params["engine"] = engine
|
||||
|
||||
|
|
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
|
|||
"""
|
||||
Extract engine from model name.
|
||||
|
||||
speech() strips the "aws_polly/" prefix before the config is called,
|
||||
so the bare engine name is what normally arrives here.
|
||||
|
||||
Examples:
|
||||
- aws_polly/neural -> neural
|
||||
- aws_polly/standard -> standard
|
||||
- standard -> standard
|
||||
- aws_polly/long-form -> long-form
|
||||
- aws_polly -> neural (default)
|
||||
"""
|
||||
if "/" in model:
|
||||
parts: Final = model.split("/")
|
||||
if len(parts) >= 2:
|
||||
engine: Final = parts[1].lower()
|
||||
if engine in self.VALID_ENGINES:
|
||||
return engine
|
||||
engine: Final = model.rsplit("/", 1)[-1].lower()
|
||||
if engine in self.VALID_ENGINES:
|
||||
return engine
|
||||
return self.DEFAULT_ENGINE
|
||||
|
||||
def validate_environment(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from collections.abc import Mapping
|
|||
from copy import deepcopy
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
import httpx
|
||||
from openai.types.responses import ResponseReasoningItem
|
||||
|
|
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
|
|||
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
|
||||
"""
|
||||
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
|
||||
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
|
||||
parsed_base: Final = urlsplit(resolved_api_base or "")
|
||||
path: Final = parsed_base.path.rstrip("/")
|
||||
if path.endswith(("/openai/v1", "/openai/v1/responses")):
|
||||
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
|
||||
return BaseAzureLLM.get_base_azure_url(
|
||||
api_base=parsed_base._replace(path=responses_path).geturl(),
|
||||
litellm_params=responses_params,
|
||||
route="/openai/v1/responses",
|
||||
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
|
||||
)
|
||||
|
||||
return BaseAzureLLM.get_base_azure_url(
|
||||
api_base=resolved_api_base,
|
||||
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
|
||||
litellm_params=responses_params,
|
||||
route="/openai/responses",
|
||||
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
audio_or_image_in_message_content,
|
||||
convert_content_list_to_str,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
|
|
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
Operates on a deep copy so the caller's messages keep their thinking blocks
|
||||
and provider metadata, which a fallback to another provider still needs.
|
||||
"""
|
||||
stripped_messages: Final = copy.deepcopy(messages)
|
||||
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
|
||||
for message in stripped_messages:
|
||||
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
|
||||
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ from typing_extensions import Self
|
|||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.prompt_templates.factory import (
|
||||
chat_completion_tools_for,
|
||||
is_flat_responses_tool,
|
||||
|
|
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
|
|||
def _hugging_face_ranks_path() -> str:
|
||||
import huggingface_hub
|
||||
|
||||
return huggingface_hub.hf_hub_download(
|
||||
repo_id=KIMI_K3_REPO,
|
||||
filename=KIMI_K3_RANKS_FILE,
|
||||
revision=KIMI_K3_REVISION,
|
||||
local_files_only=litellm.disable_hf_tokenizer_download is True,
|
||||
)
|
||||
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
|
||||
cached: Final = huggingface_hub.try_to_load_from_cache(
|
||||
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
|
||||
)
|
||||
if not isinstance(cached, str):
|
||||
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
|
||||
return cached
|
||||
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
|
||||
|
||||
|
||||
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)
|
||||
|
|
|
|||
|
|
@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
|
|||
from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params
|
||||
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
|
||||
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.realtime_errors import (
|
||||
close_after_upstream_handshake_refusal,
|
||||
realtime_error_event,
|
||||
|
|
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
|
|||
)
|
||||
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
from litellm.llms.base_llm.anthropic_messages.transformation import (
|
||||
BaseAnthropicMessagesConfig,
|
||||
)
|
||||
|
|
@ -2457,13 +2460,13 @@ class BaseLLMHTTPHandler:
|
|||
# Also check for extra_headers in kwargs (from config or direct calls)
|
||||
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
|
||||
# Merge all header sources: forwarded < extra_headers < provider_specific
|
||||
merged_headers: Final = {}
|
||||
if forwarded_headers:
|
||||
merged_headers.update(forwarded_headers)
|
||||
if extra_headers_from_kwargs:
|
||||
merged_headers.update(extra_headers_from_kwargs)
|
||||
if provider_specific_headers:
|
||||
merged_headers.update(provider_specific_headers)
|
||||
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
|
||||
{
|
||||
**(forwarded_headers or {}),
|
||||
**(extra_headers_from_kwargs or {}),
|
||||
**(provider_specific_headers or {}),
|
||||
}
|
||||
)
|
||||
request_params: Final = reduce(
|
||||
delete_nested_value,
|
||||
litellm_params.get("additional_drop_params") or (),
|
||||
|
|
@ -2474,11 +2477,19 @@ class BaseLLMHTTPHandler:
|
|||
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
|
||||
):
|
||||
await aload_beta_headers_config()
|
||||
affinity_api_base: Final = (
|
||||
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
(
|
||||
headers,
|
||||
api_base,
|
||||
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
|
||||
headers=merged_headers or {},
|
||||
headers=(
|
||||
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
|
||||
if default_affinity_header is not None
|
||||
else dict(merged_headers)
|
||||
),
|
||||
model=model,
|
||||
messages=messages,
|
||||
optional_params=request_params,
|
||||
|
|
|
|||
10
litellm/llms/opencode/common_utils.py
Normal file
10
litellm/llms/opencode/common_utils.py
Normal file
|
|
@ -0,0 +1,10 @@
|
|||
import re
|
||||
|
||||
|
||||
def is_opencode_api_base(api_base: str | None) -> bool:
|
||||
return bool(
|
||||
api_base
|
||||
and re.fullmatch(
|
||||
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
|
||||
)
|
||||
)
|
||||
6
litellm/llms/opencode/transformation.py
Normal file
6
litellm/llms/opencode/transformation.py
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
|
||||
|
||||
class OpenCodeProviderAdapter:
|
||||
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
|
||||
return "x-opencode-session" if is_opencode_api_base(api_base) else None
|
||||
|
|
@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
|
|||
with_control_options,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_provider_affinity_header,
|
||||
invalid_provider_routing_hints_message,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_provider_specific_headers import (
|
||||
|
|
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
|
|||
from litellm.llms.cohere.common_utils import CohereModelInfo
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
|
||||
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
|
||||
from litellm.llms.vertex_ai.common_utils import (
|
||||
VertexAIModelRoute,
|
||||
|
|
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
|
|||
resolve_chat_reasoning,
|
||||
)
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
|
||||
from litellm.types.completion import (
|
||||
CompletionDispatchContext,
|
||||
_CompletionDispatchResult,
|
||||
|
|
@ -169,7 +172,6 @@ from litellm.utils import (
|
|||
ProviderConfigManager,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
TextCompletionStreamWrapper,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
add_provider_specific_params_to_optional_params,
|
||||
|
|
@ -5996,19 +5998,28 @@ def completion(
|
|||
},
|
||||
)
|
||||
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
|
||||
if litellm_params.get("provider_affinity_header") is not None:
|
||||
affinity_api_base: Final = (
|
||||
str(client.base_url)
|
||||
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
|
||||
else get_openai_credentials(api_base=api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
|
||||
try:
|
||||
headers = add_provider_affinity_header(
|
||||
headers=headers or litellm.headers or MappingProxyType({}),
|
||||
litellm_params=MappingProxyType(
|
||||
{
|
||||
"provider_affinity_header": litellm_params["provider_affinity_header"],
|
||||
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
|
||||
"litellm_session_id": kwargs.get("litellm_session_id"),
|
||||
"session_id": kwargs.get("session_id"),
|
||||
"metadata": metadata,
|
||||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
raise litellm.BadRequestError(
|
||||
|
|
|
|||
|
|
@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
|
|||
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
|
||||
{
|
||||
**detectors.COMMON,
|
||||
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
|
||||
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
|
||||
Feature.RESPONSE_CACHE: detectors.response_cache,
|
||||
Feature.MODIFY_PARAMS: detectors.modify_params,
|
||||
|
|
|
|||
|
|
@ -12827,6 +12827,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 16384,
|
||||
|
|
@ -12840,6 +12841,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/Phi-4-mini-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12851,6 +12853,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_audio_token": 4e-06,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -12865,6 +12868,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12876,6 +12880,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 32768,
|
||||
|
|
@ -24892,27 +24897,118 @@
|
|||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_multilingual_v2": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4": {
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4_turbo": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3_conversational": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
|
|
@ -42337,6 +42433,134 @@
|
|||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3-0": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3-0": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3-0": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3-0": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-small-2507": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -48499,6 +48723,17 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://www.together.ai/pricing",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -52906,12 +53141,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"input_cost_per_token_batches": 1.75e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.15e-06,
|
||||
"output_cost_per_token_batches": 5.75e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -52926,12 +53163,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 10000000,
|
||||
"max_output_tokens": 10000000,
|
||||
"max_tokens": 10000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"output_cost_per_token_batches": 3.5e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -73330,6 +73569,7 @@
|
|||
]
|
||||
},
|
||||
"vertex_ai/veo-2.0-generate-001": {
|
||||
"deprecation_date": "2026-06-30",
|
||||
"litellm_provider": "vertex_ai-video-models",
|
||||
"max_input_tokens": 1024,
|
||||
"max_tokens": 1024,
|
||||
|
|
|
|||
BIN
litellm/proxy/_experimental/out/assets/aria-agent.png
Normal file
BIN
litellm/proxy/_experimental/out/assets/aria-agent.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 685 KiB |
|
|
@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
|
|||
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
|
||||
path_prefixes=("/vantage",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="usage_ai",
|
||||
module_path="litellm.proxy.management_endpoints.usage_endpoints",
|
||||
path_prefixes=("/usage/ai",),
|
||||
),
|
||||
LazyFeature(
|
||||
name="prompts",
|
||||
module_path="litellm.proxy.prompts.prompt_endpoints",
|
||||
|
|
|
|||
|
|
@ -57898,163 +57898,6 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"usage_ai": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
"ChatMessage": {
|
||||
"properties": {
|
||||
"content": {
|
||||
"title": "Content",
|
||||
"type": "string"
|
||||
},
|
||||
"role": {
|
||||
"enum": [
|
||||
"user",
|
||||
"assistant"
|
||||
],
|
||||
"title": "Role",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"role",
|
||||
"content"
|
||||
],
|
||||
"title": "ChatMessage",
|
||||
"type": "object"
|
||||
},
|
||||
"HTTPValidationError": {
|
||||
"properties": {
|
||||
"detail": {
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ValidationError"
|
||||
},
|
||||
"title": "Detail",
|
||||
"type": "array"
|
||||
}
|
||||
},
|
||||
"title": "HTTPValidationError",
|
||||
"type": "object"
|
||||
},
|
||||
"UsageAIChatRequest": {
|
||||
"properties": {
|
||||
"messages": {
|
||||
"description": "Chat messages (user/assistant history)",
|
||||
"items": {
|
||||
"$ref": "#/components/schemas/ChatMessage"
|
||||
},
|
||||
"title": "Messages",
|
||||
"type": "array"
|
||||
},
|
||||
"model": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
],
|
||||
"description": "Model to use for AI chat",
|
||||
"title": "Model"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"messages"
|
||||
],
|
||||
"title": "UsageAIChatRequest",
|
||||
"type": "object"
|
||||
},
|
||||
"ValidationError": {
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"title": "Context",
|
||||
"type": "object"
|
||||
},
|
||||
"input": {
|
||||
"title": "Input"
|
||||
},
|
||||
"loc": {
|
||||
"items": {
|
||||
"anyOf": [
|
||||
{
|
||||
"type": "string"
|
||||
},
|
||||
{
|
||||
"type": "integer"
|
||||
}
|
||||
]
|
||||
},
|
||||
"title": "Location",
|
||||
"type": "array"
|
||||
},
|
||||
"msg": {
|
||||
"title": "Message",
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"title": "Error Type",
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"loc",
|
||||
"msg",
|
||||
"type"
|
||||
],
|
||||
"title": "ValidationError",
|
||||
"type": "object"
|
||||
}
|
||||
}
|
||||
},
|
||||
"paths": {
|
||||
"/usage/ai/chat": {
|
||||
"post": {
|
||||
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
|
||||
"operationId": "usage_ai_chat_usage_ai_chat_post",
|
||||
"requestBody": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/UsageAIChatRequest"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": true
|
||||
},
|
||||
"responses": {
|
||||
"200": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {}
|
||||
}
|
||||
},
|
||||
"description": "Successful Response"
|
||||
},
|
||||
"422": {
|
||||
"content": {
|
||||
"application/json": {
|
||||
"schema": {
|
||||
"$ref": "#/components/schemas/HTTPValidationError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"description": "Validation Error"
|
||||
}
|
||||
},
|
||||
"security": [
|
||||
{
|
||||
"APIKeyHeader": []
|
||||
}
|
||||
],
|
||||
"summary": "Usage Ai Chat",
|
||||
"tags": [
|
||||
"usage_ai"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"vantage": {
|
||||
"components": {
|
||||
"schemas": {
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from typing import Final
|
|||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._version import version
|
||||
from litellm.constants import HIBP_RANGE_API_BASE
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException
|
||||
from litellm.types.llms.custom_http import httpxSpecialProvider
|
||||
|
|
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
|
|||
|
||||
|
||||
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
|
||||
return general_settings.get("password_policy_check_breached_passwords", True) is not False
|
||||
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
|
||||
|
||||
|
||||
async def is_password_breached(
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ import json
|
|||
import os
|
||||
import random
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
|
||||
from contextlib import asynccontextmanager
|
||||
from contextvars import ContextVar
|
||||
|
|
@ -91,6 +90,7 @@ from litellm.proxy.spend_tracking.savings import (
|
|||
)
|
||||
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
|
||||
from litellm.repositories.prisma_protocols import BatchTable
|
||||
from litellm.responses.streaming_iterator import spawn_logging_task
|
||||
from litellm.types.utils import CallTypes
|
||||
|
||||
if TYPE_CHECKING:
|
||||
|
|
@ -329,6 +329,38 @@ def _merged_spend_restore(first: _SpendRestore, second: _SpendRestore) -> _Spend
|
|||
return {**first, **second}
|
||||
|
||||
|
||||
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
|
||||
spend_log_error(
|
||||
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
|
||||
step_name,
|
||||
payload.get("request_id"),
|
||||
payload.get("api_key"),
|
||||
exc=exc,
|
||||
)
|
||||
|
||||
|
||||
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
|
||||
|
||||
|
||||
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
|
||||
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
|
||||
raw: Final = payload.get("metadata")
|
||||
if not isinstance(raw, str):
|
||||
return _EMPTY_SPEND_LOG_METADATA
|
||||
try:
|
||||
parsed: Final[SpendLogsMetadata] = json.loads(raw)
|
||||
except json.JSONDecodeError as e:
|
||||
spend_log_error(
|
||||
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
|
||||
"metadata-derived fields. request_id=%s api_key=%s",
|
||||
payload.get("request_id"),
|
||||
payload.get("api_key"),
|
||||
exc=e,
|
||||
)
|
||||
return _EMPTY_SPEND_LOG_METADATA
|
||||
return parsed
|
||||
|
||||
|
||||
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
|
||||
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
|
||||
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
|
||||
|
|
@ -663,8 +695,8 @@ class DBSpendUpdateWriter:
|
|||
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
|
||||
)
|
||||
|
||||
# Single task replaces 11 create_task() calls
|
||||
asyncio.create_task(
|
||||
spawn_logging_task(
|
||||
asyncio.get_running_loop(),
|
||||
self._batch_database_updates(
|
||||
response_cost=response_cost,
|
||||
user_id=user_id,
|
||||
|
|
@ -677,7 +709,8 @@ class DBSpendUpdateWriter:
|
|||
litellm_proxy_budget_name=litellm_proxy_budget_name,
|
||||
payload=payload,
|
||||
request_model_access_groups=get_request_model_access_groups(kwargs),
|
||||
)
|
||||
),
|
||||
task_name="spend rollups",
|
||||
)
|
||||
|
||||
self._enqueue_tool_registry_upsert(
|
||||
|
|
@ -1102,91 +1135,52 @@ class DBSpendUpdateWriter:
|
|||
project_id: str | None = None,
|
||||
):
|
||||
"""
|
||||
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
|
||||
|
||||
Each helper is wrapped in try/except so one failure doesn't prevent the others.
|
||||
|
||||
The deepcopy runs here, off the awaited request path, so the daily spend
|
||||
helpers get a payload isolated from the spend-log queue entry and the caller.
|
||||
Runs every spend-update helper for one request. Each helper goes through
|
||||
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
|
||||
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
|
||||
"""
|
||||
payload_copy: Final = copy.deepcopy(payload)
|
||||
request_tags: Final = payload_copy.get("request_tags")
|
||||
try:
|
||||
await self._update_user_db(
|
||||
await self._run_spend_update_step(
|
||||
"_update_user_db",
|
||||
self._update_user_db(
|
||||
response_cost=response_cost,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
litellm_proxy_budget_name=litellm_proxy_budget_name,
|
||||
end_user_id=end_user_id,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_user_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_key_db(
|
||||
response_cost=response_cost,
|
||||
hashed_token=hashed_token,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_key_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_team_db(
|
||||
response_cost=response_cost,
|
||||
team_id=team_id,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_team_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_org_db(
|
||||
response_cost=response_cost,
|
||||
org_id=org_id,
|
||||
user_id=user_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_org_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_project_db(
|
||||
response_cost=response_cost,
|
||||
project_id=project_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_project_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self._update_tag_db(
|
||||
response_cost=response_cost,
|
||||
request_tags=request_tags,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_tag_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_key_db",
|
||||
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_team_db",
|
||||
self._update_team_db(
|
||||
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_org_db",
|
||||
self._update_org_db(
|
||||
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_project_db",
|
||||
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_tag_db",
|
||||
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._update_model_access_group_db(
|
||||
response_cost=response_cost,
|
||||
request_model_access_groups=request_model_access_groups,
|
||||
|
|
@ -1194,96 +1188,64 @@ class DBSpendUpdateWriter:
|
|||
prisma_client=prisma_client,
|
||||
router=get_llm_router(),
|
||||
)
|
||||
|
||||
_agent_id_for_spend: Final = payload_copy.get("agent_id")
|
||||
try:
|
||||
await self._update_agent_db(
|
||||
response_cost=response_cost,
|
||||
agent_id=_agent_id_for_spend,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _update_agent_db failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_user_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_end_user_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_agent_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_team_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"_update_agent_db",
|
||||
self._update_agent_db(
|
||||
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_user_transaction",
|
||||
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_end_user_transaction",
|
||||
self.add_spend_log_transaction_to_daily_end_user_transaction(
|
||||
payload=payload_copy, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_agent_transaction",
|
||||
self.add_spend_log_transaction_to_daily_agent_transaction(
|
||||
payload=payload_copy, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_team_transaction",
|
||||
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
try:
|
||||
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
|
||||
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: _record_request_error failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_org_transaction(
|
||||
payload=payload_copy,
|
||||
org_id=org_id,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
try:
|
||||
await self.add_spend_log_transaction_to_daily_tag_transaction(
|
||||
payload=payload_copy,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.debug(
|
||||
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
|
||||
traceback.format_exc(),
|
||||
)
|
||||
|
||||
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
|
||||
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_org_transaction",
|
||||
self.add_spend_log_transaction_to_daily_org_transaction(
|
||||
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
|
||||
),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._run_spend_update_step(
|
||||
"add_spend_log_transaction_to_daily_tag_transaction",
|
||||
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
|
||||
payload=payload_copy,
|
||||
)
|
||||
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
|
||||
|
||||
@staticmethod
|
||||
async def _run_spend_update_step(
|
||||
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
|
||||
) -> None:
|
||||
try:
|
||||
await step
|
||||
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
|
||||
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
|
||||
|
||||
async def _update_key_db(
|
||||
self,
|
||||
response_cost: float | None,
|
||||
|
|
@ -2857,7 +2819,7 @@ class DBSpendUpdateWriter:
|
|||
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
|
||||
if not date:
|
||||
return
|
||||
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
metadata: Final = _parse_spend_log_metadata(payload)
|
||||
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
|
||||
request_error_accumulator.record(
|
||||
payload=payload,
|
||||
|
|
@ -2908,7 +2870,7 @@ class DBSpendUpdateWriter:
|
|||
# spend, token and per-entity columns written here stay either way.
|
||||
request_status: Final = prisma_client.get_request_status(payload)
|
||||
verbose_proxy_logger.debug("Logged request status: %s", request_status)
|
||||
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
|
||||
_metadata: Final = _parse_spend_log_metadata(payload)
|
||||
usage_obj: Final = _metadata.get("usage_object", {}) or {}
|
||||
if isinstance(payload["startTime"], datetime):
|
||||
start_time: Final = payload["startTime"].isoformat()
|
||||
|
|
|
|||
|
|
@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
|
|||
"/v1/messages",
|
||||
"responses",
|
||||
"files",
|
||||
"interactions",
|
||||
)
|
||||
|
||||
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
|
||||
|
|
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
|||
LlmProviders.VERTEX_AI.value,
|
||||
)
|
||||
)
|
||||
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
|
||||
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
|
||||
)
|
||||
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
|
||||
|
||||
|
||||
|
|
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
|
|||
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
|
||||
),
|
||||
ProviderSpecificHeader(
|
||||
custom_llm_provider=LlmProviders.DATABRICKS.value,
|
||||
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
|
||||
extra_headers=anthropic_api_headers,
|
||||
native_anthropic_messages_only=True,
|
||||
),
|
||||
|
|
|
|||
|
|
@ -2093,7 +2093,9 @@ async def get_user_key_counts(
|
|||
return result
|
||||
|
||||
|
||||
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
|
||||
def _validate_sort_params(
|
||||
sort_by: str | None, sort_order: str
|
||||
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
|
||||
order_by: Final[dict[str, str]] = {}
|
||||
|
||||
if sort_by is None:
|
||||
|
|
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
"spend",
|
||||
"user_alias",
|
||||
"user_role",
|
||||
"max_budget",
|
||||
"updated_at",
|
||||
"sso_user_id",
|
||||
"status",
|
||||
"key_count",
|
||||
]
|
||||
if sort_by not in valid_columns:
|
||||
raise HTTPException(
|
||||
|
|
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
|
||||
)
|
||||
|
||||
if sort_by in {"status", "key_count"}:
|
||||
return None
|
||||
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
|
||||
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
|
||||
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
|
||||
order_by[sort_by] = sort_order.lower()
|
||||
|
||||
return order_by
|
||||
|
|
@ -2338,16 +2350,40 @@ async def get_users(
|
|||
|
||||
# Build order_by conditions
|
||||
|
||||
order_by: Final[dict[str, str] | None] = (
|
||||
order_by: Final = (
|
||||
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
|
||||
)
|
||||
|
||||
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
|
||||
where=where,
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
|
||||
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
|
||||
|
||||
computed_sort: Final = sort_by in ("status", "key_count")
|
||||
sorted_ids: Final = (
|
||||
await sorted_user_ids(
|
||||
prisma_client,
|
||||
sort_by="status" if sort_by == "status" else "key_count",
|
||||
descending=sort_order.lower() == "desc",
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
role=role,
|
||||
user_ids=user_ids,
|
||||
user_email=user_email,
|
||||
team=team,
|
||||
sso_user_ids=sso_user_ids,
|
||||
organization_ids=organization_ids,
|
||||
search=search,
|
||||
)
|
||||
if computed_sort
|
||||
else ()
|
||||
)
|
||||
page_users: Final = await UserRepository(prisma_client).table.find_many(
|
||||
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
|
||||
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
|
||||
skip=0 if computed_sort else skip,
|
||||
take=page_size,
|
||||
order=order_by if order_by else {"created_at": "desc"},
|
||||
)
|
||||
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
|
||||
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
|
||||
|
||||
# Get total count of user rows
|
||||
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)
|
||||
|
|
|
|||
|
|
@ -7415,20 +7415,26 @@ async def key_aliases(
|
|||
)
|
||||
|
||||
|
||||
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
|
||||
order_by: Final[dict[str, str]] = {}
|
||||
|
||||
def _validate_sort_params(
|
||||
sort_by: str | None, sort_order: str
|
||||
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
|
||||
if sort_by is None:
|
||||
return None
|
||||
# Validate sort_by is a valid column
|
||||
valid_columns: Final = [
|
||||
valid_columns: Final = (
|
||||
"spend",
|
||||
"max_budget",
|
||||
"created_at",
|
||||
"updated_at",
|
||||
"token",
|
||||
"key_alias",
|
||||
]
|
||||
"expires",
|
||||
"last_active",
|
||||
"budget_reset_at",
|
||||
"total_spend",
|
||||
"tpm_limit",
|
||||
"rpm_limit",
|
||||
)
|
||||
if sort_by not in valid_columns:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
|
|
@ -7442,9 +7448,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
|
|||
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
|
||||
)
|
||||
|
||||
order_by[sort_by] = sort_order.lower()
|
||||
|
||||
return order_by
|
||||
direction: Final = sort_order.lower()
|
||||
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
|
||||
if sort_by not in (*nullable_sort_columns, "total_spend"):
|
||||
return {sort_by: direction}
|
||||
return [
|
||||
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
|
||||
{"token": "asc"},
|
||||
]
|
||||
|
||||
|
||||
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
|
||||
|
|
@ -7686,7 +7697,7 @@ async def list_key_helper(
|
|||
|
||||
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
|
||||
|
||||
order_by: Final[dict[str, str] | None] = (
|
||||
order_by: Final = (
|
||||
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Annotated, Final
|
||||
|
||||
|
|
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
|
|||
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
|
||||
|
||||
|
||||
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
|
||||
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰0𐒠𐴰𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑜰𑣠𑥐𑱐𑵐𑶠\U00011de0𑽐𖩠𖫀𖭐𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞥐🯰"
|
||||
_DECIMAL_DIGITS: Final = "".join(
|
||||
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
|
||||
)
|
||||
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
|
||||
|
||||
|
||||
_BUDGET_DURATION_SECONDS: Final = f"""(
|
||||
SELECT EXTRACT(EPOCH FROM (
|
||||
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
|
||||
(parts[1] || ' ' || CASE parts[2]
|
||||
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
|
||||
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
|
||||
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
|
||||
)) FROM regexp_match(
|
||||
CASE lower(trim(budget_duration))
|
||||
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
|
||||
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
|
||||
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
|
||||
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
|
||||
'^([0-9]+)(mo|[smhdw])'
|
||||
) AS duration(parts)
|
||||
)"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PrismaBudgetListExecutor:
|
||||
"""The database half of the budgets list. Every caller-supplied value is bound to a
|
||||
|
|
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
|
|||
|
||||
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
|
||||
clauses, params = where_sql(plan.where)
|
||||
ordering: Final = ", ".join(
|
||||
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
|
||||
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
|
||||
else order_by_sql((key,))
|
||||
for key in plan.order
|
||||
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
|
||||
sql: Final = (
|
||||
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
|
||||
+ (f" WHERE {clauses}" if clauses else "")
|
||||
+ f" ORDER BY {order_by_sql(plan.order)}"
|
||||
+ f" ORDER BY {ordering}"
|
||||
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
|
||||
)
|
||||
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
|
||||
|
|
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
|
|||
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
|
||||
|
||||
|
||||
# budget_duration is deliberately absent from `sortable`: the column holds strings
|
||||
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
|
||||
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
|
||||
{
|
||||
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
|
||||
|
|
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
|
|||
|
||||
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
|
||||
resource="budgets",
|
||||
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
|
||||
sortable=frozenset(
|
||||
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
|
||||
),
|
||||
searchable=frozenset(("budget_id",)),
|
||||
filters=BUDGET_FILTERS,
|
||||
default_sort=(SortKey(field="created_at", descending=True),),
|
||||
|
|
@ -154,8 +187,9 @@ async def list_budgets(
|
|||
way to page, sort or filter it.
|
||||
|
||||
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
|
||||
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
|
||||
and defaults to `-created_at`. `budget_id` is appended to every sort as the
|
||||
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
|
||||
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
|
||||
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
|
||||
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
|
||||
`page_size` defaults to 50 and is capped at 100. Filters are
|
||||
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.constants import (
|
|||
MAX_COMPETITOR_NAMES,
|
||||
)
|
||||
from litellm.integrations.custom_guardrail import CustomGuardrail
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.llms.openai.chat.guardrail_translation.handler import (
|
||||
OpenAIChatCompletionsHandler,
|
||||
)
|
||||
|
|
@ -657,7 +658,7 @@ async def get_policy_templates(
|
|||
Fetches from GitHub with automatic fallback to local backup on failure.
|
||||
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
|
||||
"""
|
||||
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
|
||||
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
|
||||
"true",
|
||||
"1",
|
||||
"yes",
|
||||
|
|
|
|||
|
|
@ -6117,13 +6117,52 @@ async def list_team_v2(
|
|||
"total_pages": 0,
|
||||
}
|
||||
|
||||
# Build order_by conditions
|
||||
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
|
||||
order_by = None
|
||||
if sort_by and sort_by in valid_sort_columns:
|
||||
if sort_order.lower() not in ["asc", "desc"]:
|
||||
sort_order = "asc"
|
||||
order_by = {sort_by: sort_order.lower()}
|
||||
from litellm.proxy.management_helpers.team_list_sorting import (
|
||||
COMPUTED_TEAM_SORTS,
|
||||
UnsupportedTeamFilter,
|
||||
sorted_team_ids,
|
||||
)
|
||||
|
||||
valid_sort_columns: Final = (
|
||||
"team_id",
|
||||
"team_alias",
|
||||
"created_at",
|
||||
"spend",
|
||||
"max_budget",
|
||||
"tpm_limit",
|
||||
"rpm_limit",
|
||||
"updated_at",
|
||||
)
|
||||
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
|
||||
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
|
||||
order_by: Final = (
|
||||
(
|
||||
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
|
||||
{"team_id": "asc"},
|
||||
)
|
||||
if nullable_sort
|
||||
else ({sort_by: sort_direction}, {"team_id": "asc"})
|
||||
if sort_by in ("spend", "updated_at")
|
||||
else {sort_by: sort_direction}
|
||||
if sort_by in valid_sort_columns
|
||||
else None
|
||||
)
|
||||
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
|
||||
sorted_ids: Final = (
|
||||
await sorted_team_ids(
|
||||
prisma_client,
|
||||
where=where_conditions,
|
||||
sort_by=sort_by or "",
|
||||
descending=sort_direction == "desc",
|
||||
skip=skip,
|
||||
take=page_size,
|
||||
)
|
||||
if computed_sort
|
||||
else ()
|
||||
)
|
||||
|
||||
if isinstance(sorted_ids, UnsupportedTeamFilter):
|
||||
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
|
||||
|
||||
# Get teams with pagination
|
||||
if use_deleted_table:
|
||||
|
|
@ -6138,26 +6177,32 @@ async def list_team_v2(
|
|||
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
|
||||
else:
|
||||
teams = await _team_db(prisma_client).find_many(
|
||||
where=where_conditions,
|
||||
skip=skip,
|
||||
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
|
||||
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
|
||||
skip=0 if computed_sort else skip,
|
||||
take=page_size,
|
||||
order=order_by if order_by else {"created_at": "desc"}, # Default sort
|
||||
order=order_by if order_by else {"created_at": "desc"},
|
||||
include=_INCLUDE_MODEL_TABLE,
|
||||
)
|
||||
# Get total count for pagination
|
||||
total_count = await _team_db(prisma_client).count(where=where_conditions)
|
||||
|
||||
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
|
||||
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
|
||||
|
||||
# Calculate total pages
|
||||
total_pages: Final = -(-total_count // page_size) # Ceiling division
|
||||
|
||||
# Aggregate virtual-key counts per team for the current page. The deleted
|
||||
# table does not carry keys_count, so it is skipped.
|
||||
keys_count_by_team: dict[str, int] = {}
|
||||
if not use_deleted_table:
|
||||
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
|
||||
keys_count_by_team: Final = (
|
||||
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
|
||||
)
|
||||
|
||||
# Convert Prisma models to response models with members_count and keys_count
|
||||
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
|
||||
team_list: Final = _convert_teams_to_response_models(
|
||||
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
|
||||
)
|
||||
|
||||
# Resolve resources inherited from access groups (single batch query)
|
||||
if not use_deleted_table:
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ from litellm.proxy.utils import (
|
|||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import SSOConfigRepository
|
||||
from litellm.repositories.team_repository import TeamRepository
|
||||
from litellm.repositories.user_repository import UserRepository
|
||||
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
|
||||
from litellm.secret_managers.main import get_secret_bool, get_secret_str
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
|
||||
|
|
@ -1085,25 +1085,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
|
|||
)
|
||||
|
||||
|
||||
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
|
||||
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
|
||||
if premium_user is True:
|
||||
return
|
||||
if prisma_client is None:
|
||||
raise ProxyException(
|
||||
message=CommonProxyErrors.db_not_connected_error.value,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="premium_user",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
|
||||
if billable_users and billable_users > 5:
|
||||
raise ProxyException(
|
||||
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param="premium_user",
|
||||
code=status.HTTP_403_FORBIDDEN,
|
||||
)
|
||||
class _SSOUserAdmissionError(ProxyException):
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
|
||||
|
|
@ -1123,8 +1106,6 @@ async def google_login(
|
|||
from litellm.proxy.proxy_server import (
|
||||
cli_sso_session_cache,
|
||||
general_settings,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
saml_sso_cache,
|
||||
user_custom_ui_sso_sign_in_handler,
|
||||
)
|
||||
|
|
@ -1139,15 +1120,6 @@ async def google_login(
|
|||
if admin_ui_is_disabled and not is_cli_sso_login:
|
||||
return admin_ui_disabled()
|
||||
|
||||
####### Check if user is a Enterprise / Premium User #######
|
||||
if (
|
||||
microsoft_client_id is not None
|
||||
or google_client_id is not None
|
||||
or generic_client_id is not None
|
||||
or SAMLAuthHandler.is_saml_configured()
|
||||
):
|
||||
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
|
||||
|
||||
####### Detect DB + MASTER KEY in .env #######
|
||||
missing_env_vars: Final = show_missing_vars_in_env()
|
||||
if missing_env_vars is not None:
|
||||
|
|
@ -1972,6 +1944,8 @@ async def get_user_info_from_db(
|
|||
)
|
||||
|
||||
return user_info
|
||||
except _SSOUserAdmissionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
|
||||
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
|
||||
|
|
@ -2337,7 +2311,6 @@ async def saml_callback(request: Request):
|
|||
general_settings,
|
||||
jwt_handler,
|
||||
master_key,
|
||||
premium_user,
|
||||
prisma_client,
|
||||
saml_sso_cache,
|
||||
)
|
||||
|
|
@ -2361,8 +2334,6 @@ async def saml_callback(request: Request):
|
|||
|
||||
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
|
||||
|
||||
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
|
||||
|
||||
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
|
||||
relay_state: Final = post_data.get("RelayState")
|
||||
cp_return_to: Final[str | None] = (
|
||||
|
|
@ -2826,12 +2797,20 @@ async def insert_sso_user(
|
|||
if result_openid and hasattr(result_openid, "provider"):
|
||||
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
|
||||
|
||||
response: Final = await new_user(
|
||||
data=new_user_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
from litellm.proxy.proxy_server import premium_user
|
||||
|
||||
return response
|
||||
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
|
||||
try:
|
||||
return await new_user(
|
||||
data=new_user_request,
|
||||
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
except ProxyException as error:
|
||||
if premium_user is True:
|
||||
raise
|
||||
raise _SSOUserAdmissionError(
|
||||
message=error.message, type=error.type, param=error.param, code=error.code
|
||||
) from error
|
||||
|
||||
|
||||
DefaultTeamJoin: TypeAlias = Callable[
|
||||
|
|
@ -3572,6 +3551,8 @@ class SSOAuthenticationHandler:
|
|||
prisma_client=prisma_client,
|
||||
)
|
||||
return user_info
|
||||
except _SSOUserAdmissionError:
|
||||
raise
|
||||
except Exception as e:
|
||||
if (
|
||||
user_info is None
|
||||
|
|
|
|||
|
|
@ -1,9 +0,0 @@
|
|||
"""
|
||||
Usage endpoints package.
|
||||
|
||||
Re-exports the router from endpoints module.
|
||||
"""
|
||||
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
|
||||
router,
|
||||
)
|
||||
|
|
@ -1,627 +0,0 @@
|
|||
"""
|
||||
AI Usage Chat - uses LLM tool calling to answer questions about
|
||||
usage/spend data by querying the aggregated daily activity endpoints.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
|
||||
from datetime import date
|
||||
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
|
||||
|
||||
from fastapi import HTTPException
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
from litellm.proxy._types import CommonProxyErrors
|
||||
from litellm.types.proxy.management_endpoints.common_daily_activity import (
|
||||
SpendAnalyticsPaginatedResponse,
|
||||
)
|
||||
from litellm.types.utils import ChatCompletionMessageToolCall
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
USAGE_AI_TEMPERATURE: Final = 0.2
|
||||
|
||||
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
|
||||
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
|
||||
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
|
||||
|
||||
ENTITY_FIELD_USER: Final = "user_id"
|
||||
ENTITY_FIELD_TEAM: Final = "team_id"
|
||||
ENTITY_FIELD_TAG: Final = "tag"
|
||||
|
||||
PAGINATED_PAGE_SIZE: Final = 200
|
||||
MAX_CHAT_MESSAGES: Final = 20
|
||||
TOP_N_MODELS: Final = 15
|
||||
TOP_N_PROVIDERS: Final = 10
|
||||
TOP_N_KEYS: Final = 10
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Types
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class SSEStatusEvent(TypedDict):
|
||||
type: Literal["status"]
|
||||
message: str
|
||||
|
||||
|
||||
class SSEToolCallEvent(TypedDict, total=False):
|
||||
type: Literal["tool_call"]
|
||||
tool_name: str
|
||||
tool_label: str
|
||||
arguments: dict[str, str]
|
||||
status: Literal["running", "complete", "error"]
|
||||
error: str
|
||||
|
||||
|
||||
class SSEChunkEvent(TypedDict):
|
||||
type: Literal["chunk"]
|
||||
content: str
|
||||
|
||||
|
||||
class SSEDoneEvent(TypedDict):
|
||||
type: Literal["done"]
|
||||
|
||||
|
||||
class SSEErrorEvent(TypedDict):
|
||||
type: Literal["error"]
|
||||
message: str
|
||||
|
||||
|
||||
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
|
||||
|
||||
|
||||
class _EntityEntry(TypedDict, total=False):
|
||||
metrics: ReadOnly[Mapping[str, float]]
|
||||
metadata: ReadOnly[Mapping[str, str]]
|
||||
|
||||
|
||||
class _DayDump(TypedDict, total=False):
|
||||
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
|
||||
|
||||
|
||||
class _EntityTotal(NamedTuple):
|
||||
"""Running per-entity totals accumulated while summarising a usage dump."""
|
||||
|
||||
alias: str
|
||||
spend: float
|
||||
requests: float
|
||||
tokens: float
|
||||
|
||||
|
||||
class _UsageDump(Protocol):
|
||||
@overload
|
||||
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
|
||||
@overload
|
||||
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
|
||||
|
||||
|
||||
class _ToolFunctionDef(TypedDict):
|
||||
name: ReadOnly[str]
|
||||
description: ReadOnly[str]
|
||||
parameters: ReadOnly[Mapping[str, object]]
|
||||
|
||||
|
||||
class _ToolDef(TypedDict):
|
||||
type: ReadOnly[str]
|
||||
function: ReadOnly[_ToolFunctionDef]
|
||||
|
||||
|
||||
class ToolHandler(TypedDict):
|
||||
fetch: Callable[..., Awaitable[_UsageDump]]
|
||||
summarise: Callable[[_UsageDump], str]
|
||||
label: str
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool definitions (OpenAI function-calling schema)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_DATE_PARAMS: Final = {
|
||||
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
|
||||
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
|
||||
}
|
||||
|
||||
_TOOL_USAGE: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_usage_data",
|
||||
"description": (
|
||||
"Fetch aggregated global usage/spend data. Returns daily spend, "
|
||||
"token counts, request counts, and breakdowns by model, provider, "
|
||||
"and API key. Use for overall spend, top models, top providers."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"user_id": {
|
||||
"type": "string",
|
||||
"description": "Optional user ID filter. Omit for global view.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_TOOL_TEAM: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_team_usage_data",
|
||||
"description": (
|
||||
"Fetch usage/spend data broken down by team. Use for questions "
|
||||
"like 'which team spends the most' or 'show me team X usage'."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"team_ids": {
|
||||
"type": "string",
|
||||
"description": "Optional comma-separated team IDs. Omit for all teams.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
_TOOL_TAG: Final[_ToolDef] = {
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_tag_usage_data",
|
||||
"description": (
|
||||
"Fetch usage/spend data broken down by tag. Tags are labels "
|
||||
"attached to requests (features, environments, credentials)."
|
||||
),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
**_DATE_PARAMS,
|
||||
"tags": {
|
||||
"type": "string",
|
||||
"description": "Optional comma-separated tag names. Omit for all tags.",
|
||||
},
|
||||
},
|
||||
"required": ["start_date", "end_date"],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
TOOLS_BASE: Final = [_TOOL_USAGE]
|
||||
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
|
||||
|
||||
|
||||
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
|
||||
"""Return the tool list appropriate for the user's role."""
|
||||
return TOOLS_ADMIN if is_admin else TOOLS_BASE
|
||||
|
||||
|
||||
_SYSTEM_PROMPT_BASE: Final = (
|
||||
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
|
||||
"You help users understand their LLM API spend and usage data.\n\n"
|
||||
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
|
||||
"You may call multiple tools if the question spans different dimensions.\n\n"
|
||||
"Guidelines:\n"
|
||||
"- Be concise and specific. Use exact numbers from the data.\n"
|
||||
"- Format costs as dollar amounts (e.g. $12.34).\n"
|
||||
"- When comparing entities, show a ranked list.\n"
|
||||
"- If data is empty or no results found, say so clearly.\n"
|
||||
"- Do not hallucinate data — only use what the tools return.\n"
|
||||
"- Today's date will be provided below. Use it to interpret relative dates "
|
||||
"like 'this week', 'this month', 'last 7 days', etc."
|
||||
)
|
||||
|
||||
_TOOL_DESCRIPTIONS_ADMIN: Final = (
|
||||
"You have access to these tools:\n"
|
||||
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
|
||||
"- `get_team_usage_data`: Team-level usage breakdown\n"
|
||||
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
|
||||
)
|
||||
|
||||
_TOOL_DESCRIPTIONS_BASE: Final = (
|
||||
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
|
||||
)
|
||||
|
||||
|
||||
def _build_system_prompt(is_admin: bool) -> str:
|
||||
"""Build role-appropriate system prompt with today's date."""
|
||||
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
|
||||
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
|
||||
|
||||
|
||||
# keep a public reference for test assertions
|
||||
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Data fetchers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _parse_csv_ids(raw: str | None) -> list[str] | None:
|
||||
if not raw:
|
||||
return None
|
||||
return [t.strip() for t in raw.split(",") if t.strip()]
|
||||
|
||||
|
||||
async def _query_activity(
|
||||
table_name: str,
|
||||
entity_id_field: str,
|
||||
entity_id: str | list[str] | None,
|
||||
start_date: str,
|
||||
end_date: str,
|
||||
*,
|
||||
use_aggregated: bool = False,
|
||||
) -> SpendAnalyticsPaginatedResponse:
|
||||
"""Shared helper that calls the daily activity query layer."""
|
||||
from litellm.proxy.management_endpoints.common_daily_activity import (
|
||||
daily_activity_repository,
|
||||
daily_activity_scope,
|
||||
get_daily_activity,
|
||||
get_daily_activity_aggregated,
|
||||
)
|
||||
from litellm.proxy.proxy_server import prisma_client
|
||||
|
||||
if use_aggregated:
|
||||
if prisma_client is None:
|
||||
raise HTTPException(
|
||||
status_code=500,
|
||||
detail={"error": CommonProxyErrors.db_not_connected_error.value},
|
||||
)
|
||||
repository: Final = daily_activity_repository(prisma_client)
|
||||
scope: Final = daily_activity_scope(
|
||||
table_name,
|
||||
entity_id_field,
|
||||
entity_id,
|
||||
None,
|
||||
None,
|
||||
start_date,
|
||||
end_date,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
return await get_daily_activity_aggregated(
|
||||
repository,
|
||||
scope,
|
||||
)
|
||||
return await get_daily_activity(
|
||||
prisma_client=prisma_client,
|
||||
table_name=table_name,
|
||||
entity_id_field=entity_id_field,
|
||||
entity_id=entity_id,
|
||||
entity_metadata_field=None,
|
||||
start_date=start_date,
|
||||
end_date=end_date,
|
||||
model=None,
|
||||
api_key=None,
|
||||
page=1,
|
||||
page_size=PAGINATED_PAGE_SIZE,
|
||||
)
|
||||
|
||||
|
||||
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_USER_SPEND,
|
||||
ENTITY_FIELD_USER,
|
||||
user_id,
|
||||
start_date,
|
||||
end_date,
|
||||
use_aggregated=True,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_TEAM_SPEND,
|
||||
ENTITY_FIELD_TEAM,
|
||||
_parse_csv_ids(team_ids),
|
||||
start_date,
|
||||
end_date,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
|
||||
resp: Final = await _query_activity(
|
||||
TABLE_DAILY_TAG_SPEND,
|
||||
ENTITY_FIELD_TAG,
|
||||
_parse_csv_ids(tags),
|
||||
start_date,
|
||||
end_date,
|
||||
)
|
||||
return resp.model_dump(mode="json")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Summarisers — convert raw JSON to concise text the LLM can reason over
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _accumulate_breakdown(
|
||||
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
|
||||
) -> dict[str, dict[str, float]]:
|
||||
"""Aggregate a single breakdown dimension across days."""
|
||||
totals: Final[dict[str, dict[str, float]]] = {}
|
||||
for day in results:
|
||||
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
|
||||
if key not in totals:
|
||||
totals[key] = {f: 0.0 for f in fields}
|
||||
m = entry.get("metrics", {})
|
||||
for f in fields:
|
||||
totals[key][f] += m.get(f, 0)
|
||||
return totals
|
||||
|
||||
|
||||
def _ranked_lines(
|
||||
totals: dict[str, dict[str, float]],
|
||||
fmt: Callable[[str, dict[str, float]], str],
|
||||
limit: int,
|
||||
) -> list[str]:
|
||||
"""Sort by spend descending, format each entry, and truncate."""
|
||||
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
|
||||
|
||||
|
||||
def _summarise_usage_data(data: _UsageDump) -> str:
|
||||
meta: Final = data.get("metadata", {})
|
||||
results: Final = data.get("results", [])
|
||||
|
||||
header: Final = (
|
||||
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
|
||||
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
|
||||
f"Successful: {meta.get('total_successful_requests', 0)} | "
|
||||
f"Failed: {meta.get('total_failed_requests', 0)}\n"
|
||||
f"Total Tokens: {meta.get('total_tokens', 0)}"
|
||||
)
|
||||
|
||||
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
|
||||
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
|
||||
|
||||
model_lines: Final = _ranked_lines(
|
||||
models,
|
||||
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
|
||||
TOP_N_MODELS,
|
||||
)
|
||||
provider_lines: Final = _ranked_lines(
|
||||
providers,
|
||||
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
|
||||
TOP_N_PROVIDERS,
|
||||
)
|
||||
|
||||
sections = [header, ""]
|
||||
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
|
||||
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
|
||||
return "\n".join(sections)
|
||||
|
||||
|
||||
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
|
||||
"""Summarise team/tag entity usage data."""
|
||||
results: Final = data.get("results", [])
|
||||
if not results:
|
||||
return f"No {entity_label} usage data found for the given date range."
|
||||
|
||||
totals: Final[dict[str, _EntityTotal]] = {}
|
||||
for day in results:
|
||||
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
|
||||
previous = totals.get(eid)
|
||||
m = entry.get("metrics", {})
|
||||
totals[eid] = _EntityTotal(
|
||||
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
|
||||
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
|
||||
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
|
||||
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
|
||||
)
|
||||
|
||||
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
|
||||
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
|
||||
label = d.alias if d.alias != eid else eid
|
||||
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Tool dispatch registry
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
|
||||
"get_usage_data": ToolHandler(
|
||||
fetch=_fetch_usage_data,
|
||||
summarise=_summarise_usage_data,
|
||||
label="global usage data",
|
||||
),
|
||||
"get_team_usage_data": ToolHandler(
|
||||
fetch=_fetch_team_usage_data,
|
||||
summarise=lambda data: _summarise_entity_data(data, "Team"),
|
||||
label="team usage data",
|
||||
),
|
||||
"get_tag_usage_data": ToolHandler(
|
||||
fetch=_fetch_tag_usage_data,
|
||||
summarise=lambda data: _summarise_entity_data(data, "Tag"),
|
||||
label="tag usage data",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# SSE streaming
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _sse(event: SSEEvent) -> str:
|
||||
return f"data: {json.dumps(event)}\n\n"
|
||||
|
||||
|
||||
def _resolve_fetch_kwargs(
|
||||
fn_name: str,
|
||||
fn_args: Mapping[str, str],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> dict[str, str]:
|
||||
"""Build keyword arguments for a tool's fetch function."""
|
||||
start_date: Final = fn_args.get("start_date", "")
|
||||
end_date: Final = fn_args.get("end_date", "")
|
||||
if not start_date or not end_date:
|
||||
raise ValueError("Missing required start_date or end_date from tool arguments")
|
||||
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
|
||||
if fn_name == "get_usage_data":
|
||||
if not is_admin:
|
||||
if user_id is None:
|
||||
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
|
||||
# should have already rejected this. If we ever reach here it means
|
||||
# a future caller invoked the helper without scoping — fail loudly
|
||||
# rather than issuing an unfiltered global query.
|
||||
raise ValueError(
|
||||
"Non-admin caller has user_id=None; refusing to issue an "
|
||||
"unscoped query. Endpoint-level guard missing."
|
||||
)
|
||||
kwargs["user_id"] = user_id
|
||||
elif fn_args.get("user_id"):
|
||||
kwargs["user_id"] = fn_args["user_id"]
|
||||
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
|
||||
kwargs["team_ids"] = fn_args["team_ids"]
|
||||
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
|
||||
kwargs["tags"] = fn_args["tags"]
|
||||
return kwargs
|
||||
|
||||
|
||||
async def _execute_tool_call(
|
||||
handler: ToolHandler,
|
||||
fn_name: str,
|
||||
fn_args: Mapping[str, str],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> str:
|
||||
"""Run a single tool and return the summarised result text, or the reason the query
|
||||
layer rejected its dates so the model can correct them."""
|
||||
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
|
||||
try:
|
||||
raw_data: Final = await handler["fetch"](**kwargs)
|
||||
except HTTPException as e:
|
||||
reason: Final = _invalid_date_reason(e)
|
||||
if reason is None:
|
||||
raise
|
||||
return f"Invalid date range: {reason}"
|
||||
return handler["summarise"](raw_data)
|
||||
|
||||
|
||||
def _invalid_date_reason(error: HTTPException) -> str | None:
|
||||
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
|
||||
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
|
||||
match detail:
|
||||
case {"error": str() as reason} if error.status_code == 400:
|
||||
return reason
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
async def _process_tool_call(
|
||||
tc: ChatCompletionMessageToolCall,
|
||||
chat_messages: list[Mapping[str, object]],
|
||||
user_id: str | None,
|
||||
is_admin: bool,
|
||||
) -> AsyncIterator[str]:
|
||||
"""Execute a single tool call, yielding SSE events for status."""
|
||||
fn_name: Final = tc.function.name
|
||||
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
|
||||
|
||||
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
|
||||
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
|
||||
|
||||
if fn_name is None or fn_name not in allowed_names or not handler:
|
||||
chat_messages.append(
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": tc.id,
|
||||
"content": f"Tool not available: {fn_name}",
|
||||
}
|
||||
)
|
||||
return
|
||||
|
||||
tool_event_base: Final = {
|
||||
"type": "tool_call",
|
||||
"tool_name": fn_name,
|
||||
"tool_label": handler["label"],
|
||||
"arguments": fn_args,
|
||||
}
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
|
||||
|
||||
try:
|
||||
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
|
||||
tool_result = f"Error fetching {handler['label']}. Please try again."
|
||||
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
|
||||
|
||||
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
|
||||
|
||||
|
||||
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
|
||||
"""Stream the final LLM response after tool results are appended."""
|
||||
yield _sse({"type": "status", "message": "Analyzing results..."})
|
||||
|
||||
response: Final = await litellm.acompletion(
|
||||
model=model,
|
||||
messages=chat_messages,
|
||||
stream=True,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
)
|
||||
async for chunk in response:
|
||||
delta = chunk.choices[0].delta.content
|
||||
if delta:
|
||||
yield _sse({"type": "chunk", "content": delta})
|
||||
|
||||
|
||||
async def stream_usage_ai_chat(
|
||||
messages: list[dict[str, str]],
|
||||
model: str | None = None,
|
||||
user_id: str | None = None,
|
||||
is_admin: bool = False,
|
||||
) -> AsyncGenerator[str, None]:
|
||||
"""Stream SSE events: status → tool_call → chunk → done."""
|
||||
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
|
||||
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
|
||||
chat_messages: Final[list[Mapping[str, object]]] = [
|
||||
{"role": "system", "content": _build_system_prompt(is_admin)},
|
||||
*truncated,
|
||||
]
|
||||
|
||||
try:
|
||||
yield _sse({"type": "status", "message": "Thinking..."})
|
||||
tools: Final = get_tools_for_role(is_admin)
|
||||
response: Final = await litellm.acompletion(
|
||||
model=resolved_model,
|
||||
messages=chat_messages,
|
||||
tools=tools,
|
||||
temperature=USAGE_AI_TEMPERATURE,
|
||||
)
|
||||
choice: Final = response.choices[0]
|
||||
|
||||
if not choice.message.tool_calls:
|
||||
if choice.message.content:
|
||||
yield _sse({"type": "chunk", "content": choice.message.content})
|
||||
yield _sse({"type": "done"})
|
||||
return
|
||||
|
||||
chat_messages.append(choice.message.model_dump())
|
||||
for tc in choice.message.tool_calls:
|
||||
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
|
||||
yield event
|
||||
async for event in _stream_final_response(resolved_model, chat_messages):
|
||||
yield event
|
||||
yield _sse({"type": "done"})
|
||||
|
||||
except Exception as e:
|
||||
verbose_proxy_logger.error("AI usage chat failed: %s", e)
|
||||
yield _sse(
|
||||
{
|
||||
"type": "error",
|
||||
"message": "An internal error occurred. Please try again.",
|
||||
}
|
||||
)
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
"""
|
||||
USAGE AI CHAT ENDPOINTS
|
||||
|
||||
/usage/ai/chat - Stream AI chat responses about usage data
|
||||
"""
|
||||
|
||||
from typing import Final, Literal
|
||||
|
||||
from fastapi import APIRouter, Depends, Request
|
||||
from fastapi.responses import StreamingResponse
|
||||
from pydantic import Field
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.proxy.common_utils.sse_keepalive import (
|
||||
SSE_COMMENT_PING,
|
||||
wrap_sse_stream_with_keepalive_pings,
|
||||
)
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
router: Final = APIRouter()
|
||||
|
||||
|
||||
class ChatMessage(LiteLLMBaseModel):
|
||||
role: Literal["user", "assistant"]
|
||||
content: str
|
||||
|
||||
|
||||
class UsageAIChatRequest(LiteLLMBaseModel):
|
||||
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
|
||||
model: str | None = Field(default=None, description="Model to use for AI chat")
|
||||
|
||||
|
||||
@router.post(
|
||||
"/usage/ai/chat",
|
||||
tags=["Budget & Spend Tracking"],
|
||||
dependencies=[Depends(user_api_key_auth)],
|
||||
)
|
||||
async def usage_ai_chat(
|
||||
data: UsageAIChatRequest,
|
||||
request: Request,
|
||||
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
|
||||
):
|
||||
"""
|
||||
AI chat about usage data. Streams SSE events with the AI response.
|
||||
The AI agent has access to tools that query aggregated daily activity data.
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.common_utils import (
|
||||
require_caller_user_id_for_non_admin,
|
||||
user_api_key_has_admin_view,
|
||||
)
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
stream_usage_ai_chat,
|
||||
)
|
||||
|
||||
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
|
||||
if is_admin:
|
||||
user_id = user_api_key_dict.user_id
|
||||
else:
|
||||
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
|
||||
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
|
||||
|
||||
return StreamingResponse(
|
||||
wrap_sse_stream_with_keepalive_pings(
|
||||
stream_usage_ai_chat(
|
||||
messages=messages,
|
||||
model=data.model,
|
||||
user_id=user_id,
|
||||
is_admin=is_admin,
|
||||
),
|
||||
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
|
||||
ping_chunk=SSE_COMMENT_PING,
|
||||
),
|
||||
media_type="text/event-stream",
|
||||
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
|
||||
)
|
||||
109
litellm/proxy/management_helpers/team_list_sorting.py
Normal file
109
litellm/proxy/management_helpers/team_list_sorting.py
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
|
||||
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
|
||||
{
|
||||
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
|
||||
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
|
||||
"NULLIF(t.organization_id, ''))",
|
||||
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
|
||||
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
|
||||
"models": "cardinality(t.models)",
|
||||
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
|
||||
}
|
||||
)
|
||||
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class UnsupportedTeamFilter:
|
||||
field: str
|
||||
|
||||
|
||||
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
|
||||
if field not in ("team_id", "team_alias", "organization_id"):
|
||||
return UnsupportedTeamFilter(field)
|
||||
column: Final = f't."{field}"'
|
||||
match value:
|
||||
case str(text):
|
||||
return f"{column} = ${index}", (text,)
|
||||
case {"in": list(values)}:
|
||||
return f"{column} = ANY(${index}::text[])", (values,)
|
||||
case {"contains": str(text), "mode": "insensitive"}:
|
||||
return f"{column} ILIKE ${index}", (f"%{text}%",)
|
||||
case {"startsWith": str(text)}:
|
||||
return f"{column} LIKE ${index}", (f"{text}%",)
|
||||
case _:
|
||||
return UnsupportedTeamFilter(field)
|
||||
|
||||
|
||||
def _group_tokens(
|
||||
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
|
||||
) -> tuple[str | tuple[str, JsonValue], ...]:
|
||||
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
|
||||
if not children:
|
||||
return ("FALSE" if field == "OR" else "TRUE",)
|
||||
return (
|
||||
"(",
|
||||
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
|
||||
")",
|
||||
)
|
||||
|
||||
|
||||
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
|
||||
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
|
||||
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
|
||||
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
|
||||
while pending:
|
||||
match pending.pop():
|
||||
case str(fragment):
|
||||
fragments.append(fragment)
|
||||
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
|
||||
pending.extend(reversed(_group_tokens(field, value)))
|
||||
case (field, value):
|
||||
match _field_sql(field, value, len(params) + 1):
|
||||
case UnsupportedTeamFilter() as failure:
|
||||
return failure
|
||||
case (fragment, values):
|
||||
fragments.append(fragment)
|
||||
params.extend(values)
|
||||
return "".join(fragments), tuple(params)
|
||||
|
||||
|
||||
class _TeamId(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
team_id: str
|
||||
|
||||
|
||||
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
|
||||
|
||||
|
||||
async def sorted_team_ids(
|
||||
prisma_client: PrismaClient,
|
||||
*,
|
||||
where: Mapping[str, object],
|
||||
sort_by: str,
|
||||
descending: bool,
|
||||
skip: int,
|
||||
take: int,
|
||||
) -> Sequence[str] | UnsupportedTeamFilter:
|
||||
result: Final = _where_sql(_WHERE.validate_python(where))
|
||||
if isinstance(result, UnsupportedTeamFilter):
|
||||
return result
|
||||
predicate, params = result
|
||||
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
|
||||
direction: Final = "DESC" if descending else "ASC"
|
||||
sql: Final = (
|
||||
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
|
||||
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
|
||||
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
|
||||
)
|
||||
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
|
||||
return tuple(row.team_id for row in rows)
|
||||
77
litellm/proxy/management_helpers/user_list_sorting.py
Normal file
77
litellm/proxy/management_helpers/user_list_sorting.py
Normal file
|
|
@ -0,0 +1,77 @@
|
|||
from collections.abc import Sequence
|
||||
from typing import Final, Literal
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
class _UserId(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
user_id: str
|
||||
|
||||
|
||||
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
|
||||
|
||||
|
||||
async def sorted_user_ids(
|
||||
prisma_client: PrismaClient,
|
||||
*,
|
||||
sort_by: Literal["status", "key_count"],
|
||||
descending: bool,
|
||||
skip: int,
|
||||
take: int,
|
||||
role: str | None,
|
||||
user_ids: str | None,
|
||||
user_email: str | None,
|
||||
team: str | None,
|
||||
sso_user_ids: str | None,
|
||||
organization_ids: str | None,
|
||||
search: str | None,
|
||||
) -> Sequence[str]:
|
||||
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
|
||||
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
|
||||
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
|
||||
expression: Final = (
|
||||
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
|
||||
if sort_by == "status"
|
||||
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
|
||||
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
|
||||
)
|
||||
direction: Final = "DESC" if descending else "ASC"
|
||||
sql: Final = f"""
|
||||
SELECT u.user_id FROM "LiteLLM_UserTable" u
|
||||
WHERE ($1::text IS NULL OR u.user_role = $1)
|
||||
AND ($2::text[] IS NULL
|
||||
OR CASE WHEN cardinality($2::text[]) = 1
|
||||
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
|
||||
ELSE u.user_id = ANY($2::text[]) END)
|
||||
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
|
||||
AND ($4::text IS NULL OR $4 = ANY(u.teams))
|
||||
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
|
||||
AND (cardinality($7::text[]) = 0 OR EXISTS (
|
||||
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
|
||||
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
|
||||
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
|
||||
OR u.user_email ILIKE '%' || $8 || '%')
|
||||
ORDER BY {expression} {direction}, u.user_id ASC
|
||||
LIMIT $9 OFFSET $10
|
||||
"""
|
||||
rows: Final = _USER_IDS.validate_python(
|
||||
await prisma_client.db.query_raw(
|
||||
sql,
|
||||
role or None,
|
||||
ids if user_ids else None,
|
||||
user_email,
|
||||
team,
|
||||
sso_user_ids is None,
|
||||
sso_ids,
|
||||
org_ids,
|
||||
search or None,
|
||||
take,
|
||||
skip,
|
||||
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
|
||||
)
|
||||
)
|
||||
return tuple(row.user_id for row in rows)
|
||||
|
|
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
|
|||
get_blog_posts,
|
||||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.proxy._types import (
|
||||
CommonProxyErrors,
|
||||
)
|
||||
|
|
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
|
|||
url: str,
|
||||
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
|
||||
) -> Mapping[str, AutoRouterPresetRecord]:
|
||||
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
|
||||
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
|
||||
return _load_bundled_autorouter_presets()
|
||||
try:
|
||||
return await fetch(url)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import xml.etree.ElementTree as ET
|
||||
from collections import Counter
|
||||
|
|
@ -13,6 +14,7 @@ from pydantic import ValidationError
|
|||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.caching.in_memory_cache import InMemoryCache
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
from litellm.types.llms.base import LiteLLMBaseModel
|
||||
|
||||
|
|
@ -88,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
|
|||
return _latest_release_fetch_lock
|
||||
|
||||
|
||||
def _default_release_check_enabled() -> bool:
|
||||
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
|
||||
|
||||
|
||||
def _bucket_for(line: str) -> _Bucket | None:
|
||||
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
|
||||
return None
|
||||
|
|
@ -224,12 +230,16 @@ async def latest_release_info(
|
|||
client: Annotated[_AsyncGetClient, Depends(_default_client)],
|
||||
cache: Annotated[InMemoryCache, Depends(_default_cache)],
|
||||
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
|
||||
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
|
||||
) -> LatestReleaseInfo | None:
|
||||
"""
|
||||
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
|
||||
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
|
||||
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
|
||||
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
|
||||
"""
|
||||
if not release_check_enabled:
|
||||
return None
|
||||
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
|
||||
if isinstance(result, LatestReleaseUnavailable):
|
||||
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)
|
||||
|
|
|
|||
|
|
@ -6071,13 +6071,12 @@ class PrismaClient:
|
|||
elif table_name == "user":
|
||||
db_data = self.jsonify_object(data=data)
|
||||
try:
|
||||
new_user_row: Final = await UserRepository(self).table.upsert(
|
||||
where={"user_id": data["user_id"]},
|
||||
data={
|
||||
"create": {**db_data},
|
||||
"update": {}, # don't do anything if it already exists
|
||||
},
|
||||
)
|
||||
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
|
||||
if new_user_row is None:
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
|
||||
)
|
||||
except Exception as e:
|
||||
if (
|
||||
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"
|
||||
|
|
|
|||
|
|
@ -2,8 +2,11 @@
|
|||
User repository for database operations on LiteLLM_UserTable.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Generator, Mapping, Sequence
|
||||
from contextlib import AbstractAsyncContextManager, contextmanager
|
||||
from contextvars import ContextVar
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final, Protocol
|
||||
|
||||
|
|
@ -35,6 +38,16 @@ ORDER BY p.user_id
|
|||
"""
|
||||
|
||||
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
|
||||
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
|
||||
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_FREE_SSO_CREATION.reset(token)
|
||||
|
||||
|
||||
class _UserDb(Protocol):
|
||||
|
|
@ -42,6 +55,12 @@ class _UserDb(Protocol):
|
|||
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
|
||||
|
||||
|
||||
class _UserTransaction(_UserDb, Protocol):
|
||||
async def execute_raw(self, query: str) -> int: ...
|
||||
|
||||
async def query_raw(self, query: str) -> object: ...
|
||||
|
||||
|
||||
class _PrismaClientView(Protocol):
|
||||
@property
|
||||
def db(self) -> _UserDb: ...
|
||||
|
|
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
|
|||
@property
|
||||
def writer_db(self) -> _UserDb: ...
|
||||
|
||||
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
|
||||
|
||||
|
||||
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
|
||||
from prisma import Json
|
||||
|
||||
total: Final = await table.count()
|
||||
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
|
||||
return max(0, total - deactivated)
|
||||
|
||||
|
||||
class UserRepository(BaseRepository[LiteLLM_UserTable]):
|
||||
"""Repository for user database operations."""
|
||||
|
|
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
|
|||
null, or true all count, so seats are counted as total users minus
|
||||
the deactivated ones.
|
||||
"""
|
||||
from prisma import Json # pyright: ignore[reportUnknownVariableType]
|
||||
return await _count_billable_users(self.table)
|
||||
|
||||
total: Final = await self.count()
|
||||
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
|
||||
return max(0, total - deactivated)
|
||||
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
|
||||
"""Return the user row, or None when a new free SSO user would exceed capacity."""
|
||||
scope: Final = _FREE_SSO_CREATION.get()
|
||||
if (
|
||||
scope is None
|
||||
or (scope[0] is not None and scope[0] != data["user_id"])
|
||||
or scope[1] is not asyncio.current_task()
|
||||
):
|
||||
return await self.table.upsert(
|
||||
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
|
||||
)
|
||||
|
||||
client: Final[_PrismaClientView] = self.prisma_client
|
||||
async with client.tx() as transaction:
|
||||
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
|
||||
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
|
||||
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
|
||||
if existing is not None:
|
||||
return existing
|
||||
if await _count_billable_users(transaction.litellm_usertable) >= 5:
|
||||
return None
|
||||
return await transaction.litellm_usertable.upsert(
|
||||
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
|
||||
)
|
||||
|
||||
async def create_user(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
|
|||
from litellm.litellm_core_utils.asyncify import run_async_function
|
||||
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
|
||||
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
|
||||
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
|
|
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
|
||||
from litellm.responses.litellm_completion_transformation.compaction_items import (
|
||||
with_compaction_items_as_summaries,
|
||||
|
|
@ -1467,6 +1469,12 @@ def responses(
|
|||
|
||||
# get llm provider logic
|
||||
litellm_params: Final = GenericLiteLLMParams(**kwargs)
|
||||
affinity_api_base: Final = (
|
||||
get_openai_credentials(api_base=litellm_params.api_base).api_base
|
||||
if custom_llm_provider == "openai"
|
||||
else litellm_params.api_base
|
||||
)
|
||||
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
|
||||
try:
|
||||
effective_extra_headers: Final = (
|
||||
add_provider_affinity_header(
|
||||
|
|
@ -1480,8 +1488,9 @@ def responses(
|
|||
"litellm_metadata": kwargs.get("litellm_metadata"),
|
||||
}
|
||||
),
|
||||
default_header=default_affinity_header,
|
||||
)
|
||||
if litellm_params.provider_affinity_header is not None
|
||||
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
|
||||
else extra_headers
|
||||
)
|
||||
except ValueError as affinity_error:
|
||||
|
|
|
|||
|
|
@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
|||
}
|
||||
|
||||
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
|
||||
Feature.PROVIDER_AFFINITY,
|
||||
Feature.SAFEGUARDS_CLASSIFIER,
|
||||
Feature.PRE_REQUEST_HOOK,
|
||||
Feature.WEBSEARCH_INTERCEPTION,
|
||||
|
|
|
|||
|
|
@ -5,10 +5,15 @@ from __future__ import annotations
|
|||
from typing import Final
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.detectors import overridden, present, streaming
|
||||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
|
||||
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
|
||||
|
||||
|
||||
def provider_affinity(request: NativeCall) -> bool:
|
||||
return is_opencode_api_base(api_base(request))
|
||||
|
||||
|
||||
def anthropic_workload_identity(request: NativeCall) -> bool:
|
||||
"""Python mints an Anthropic federation token when no static credential is configured."""
|
||||
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay
|
|||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm.llms.opencode.common_utils import is_opencode_api_base
|
||||
from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present
|
||||
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
|
||||
|
||||
|
|
@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool:
|
|||
|
||||
def provider_affinity(request: NativeCall) -> bool:
|
||||
"""Python validates and fills the affinity header from the session ID."""
|
||||
return present(request, "provider_affinity_header")
|
||||
return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request))
|
||||
|
||||
|
||||
def text_format(request: NativeCall) -> bool:
|
||||
|
|
|
|||
0
litellm/text_completions/__init__.py
Normal file
0
litellm/text_completions/__init__.py
Normal file
88
litellm/text_completions/streaming_iterator.py
Normal file
88
litellm/text_completions/streaming_iterator.py
Normal file
|
|
@ -0,0 +1,88 @@
|
|||
from typing import Final
|
||||
|
||||
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
completion_stream,
|
||||
model,
|
||||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse):
|
||||
try:
|
||||
response: Final = TextCompletionResponse()
|
||||
response["id"] = chunk.get("id", None)
|
||||
response["object"] = "text_completion"
|
||||
response["created"] = chunk.get("created", None)
|
||||
response["model"] = chunk.get("model", None)
|
||||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
response.usage = chunk.get("usage", None)
|
||||
else:
|
||||
response.usage = None
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
TextCompletionResponse()
|
||||
try:
|
||||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
except Exception as e:
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
|
||||
raise exception_type(
|
||||
model=self.model,
|
||||
custom_llm_provider=self.custom_llm_provider or "",
|
||||
original_exception=e,
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
async def __anext__(self):
|
||||
try:
|
||||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
103
litellm/utils.py
103
litellm/utils.py
|
|
@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
|
|||
)
|
||||
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
|
||||
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
|
||||
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
|
||||
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
|
||||
|
|
@ -218,6 +219,7 @@ def _get_cached_audio_utils():
|
|||
return _audio_utils_module
|
||||
|
||||
|
||||
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper
|
||||
from litellm.types.llms.openai import (
|
||||
AllMessageValues,
|
||||
AllPromptValues,
|
||||
|
|
@ -262,13 +264,17 @@ from litellm.types.utils import (
|
|||
SearchProviders,
|
||||
SelectTokenizerResponse,
|
||||
StreamingChoices,
|
||||
TextChoices,
|
||||
TextCompletionResponse,
|
||||
TranscriptionResponse,
|
||||
Usage,
|
||||
credential_api_key_secret_reference,
|
||||
is_litellm_owned_kwarg,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
TextChoices as _TextChoices,
|
||||
)
|
||||
from litellm.types.utils import (
|
||||
TextCompletionResponse as _TextCompletionResponse,
|
||||
)
|
||||
|
||||
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
|
||||
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
|
||||
|
|
@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError):
|
|||
claude_json_str = json.dumps(json_data)
|
||||
import importlib.metadata
|
||||
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
|
||||
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
|
||||
|
||||
from typing_extensions import assert_never
|
||||
|
||||
TextChoices: TypeAlias = _TextChoices
|
||||
TextCompletionResponse: TypeAlias = _TextCompletionResponse
|
||||
TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper
|
||||
|
||||
from litellm import utils as litellm_utils
|
||||
|
||||
# These are lazy loaded via __getattr__
|
||||
|
|
@ -340,7 +350,8 @@ if TYPE_CHECKING:
|
|||
|
||||
# Type stubs for lazy-loaded functions to help mypy understand their types
|
||||
# These imports allow mypy to understand the types when these are accessed via __getattr__
|
||||
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
|
||||
exception_type: Callable[..., Exception]
|
||||
|
||||
from litellm.litellm_core_utils.get_litellm_params import (
|
||||
get_base_model_from_litellm_call_metadata,
|
||||
get_litellm_params,
|
||||
|
|
@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
|
|||
|
||||
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
|
||||
kind: Final = huggingface_tokenizer_kind(model)
|
||||
if kind is None:
|
||||
if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()):
|
||||
return None
|
||||
backend: Final = _huggingface_tokenizer_backend()
|
||||
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
|
||||
|
|
@ -7945,88 +7956,6 @@ def register_prompt_template(
|
|||
return litellm.custom_prompt_dict
|
||||
|
||||
|
||||
class TextCompletionStreamWrapper:
|
||||
def __init__(
|
||||
self,
|
||||
completion_stream,
|
||||
model,
|
||||
stream_options: dict | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
):
|
||||
self.completion_stream = completion_stream
|
||||
self.model = model
|
||||
self.stream_options = stream_options
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
|
||||
def __iter__(self):
|
||||
return self
|
||||
|
||||
def __aiter__(self):
|
||||
return self
|
||||
|
||||
def convert_to_text_completion_object(self, chunk: ModelResponse):
|
||||
try:
|
||||
response: Final = TextCompletionResponse()
|
||||
response["id"] = chunk.get("id", None)
|
||||
response["object"] = "text_completion"
|
||||
response["created"] = chunk.get("created", None)
|
||||
response["model"] = chunk.get("model", None)
|
||||
text_choices: Final = TextChoices()
|
||||
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
|
||||
raise Exception
|
||||
|
||||
if chunk["choices"]:
|
||||
delta: Final = chunk["choices"][0]["delta"]
|
||||
text_choices["text"] = delta["content"]
|
||||
text_choices["reasoning_content"] = delta.get("reasoning_content")
|
||||
text_choices["index"] = chunk["choices"][0]["index"]
|
||||
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
|
||||
response["choices"] = [text_choices]
|
||||
else:
|
||||
response["choices"] = []
|
||||
|
||||
# only pass usage when stream_options["include_usage"] is True
|
||||
if self.stream_options and self.stream_options.get("include_usage", False) is True:
|
||||
response["usage"] = chunk.get("usage", None)
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
|
||||
|
||||
def __next__(self):
|
||||
# model_response = ModelResponse(stream=True, model=self.model)
|
||||
TextCompletionResponse()
|
||||
try:
|
||||
for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopIteration
|
||||
except Exception as e:
|
||||
exception_type: Final = litellm_utils.exception_type
|
||||
raise exception_type(
|
||||
model=self.model,
|
||||
custom_llm_provider=self.custom_llm_provider or "",
|
||||
original_exception=e,
|
||||
completion_kwargs={},
|
||||
extra_kwargs={},
|
||||
)
|
||||
|
||||
async def __anext__(self):
|
||||
try:
|
||||
async for chunk in self.completion_stream:
|
||||
if chunk == "None" or chunk is None:
|
||||
raise Exception
|
||||
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
|
||||
return processed_chunk
|
||||
raise StopIteration
|
||||
except StopIteration:
|
||||
raise StopAsyncIteration
|
||||
|
||||
|
||||
def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream:
|
||||
return ModelResponseStream(
|
||||
id=model_response.id,
|
||||
|
|
|
|||
|
|
@ -12827,6 +12827,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 16384,
|
||||
|
|
@ -12840,6 +12841,7 @@
|
|||
"supports_vision": false
|
||||
},
|
||||
"azure_ai/Phi-4-mini-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12851,6 +12853,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_audio_token": 4e-06,
|
||||
"input_cost_per_token": 8e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
|
|
@ -12865,6 +12868,7 @@
|
|||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 7.5e-08,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 131072,
|
||||
|
|
@ -12876,6 +12880,7 @@
|
|||
"supports_function_calling": true
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
"input_cost_per_token": 1.25e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 32768,
|
||||
|
|
@ -24892,27 +24897,118 @@
|
|||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_multilingual_v2": {
|
||||
"input_cost_per_character": 0.00018,
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4": {
|
||||
"input_cost_per_character": 8e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v4_turbo": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_v3_conversational": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_flash_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2_5": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
},
|
||||
"elevenlabs/eleven_turbo_v2": {
|
||||
"input_cost_per_character": 4e-05,
|
||||
"litellm_provider": "elevenlabs",
|
||||
"metadata": {
|
||||
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
|
||||
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
|
||||
},
|
||||
"mode": "audio_speech",
|
||||
"source": "https://elevenlabs.io/pricing/api",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
]
|
||||
|
|
@ -42337,6 +42433,134 @@
|
|||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-3-0": {
|
||||
"input_cost_per_second": 5.83333e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-small-transcribe-3-0": {
|
||||
"input_cost_per_second": 6.66667e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-transcribe-realtime-3-0": {
|
||||
"input_cost_per_second": 0.000133333,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_transcription",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/transcriptions"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_audio_input": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-mini-tts-3-0": {
|
||||
"input_cost_per_character": 1.3e-05,
|
||||
"litellm_provider": "mistral",
|
||||
"mode": "audio_speech",
|
||||
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
|
||||
"supported_endpoints": [
|
||||
"/v1/audio/speech"
|
||||
],
|
||||
"supported_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supported_output_modalities": [
|
||||
"audio"
|
||||
],
|
||||
"supports_audio_output": true
|
||||
},
|
||||
"mistral/voxtral-small-2507": {
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
|
|
@ -48499,6 +48723,17 @@
|
|||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
|
||||
"cache_read_input_token_cost": 1.5e-08,
|
||||
"input_cost_per_token": 1.5e-07,
|
||||
"litellm_provider": "together_ai",
|
||||
"max_input_tokens": 1048576,
|
||||
"max_tokens": 1048576,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://www.together.ai/pricing",
|
||||
"supports_prompt_caching": true
|
||||
},
|
||||
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
|
||||
"input_cost_per_token": 0.0,
|
||||
"litellm_provider": "together_ai",
|
||||
|
|
@ -52906,12 +53141,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
|
||||
"input_cost_per_token": 3.5e-07,
|
||||
"input_cost_per_token_batches": 1.75e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 1000000,
|
||||
"max_tokens": 1000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 1.15e-06,
|
||||
"output_cost_per_token_batches": 5.75e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -52926,12 +53163,14 @@
|
|||
},
|
||||
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
|
||||
"input_cost_per_token": 2.5e-07,
|
||||
"input_cost_per_token_batches": 1.25e-07,
|
||||
"litellm_provider": "vertex_ai-llama_models",
|
||||
"max_input_tokens": 10000000,
|
||||
"max_output_tokens": 10000000,
|
||||
"max_tokens": 10000000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"output_cost_per_token_batches": 3.5e-07,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
|
||||
"supported_modalities": [
|
||||
"text",
|
||||
|
|
@ -73330,6 +73569,7 @@
|
|||
]
|
||||
},
|
||||
"vertex_ai/veo-2.0-generate-001": {
|
||||
"deprecation_date": "2026-06-30",
|
||||
"litellm_provider": "vertex_ai-video-models",
|
||||
"max_input_tokens": 1024,
|
||||
"max_tokens": 1024,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v"
|
|||
ignoreUntil = 2026-11-01
|
||||
reason = "diskcache has no fixed release published; remove this entry once one exists"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-v4x8-3wr5-3wq8"
|
||||
ignoreUntil = 2026-11-20
|
||||
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
|
||||
|
||||
[[IgnoredVulns]]
|
||||
id = "GHSA-vfj7-8cjw-p6xm"
|
||||
ignoreUntil = 2026-11-03
|
||||
|
|
|
|||
|
|
@ -137,7 +137,7 @@ semantic-router = [
|
|||
"semantic-router>=0.1.15,<1.0; python_version < '3.14'",
|
||||
"aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'",
|
||||
]
|
||||
mlflow = ["mlflow>=3.11.1,<4.0"]
|
||||
mlflow = ["mlflow>=3.17.2,<4.0"]
|
||||
grpc = [
|
||||
# First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff
|
||||
"grpcio==1.84.1",
|
||||
|
|
@ -177,16 +177,16 @@ proxy-runtime = [
|
|||
"grpcio==1.84.1",
|
||||
"prometheus-client>=0.20.0,<1.0",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"ddtrace>=4.8.2,<5.0",
|
||||
"sentry-sdk>=2.21.0,<3.0",
|
||||
"mangum>=0.17.0,<1.0",
|
||||
"azure-ai-contentsafety>=1.0.0,<2.0",
|
||||
"azure-storage-file-datalake>=12.20.0,<13.0",
|
||||
"pypdf>=6.16.1,<7.0",
|
||||
"pypdf>=6.21.2,<7.0",
|
||||
"llm-sandbox>=0.3.39,<1.0",
|
||||
"detect-secrets>=1.5.0,<2.0",
|
||||
]
|
||||
|
|
@ -223,10 +223,10 @@ dev = [
|
|||
"types-PyYAML==6.0.12.20250915",
|
||||
"botocore-stubs==1.43.14",
|
||||
"types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"langfuse>=4.7,<5.0",
|
||||
"fastapi-offline==1.7.6",
|
||||
"fakeredis==2.34.1",
|
||||
|
|
@ -255,10 +255,10 @@ proxy-dev = [
|
|||
"hypercorn==0.17.3",
|
||||
"prometheus-client==0.20.0",
|
||||
"sentry-sdk==2.21.0",
|
||||
"opentelemetry-api==1.33.1",
|
||||
"opentelemetry-sdk==1.33.1",
|
||||
"opentelemetry-exporter-otlp==1.33.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.54b1",
|
||||
"opentelemetry-api==1.45.1",
|
||||
"opentelemetry-sdk==1.45.1",
|
||||
"opentelemetry-exporter-otlp==1.45.1",
|
||||
"opentelemetry-instrumentation-fastapi==0.66b1",
|
||||
"azure-identity==1.25.2",
|
||||
"a2a-sdk==1.1.0",
|
||||
"backports-zstd==1.8.0; python_version < '3.14'",
|
||||
|
|
@ -280,7 +280,7 @@ ci = [
|
|||
"langchain==1.3.9",
|
||||
"lunary==1.4.36; python_version == '3.10'",
|
||||
"lunary==1.4.37; python_version >= '3.11'",
|
||||
"logfire==4.6.0",
|
||||
"logfire==5.2.0",
|
||||
"traceloop-sdk==0.34.0",
|
||||
"detect-secrets==1.5.0",
|
||||
"PyGithub==2.8.1",
|
||||
|
|
@ -359,7 +359,7 @@ constraint-dependencies = [
|
|||
"packaging>=24.0",
|
||||
"soupsieve>=2.8.4",
|
||||
"httplib2>=0.32.0",
|
||||
"setuptools>=83.0.0",
|
||||
"setuptools>=84.1.0",
|
||||
]
|
||||
override-dependencies = [
|
||||
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
|
||||
|
|
@ -370,10 +370,23 @@ override-dependencies = [
|
|||
default-groups = ["dev"]
|
||||
required-version = ">=0.10.9"
|
||||
exclude-newer = "3 days"
|
||||
|
||||
[tool.uv.exclude-newer-package]
|
||||
# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window
|
||||
certifi = "2026-10-20T03:00:00Z"
|
||||
# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window
|
||||
grpcio = "2026-10-22T15:00:00Z"
|
||||
# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window
|
||||
exclude-newer-package = { certifi = "2026-10-20T03:00:00Z", grpcio = "2026-10-22T15:00:00Z", boto3 = "2026-10-21T19:31:00Z", botocore = "2026-10-21T19:31:00Z" }
|
||||
boto3 = "2026-10-21T19:31:00Z"
|
||||
botocore = "2026-10-21T19:31:00Z"
|
||||
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
|
||||
brotli = "2026-10-22T14:01:00Z"
|
||||
# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window
|
||||
mlflow = "2026-10-21T03:00:00Z"
|
||||
mlflow-skinny = "2026-10-21T03:00:00Z"
|
||||
mlflow-tracing = "2026-10-21T03:00:00Z"
|
||||
# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window
|
||||
pypdf = "2026-10-21T17:00:00Z"
|
||||
|
||||
[tool.uv.sources]
|
||||
litellm-proxy-extras = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -1,125 +0,0 @@
|
|||
"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check
|
||||
the daily activity routes answer 400 with, instead of a generic fetch error."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import JSON_OBJECT, Gateway, object_value
|
||||
from integration._support.upstream import delete_scenario, register_scenario
|
||||
from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway
|
||||
from pydantic import JsonValue
|
||||
|
||||
pytestmark: Final = pytest.mark.timeout(180)
|
||||
|
||||
_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data")
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _UsageChat:
|
||||
gateway: Gateway
|
||||
identity: str
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]:
|
||||
"""A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream."""
|
||||
identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}"
|
||||
handle: Final = register_scenario(identity, json_response({}))
|
||||
try:
|
||||
with owned_gateway(
|
||||
tmp_path_factory.mktemp("audit-usage-chat"),
|
||||
{"model_list": [], "router_settings": {"num_retries": 0}},
|
||||
environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity},
|
||||
) as owned:
|
||||
yield _UsageChat(owned, identity)
|
||||
finally:
|
||||
delete_scenario(handle)
|
||||
|
||||
|
||||
def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]:
|
||||
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
|
||||
return {
|
||||
"id": "chatcmpl-usage",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}}
|
||||
for tool in _TOOLS
|
||||
],
|
||||
},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
|
||||
|
||||
def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]:
|
||||
"""Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up
|
||||
model call carried, keyed by tool, and the tool statuses the SSE stream reported."""
|
||||
register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date)))
|
||||
Upstream(usage_chat.gateway.upstream_url).reset()
|
||||
with usage_chat.gateway.client.stream(
|
||||
"POST",
|
||||
"/usage/ai/chat",
|
||||
json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]},
|
||||
headers={"Authorization": f"Bearer {usage_chat.gateway.key}"},
|
||||
timeout=60,
|
||||
) as response:
|
||||
assert response.status_code == 200
|
||||
events: Final = tuple(
|
||||
object_value(JSON_OBJECT.validate_json(line.removeprefix("data: ")))
|
||||
for line in response.iter_lines()
|
||||
if line.startswith("data: ")
|
||||
)
|
||||
calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity)
|
||||
assert len(calls) == 2, calls
|
||||
messages: Final = object_value(calls[1]["body"]).get("messages")
|
||||
assert isinstance(messages, list), calls[1]
|
||||
tool_messages: Final = {
|
||||
str(message["tool_call_id"]).removeprefix("call-"): str(message["content"])
|
||||
for message in messages
|
||||
if isinstance(message, dict) and message.get("role") == "tool"
|
||||
}
|
||||
statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call")
|
||||
return tool_messages, statuses
|
||||
|
||||
|
||||
_REVERSED: Final = "Invalid date range: end_date must be on or after start_date"
|
||||
_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates"
|
||||
_NO_TEAM_DATA: Final = "No Team usage data found for the given date range."
|
||||
_NO_TAG_DATA: Final = "No Tag usage data found for the given date range."
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("start_date", "end_date", "team_message", "tag_message"),
|
||||
(
|
||||
pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"),
|
||||
pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"),
|
||||
pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"),
|
||||
pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"),
|
||||
),
|
||||
)
|
||||
def test_usage_chat_tools_answer_with_the_query_layers_date_check(
|
||||
usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str
|
||||
) -> None:
|
||||
"""The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query
|
||||
takes these dates without that check, still answers with its usage summary."""
|
||||
tool_messages, statuses = _tool_results(usage_chat, start_date, end_date)
|
||||
assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages
|
||||
assert tool_messages["get_team_usage_data"] == team_message, tool_messages
|
||||
assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages
|
||||
assert statuses == ("running", "complete") * len(_TOOLS), statuses
|
||||
94
tests/integration/management/test_budget_list_sorting.py
Normal file
94
tests/integration/management/test_budget_list_sorting.py
Normal file
|
|
@ -0,0 +1,94 @@
|
|||
from datetime import datetime, timezone
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
|
||||
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
def test_budget_reset_sort_orders_durations_before_pagination(
|
||||
descending: bool,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "5d", "𝟝d", "5d", None)
|
||||
ids: Final = tuple(
|
||||
scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10)
|
||||
for index, duration in enumerate(durations)
|
||||
)
|
||||
now: Final = datetime.now(timezone.utc)
|
||||
expected: Final = tuple(
|
||||
budget_id
|
||||
for _, budget_id in sorted(
|
||||
zip(durations, ids, strict=True),
|
||||
key=lambda item: (
|
||||
item[0] is None,
|
||||
duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0,
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": "-budget_duration" if descending else "budget_duration",
|
||||
"page_size": "3",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in range(1, -(-len(ids) // 3) + 1)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
|
||||
filtered: Final = gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": "budget_duration",
|
||||
"filter[budget_duration][in]": "7d,30d",
|
||||
},
|
||||
)
|
||||
assert filtered.status_code == 200, filtered.text
|
||||
assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit"))
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
def test_budget_limits_sort_unlimited_as_infinity(
|
||||
field: str, descending: bool, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-unlimited-{uuid4().hex}"
|
||||
values: Final = (20, None, 0, 3)
|
||||
ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values))
|
||||
expected: Final = tuple(
|
||||
identity
|
||||
for value, identity in sorted(
|
||||
zip(values, ids, strict=True),
|
||||
key=lambda item: float("inf") if item[0] is None else item[0],
|
||||
reverse=descending,
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/management/v1/budgets",
|
||||
params={
|
||||
"q": prefix,
|
||||
"sort": ("-" if descending else "") + field,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
|
||||
|
|
@ -1,11 +1,16 @@
|
|||
from typing import FrozenSet
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from itertools import chain
|
||||
from typing import Final, FrozenSet
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.proxy.utils import hash_token
|
||||
|
||||
from tests.integration._support.actors import TEAM_ALPHA, Actor
|
||||
from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources
|
||||
from tests.integration._support.actors import create_scratch_key
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
|
||||
from litellm.repositories.verification_token_repository import VerificationTokenRepository
|
||||
|
||||
pytestmark = pytest.mark.asyncio(loop_scope="session")
|
||||
|
||||
|
|
@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor
|
|||
},
|
||||
)
|
||||
assert resp.status_code == 403, resp.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit")
|
||||
)
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted"))
|
||||
async def test_key_list_sorts_across_pages_with_nulls_last(
|
||||
field: str,
|
||||
direction: str,
|
||||
deleted: bool,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
db: ManagementDatabase,
|
||||
resources: ScratchResources,
|
||||
) -> None:
|
||||
team_admin: Final = scenario.user(user_role="internal_user")
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
team_id: Final = scenario.team(
|
||||
members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}]
|
||||
)
|
||||
team_admin_key: Final = scenario.key(user_id=team_admin)
|
||||
table: Final = (
|
||||
DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table
|
||||
)
|
||||
values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None)
|
||||
tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values)))
|
||||
rows: Final = [
|
||||
{
|
||||
"token": token,
|
||||
"key_alias": resources.prefix,
|
||||
"team_id": team_id,
|
||||
"user_id": owner,
|
||||
field: (
|
||||
datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value)
|
||||
if field in ("expires", "last_active", "budget_reset_at") and value is not None
|
||||
else value
|
||||
),
|
||||
}
|
||||
for token, value in zip(tokens, values, strict=True)
|
||||
]
|
||||
await table.create_many(rows)
|
||||
try:
|
||||
valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None)
|
||||
expected: Final = [
|
||||
token
|
||||
for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1]))
|
||||
] + [token for value, token in zip(values, tokens, strict=True) if value is None]
|
||||
for caller in (gateway.key, team_admin_key):
|
||||
pages: Final = [
|
||||
gateway.client.get(
|
||||
"/key/list",
|
||||
headers={"Authorization": f"Bearer {caller}"},
|
||||
params={
|
||||
"key_alias": resources.prefix,
|
||||
"team_id": team_id,
|
||||
"include_team_keys": "true",
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"size": "2",
|
||||
"page": str(page),
|
||||
**({"status": "deleted"} if deleted else {}),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
]
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total_count"] == len(tokens) for page in pages)
|
||||
assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected
|
||||
finally:
|
||||
await table.delete_many(where={"token": {"in": tokens}})
|
||||
|
|
|
|||
|
|
@ -3,7 +3,9 @@ import contextlib
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from contextlib import contextmanager
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
|
|
@ -15,7 +17,7 @@ from pydantic import JsonValue
|
|||
|
||||
from tests.integration._support.client import Gateway, gateway_from_environment
|
||||
from tests.integration._support.database import read_rows
|
||||
from tests.integration._support.process import owned_proxy
|
||||
from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy
|
||||
from tests.integration._support.wire import Reply, Request, wire_server
|
||||
|
||||
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
|
||||
|
|
@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role(
|
|||
"metadata": {"auth_provider": IDP_PROVIDER},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value)
|
||||
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
|
||||
with _sso_gateway(tmp_path, {}) as gateway:
|
||||
for index in range(4):
|
||||
signed_in: Final = _sign_in(gateway, f"free-sso-{index}")
|
||||
assert signed_in.status_code == 303, signed_in.text
|
||||
|
||||
contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8))
|
||||
with ThreadPoolExecutor(max_workers=len(contenders)) as pool:
|
||||
responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders))
|
||||
|
||||
assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403]
|
||||
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}]
|
||||
for subject, response in zip(contenders, responses):
|
||||
if response.status_code == 403:
|
||||
assert "requires a LiteLLM Enterprise license" in response.text
|
||||
assert not response.cookies
|
||||
assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == []
|
||||
|
||||
existing: Final = _sign_in(gateway, "free-sso-0")
|
||||
assert existing.status_code == 303, existing.text
|
||||
|
||||
admin_created: Final = gateway.request(
|
||||
"POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False}
|
||||
)
|
||||
assert admin_created.status_code == 200, admin_created.text
|
||||
above_capacity: Final = _sign_in(gateway, "free-sso-0")
|
||||
refused: Final = _sign_in(gateway, "free-sso-extra")
|
||||
assert (above_capacity.status_code, refused.status_code) == (303, 403)
|
||||
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}]
|
||||
|
|
|
|||
244
tests/integration/management/test_team_list_sorting.py
Normal file
244
tests/integration/management/test_team_list_sorting.py
Normal file
|
|
@ -0,0 +1,244 @@
|
|||
import json
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from tests.integration._support.database import write_rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count")
|
||||
)
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_team_list_sorts_across_pages_and_respects_membership(
|
||||
field: str,
|
||||
direction: str,
|
||||
gateway: Gateway,
|
||||
scenario: Scenario,
|
||||
) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (20, 3, 3, 100, 0)
|
||||
teams: Final = tuple(
|
||||
scenario.team(
|
||||
team_id=f"{prefix}-{index}",
|
||||
team_alias=f"{prefix}-{index}",
|
||||
members_with_roles=[{"user_id": owner, "role": "admin"}],
|
||||
)
|
||||
for index in range(len(values))
|
||||
)
|
||||
outsider: Final = scenario.team(team_alias=f"{prefix}-outside")
|
||||
for team_id, value in zip(teams, values, strict=True):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, '
|
||||
"tpm_limit = %s::bigint, rpm_limit = %s::bigint, "
|
||||
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', "
|
||||
"members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) "
|
||||
"WHERE team_id = %s",
|
||||
(
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
str(value),
|
||||
json.dumps(
|
||||
[{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)]
|
||||
),
|
||||
str(value),
|
||||
team_id,
|
||||
),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
|
||||
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
|
||||
(team_id, team_id, str(value)),
|
||||
)
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s',
|
||||
(teams[-1],),
|
||||
)
|
||||
try:
|
||||
expected: Final = tuple(
|
||||
team_id
|
||||
for value, team_id in sorted(
|
||||
zip(values, teams, strict=True),
|
||||
key=lambda item: (
|
||||
(item[0] != 0 if direction == "desc" else item[0] == 0)
|
||||
if field in ("max_budget", "tpm_limit", "rpm_limit")
|
||||
else False,
|
||||
item[0] * (1 if direction == "asc" else -1),
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"search": prefix,
|
||||
"user_id": owner,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages))
|
||||
assert returned == expected
|
||||
assert outsider not in returned
|
||||
assert all(page.json()["total"] == len(teams) for page in pages)
|
||||
finally:
|
||||
for team_id in teams:
|
||||
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_team_list_sorts_organization_names_and_missing_values(
|
||||
direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-{uuid4().hex}"
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", ""))
|
||||
teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4))
|
||||
try:
|
||||
for (organization_id, alias), team_id in zip(organizations, teams):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(organization_id, alias, budget),
|
||||
)
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s',
|
||||
(organization_id, team_id),
|
||||
)
|
||||
ascending: Final = (teams[0], teams[2], teams[1])
|
||||
ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending))
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": "organization_alias",
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1])
|
||||
finally:
|
||||
for team_id in teams:
|
||||
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,))
|
||||
for organization_id, _ in organizations:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_computed_team_sorts_preserve_org_and_membership_union(
|
||||
field: str, direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-scope-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
|
||||
teams: Final = tuple(
|
||||
scenario.team(
|
||||
team_id=f"{prefix}-{i}",
|
||||
team_alias=f"{prefix}-{i}",
|
||||
members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [],
|
||||
)
|
||||
for i in range(4)
|
||||
)
|
||||
try:
|
||||
for org, alias in zip(orgs, ("alpha", "zulu"), strict=True):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(org, alias, budget),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'org_admin')",
|
||||
(owner, orgs[0]),
|
||||
)
|
||||
for index, team in enumerate(teams):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, '
|
||||
"models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s",
|
||||
(
|
||||
orgs[0] if index < 2 else orgs[1],
|
||||
json.dumps(
|
||||
[
|
||||
{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"}
|
||||
for i in range(index + 1)
|
||||
]
|
||||
),
|
||||
str(index + 1),
|
||||
team,
|
||||
),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
|
||||
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
|
||||
(team, team, str(index + 1)),
|
||||
)
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3)
|
||||
expected: Final = tuple(
|
||||
team
|
||||
for value, team in sorted(
|
||||
zip(values, teams[:3], strict=True),
|
||||
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"user_id": owner,
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == 3 for page in pages)
|
||||
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected
|
||||
denied: Final = gateway.request(
|
||||
"GET",
|
||||
"/v2/team/list",
|
||||
key=caller,
|
||||
params={
|
||||
"organization_id": orgs[1],
|
||||
"sort_by": field,
|
||||
},
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
finally:
|
||||
for team in teams:
|
||||
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
|
||||
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,))
|
||||
for org in orgs:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))
|
||||
179
tests/integration/management/test_user_list_sorting.py
Normal file
179
tests/integration/management/test_user_list_sorting.py
Normal file
|
|
@ -0,0 +1,179 @@
|
|||
from itertools import chain
|
||||
from typing import Final
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from tests.integration._support.client import Gateway, Scenario
|
||||
from tests.integration._support.database import write_rows
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None:
|
||||
prefix: Final = f"sort-{uuid4().hex}"
|
||||
ids: Final = tuple(
|
||||
scenario.user(
|
||||
user_id=f"{prefix}-{index}",
|
||||
user_role="internal_user",
|
||||
user_alias=f"Alias {value if index != 2 else 4:03}",
|
||||
user_email=f"{prefix}-{index}@example.com",
|
||||
max_budget=value,
|
||||
metadata={"scim_active": value != 3},
|
||||
)
|
||||
for index, value in enumerate((20, 3, 3, 100, 0))
|
||||
)
|
||||
for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True):
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, '
|
||||
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s",
|
||||
(f"{prefix}-{value:03}", str(value), user_id),
|
||||
) if value != 3 or user_id == ids[1] else write_rows(
|
||||
"UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s",
|
||||
(f"{prefix}-004", user_id),
|
||||
)
|
||||
for index in range(value if value < 4 else 1):
|
||||
scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}")
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s',
|
||||
(ids[-1],),
|
||||
)
|
||||
scenario.key(user_id=ids[-1])
|
||||
write_rows(
|
||||
'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1])
|
||||
)
|
||||
values: Final = (
|
||||
(0, 1, 1, 0, 0)
|
||||
if field == "status"
|
||||
else (1, 3, 3, 1, 0)
|
||||
if field == "key_count"
|
||||
else (20, 3, 4, 100, None)
|
||||
if field == "sso_user_id"
|
||||
else (20, 3, 4, 100, 0)
|
||||
if field == "user_alias"
|
||||
else (20, 3, 3, 100, None)
|
||||
)
|
||||
expected: Final = tuple(
|
||||
user_id
|
||||
for _, user_id in sorted(
|
||||
((value, user_id) for value, user_id in zip(values, ids, strict=True)),
|
||||
key=lambda item: (
|
||||
(item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None),
|
||||
(item[0] or 0) * (1 if direction == "asc" else -1),
|
||||
item[1],
|
||||
),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2, 3)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == len(ids) for page in pages)
|
||||
assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("status", "key_count"))
|
||||
def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None:
|
||||
team: Final = scenario.team()
|
||||
member: Final = scenario.member(team)
|
||||
other: Final = scenario.user(user_role="internal_user")
|
||||
for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}):
|
||||
baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"})
|
||||
response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field})
|
||||
assert response["total"] == baseline["total"]
|
||||
assert sorted(user["user_id"] for user in response["users"]) == sorted(
|
||||
user["user_id"] for user in baseline["users"]
|
||||
)
|
||||
ordinary_key: Final = scenario.key(user_id=other)
|
||||
denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field})
|
||||
assert denied.status_code == 403, denied.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("field", ("status", "key_count"))
|
||||
@pytest.mark.parametrize("direction", ("asc", "desc"))
|
||||
def test_computed_user_sorts_preserve_org_admin_scope(
|
||||
field: str, direction: str, gateway: Gateway, scenario: Scenario
|
||||
) -> None:
|
||||
prefix: Final = f"sort-org-users-{uuid4().hex}"
|
||||
owner: Final = scenario.user(user_role="internal_user")
|
||||
members: Final = tuple(
|
||||
scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1})
|
||||
for index in range(4)
|
||||
)
|
||||
budget: Final = scenario.budget(max_budget=10)
|
||||
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
|
||||
try:
|
||||
for org in orgs:
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationTable" '
|
||||
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
|
||||
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
|
||||
(org, org, budget),
|
||||
)
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'org_admin')",
|
||||
(owner, orgs[0]),
|
||||
)
|
||||
for index, member in enumerate(members):
|
||||
write_rows(
|
||||
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
|
||||
"VALUES (%s, %s, 'internal_user')",
|
||||
(member, orgs[0] if index < 3 else orgs[1]),
|
||||
)
|
||||
for _ in range(index):
|
||||
scenario.key(user_id=member)
|
||||
caller: Final = scenario.key(user_id=owner)
|
||||
values: Final = (0, 1, 0) if field == "status" else (0, 1, 2)
|
||||
expected: Final = tuple(
|
||||
member
|
||||
for value, member in sorted(
|
||||
zip(values, members[:3], strict=True),
|
||||
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
|
||||
)
|
||||
)
|
||||
pages: Final = tuple(
|
||||
gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
key=caller,
|
||||
params={
|
||||
"search": prefix,
|
||||
"sort_by": field,
|
||||
"sort_order": direction,
|
||||
"page_size": "2",
|
||||
"page": str(page),
|
||||
},
|
||||
)
|
||||
for page in (1, 2)
|
||||
)
|
||||
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
|
||||
assert all(page.json()["total"] == 3 for page in pages)
|
||||
returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages))
|
||||
assert returned == expected
|
||||
denied: Final = gateway.request(
|
||||
"GET",
|
||||
"/user/list",
|
||||
key=caller,
|
||||
params={
|
||||
"organization_ids": orgs[1],
|
||||
"sort_by": field,
|
||||
},
|
||||
)
|
||||
assert denied.status_code == 403, denied.text
|
||||
finally:
|
||||
for org in orgs:
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
|
||||
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))
|
||||
180
tests/integration/security/test_offline_mode.py
Normal file
180
tests/integration/security/test_offline_mode.py
Normal file
|
|
@ -0,0 +1,180 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from integration._support.client import Gateway, object_value
|
||||
from integration._support.forward_proxy import refusing_forward_proxy
|
||||
from integration._support.process import owned_proxy
|
||||
from pydantic import JsonValue
|
||||
|
||||
_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm"
|
||||
_CONTROL_HOST: Final = "provider.offline-mode.example"
|
||||
_REMOTE_URLS: Final = {
|
||||
"LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json",
|
||||
"LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json",
|
||||
"LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json",
|
||||
"LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml",
|
||||
}
|
||||
_SELF_INITIATED_HOSTS: Final = frozenset(
|
||||
{
|
||||
"cost-map.offline-mode.example:443",
|
||||
"beta-headers.offline-mode.example:443",
|
||||
"autorouter.offline-mode.example:443",
|
||||
"blog.offline-mode.example:443",
|
||||
"raw.githubusercontent.com:443",
|
||||
"docs.litellm.ai:443",
|
||||
"api.pwnedpasswords.com:443",
|
||||
"huggingface.co:443",
|
||||
}
|
||||
)
|
||||
_PER_FEATURE_FLAGS: Final = (
|
||||
"LITELLM_OFFLINE_MODE",
|
||||
"LITELLM_LOCAL_MODEL_COST_MAP",
|
||||
"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS",
|
||||
"LITELLM_LOCAL_AUTOROUTER_PRESETS",
|
||||
"LITELLM_LOCAL_POLICY_TEMPLATES",
|
||||
"LITELLM_LOCAL_BLOG_POSTS",
|
||||
"LITELLM_DISABLE_LATEST_RELEASE_CHECK",
|
||||
"LITELLM_LICENSE",
|
||||
)
|
||||
|
||||
|
||||
def _bundled(name: str) -> JsonValue:
|
||||
return json.loads((_PACKAGE / name).read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def _write_config(directory: Path) -> Path:
|
||||
config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml"
|
||||
config.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"model_list": [
|
||||
{
|
||||
"model_name": "offline-mode-control",
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_base": f"https://{_CONTROL_HOST}/v1",
|
||||
"api_key": "synthetic-provider-key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"model_name": "offline-mode-kimi",
|
||||
"litellm_params": {
|
||||
"model": "bedrock/us.moonshotai.kimi-k3",
|
||||
"aws_region_name": "us-east-1",
|
||||
"aws_access_key_id": "synthetic-access-key",
|
||||
"aws_secret_access_key": "synthetic-secret-key",
|
||||
},
|
||||
},
|
||||
],
|
||||
"router_settings": {"num_retries": 0},
|
||||
"general_settings": {
|
||||
"master_key": "os.environ/LITELLM_MASTER_KEY",
|
||||
"database_url": "os.environ/DATABASE_URL",
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
return config
|
||||
|
||||
|
||||
def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue:
|
||||
response: Final = proxy.request(method, path, body)
|
||||
assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}"
|
||||
return json.loads(response.content)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
@pytest.mark.parametrize(
|
||||
("offline_mode", "expected_hosts", "reload_status"),
|
||||
[
|
||||
pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"),
|
||||
pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"),
|
||||
],
|
||||
)
|
||||
def test_offline_mode_stops_every_self_initiated_request(
|
||||
gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int
|
||||
) -> None:
|
||||
with refusing_forward_proxy() as egress:
|
||||
environment: Final = {
|
||||
"HTTPS_PROXY": egress.url,
|
||||
"HTTP_PROXY": egress.url,
|
||||
"ALL_PROXY": egress.url,
|
||||
"NO_PROXY": "127.0.0.1,localhost",
|
||||
"AIOHTTP_TRUST_ENV": "True",
|
||||
**_REMOTE_URLS,
|
||||
**({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}),
|
||||
}
|
||||
with owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
environment,
|
||||
config=_write_config(tmp_path),
|
||||
remove_environment=_PER_FEATURE_FLAGS,
|
||||
workers=2,
|
||||
) as isolated:
|
||||
source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source"))
|
||||
reloaded: Final = isolated.request("POST", "/reload/model_cost_map")
|
||||
beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers"))
|
||||
release: Final = _json(isolated, "GET", "/get/latest_release_info")
|
||||
presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets"))
|
||||
templates: Final = _json(isolated, "GET", "/policy/templates")
|
||||
blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts"))
|
||||
user: Final = object_value(
|
||||
_json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"})
|
||||
)
|
||||
password_set: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/user/update",
|
||||
{"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"},
|
||||
)
|
||||
)
|
||||
tokens: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/utils/token_counter",
|
||||
{"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]},
|
||||
)
|
||||
)
|
||||
kimi_tokens: Final = object_value(
|
||||
_json(
|
||||
isolated,
|
||||
"POST",
|
||||
"/utils/token_counter",
|
||||
{"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]},
|
||||
)
|
||||
)
|
||||
control: Final = isolated.request(
|
||||
"POST",
|
||||
"/v1/chat/completions",
|
||||
{"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]},
|
||||
)
|
||||
reached: Final = frozenset(egress.targets())
|
||||
|
||||
assert reached == expected_hosts
|
||||
assert control.status_code != 200, control.text
|
||||
assert source["source"] == "local", source
|
||||
assert source["is_env_forced"] is (offline_mode is not None), source
|
||||
assert reloaded.status_code == reload_status, reloaded.text
|
||||
assert beta_headers["status"] == "success", beta_headers
|
||||
assert release is None
|
||||
assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json")))
|
||||
assert templates == _bundled("policy_templates_backup.json")
|
||||
assert [object_value(post)["url"] for post in _list(blog["posts"])] == [
|
||||
object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"])
|
||||
]
|
||||
assert password_set["user_id"] == user["user_id"], password_set
|
||||
assert tokens["tokenizer_type"] == "openai_tokenizer", tokens
|
||||
assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens
|
||||
|
||||
|
||||
def _list(value: JsonValue) -> list[JsonValue]:
|
||||
assert isinstance(value, list), value
|
||||
return value
|
||||
|
|
@ -99,16 +99,9 @@ class _Activity(BaseModel):
|
|||
results: tuple[_Day, ...]
|
||||
|
||||
|
||||
class _UpstreamMessage(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
role: str | None = None
|
||||
|
||||
|
||||
class _UpstreamRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
stream: bool | None = None
|
||||
tools: tuple[JsonValue, ...] | None = None
|
||||
messages: tuple[_UpstreamMessage, ...] = ()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
|
|
@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]:
|
|||
return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n")
|
||||
|
||||
|
||||
def _usage_tool_call() -> Mapping[str, JsonValue]:
|
||||
arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)})
|
||||
return {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def _response_object() -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
|
|
@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply:
|
|||
body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}")
|
||||
if body.stream:
|
||||
return Reply(content_type="text/event-stream", chunks=_stream_frames())
|
||||
roles: Final = frozenset(message.role for message in body.messages)
|
||||
if body.tools and "tool" not in roles:
|
||||
return Reply(body=_completion(_usage_tool_call(), "tool_calls"))
|
||||
return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop"))
|
||||
|
||||
|
||||
|
|
@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None:
|
|||
)
|
||||
|
||||
|
||||
def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]:
|
||||
return tuple(
|
||||
json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ")
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Tenant:
|
||||
label: str
|
||||
|
|
@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]:
|
|||
|
||||
|
||||
@pytest.mark.timeout(360)
|
||||
def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
|
||||
def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with _rig(gateway, tmp_path) as (proxy, database_url):
|
||||
spender: Final = _spender(proxy, database_url, "ai-chat")
|
||||
spender: Final = _spender(proxy, database_url, "repeated-timeout")
|
||||
with _pinned(proxy) as pinned:
|
||||
with _locked_spend_logs(database_url):
|
||||
with _recording(database_url) as during_chat:
|
||||
chat: Final = pinned.request(
|
||||
"POST",
|
||||
"/usage/ai/chat",
|
||||
body={
|
||||
"messages": [{"role": "user", "content": "What did we spend?"}],
|
||||
"model": "openai/gpt-4o-mini",
|
||||
},
|
||||
)
|
||||
assert chat.status_code == 200, chat.text
|
||||
assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed
|
||||
tool_call: Final = {
|
||||
"type": "tool_call",
|
||||
"tool_name": "get_usage_data",
|
||||
"tool_label": "global usage data",
|
||||
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
|
||||
}
|
||||
assert _events(chat) == (
|
||||
{"type": "status", "message": "Thinking..."},
|
||||
{**tool_call, "status": "running"},
|
||||
{**tool_call, "status": "complete"},
|
||||
{"type": "status", "message": "Analyzing results..."},
|
||||
{"type": "chunk", "content": REPLY_TEXT},
|
||||
{"type": "done"},
|
||||
), chat.text
|
||||
assert len(_lookups(during_chat)) == 1
|
||||
with _recording(database_url) as during_timeout:
|
||||
timed_out: Final = _read(pinned, spender.digest)
|
||||
assert timed_out.status_code == 200, timed_out.text
|
||||
assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed
|
||||
assert _aliases(timed_out, spender.digest) == (None,), timed_out.text
|
||||
assert len(_lookups(during_timeout)) == 1
|
||||
cached_miss: Final = _read(pinned, spender.digest)
|
||||
assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed
|
||||
assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text
|
||||
|
|
@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked
|
|||
assert gateway_activity == 200, gateway_activity
|
||||
chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}")
|
||||
assert object_value(chat["usage"]) == dict(USAGE), chat
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
with (
|
||||
scratch_database() as database_url,
|
||||
database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url),
|
||||
wire_server(_respond) as wire,
|
||||
_proxy(gateway, tmp_path, relayed_url, wire.url) as proxy,
|
||||
):
|
||||
spender: Final = _spender(proxy, database_url, "dropped-ai-chat")
|
||||
with _pinned(proxy) as pinned:
|
||||
relay.arm()
|
||||
chat: Final = pinned.request(
|
||||
"POST",
|
||||
"/usage/ai/chat",
|
||||
body={
|
||||
"messages": [{"role": "user", "content": "What did we spend?"}],
|
||||
"model": "openai/gpt-4o-mini",
|
||||
},
|
||||
)
|
||||
assert chat.status_code == 200, chat.text
|
||||
assert relay.tripped.is_set(), chat.text
|
||||
tool_call: Final = {
|
||||
"type": "tool_call",
|
||||
"tool_name": "get_usage_data",
|
||||
"tool_label": "global usage data",
|
||||
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
|
||||
}
|
||||
assert _events(chat) == (
|
||||
{"type": "status", "message": "Thinking..."},
|
||||
{**tool_call, "status": "running"},
|
||||
{**tool_call, "status": "complete"},
|
||||
{"type": "status", "message": "Analyzing results..."},
|
||||
{"type": "chunk", "content": REPLY_TEXT},
|
||||
{"type": "done"},
|
||||
), chat.text
|
||||
recovered: Final = _named_once_reconnected(pinned, spender)
|
||||
assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from collections.abc import Iterable
|
|||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from hashlib import sha256
|
||||
from pathlib import Path
|
||||
from queue import SimpleQueue
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -31,8 +30,7 @@ from integration._support.daily_activity import (
|
|||
user_row,
|
||||
user_with_an_email,
|
||||
)
|
||||
from integration._support.database import read_rows, scratch_database
|
||||
from integration._support.process import owned_proxy
|
||||
from integration._support.database import read_rows
|
||||
from integration._support.wire import Reply, Request, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
|
|
@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons
|
|||
REQUESTS_OF_A_BURST: Final = 21
|
||||
READS_DURING_A_BURST: Final = 30
|
||||
TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models")
|
||||
TOOL_CALL: Final = "call_integration_usage"
|
||||
ANSWER: Final = "One request cost $0.25"
|
||||
SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join(
|
||||
(
|
||||
"Total Spend: $0.2500",
|
||||
"Total Requests: 1",
|
||||
"Successful: 1 | Failed: 0",
|
||||
"Total Tokens: 15",
|
||||
"",
|
||||
"Top Models by Spend:",
|
||||
" - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)",
|
||||
"",
|
||||
"Top Providers by Spend:",
|
||||
" - openai: $0.2500 (1 reqs)",
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _chat_completion() -> dict[str, JsonValue]:
|
||||
|
|
@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply:
|
|||
return Reply(body=json.dumps(body).encode())
|
||||
|
||||
|
||||
def _usage_tool_call() -> dict[str, JsonValue]:
|
||||
call: Final[dict[str, JsonValue]] = {
|
||||
"id": TOOL_CALL,
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_usage_data",
|
||||
"arguments": json.dumps({"start_date": DAY, "end_date": DAY}),
|
||||
},
|
||||
}
|
||||
return {
|
||||
"id": f"chatcmpl-{uuid.uuid4().hex}",
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": None, "tool_calls": [call]},
|
||||
"finish_reason": "tool_calls",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
|
||||
}
|
||||
|
||||
|
||||
def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes:
|
||||
chunk: Final = {
|
||||
"id": "chatcmpl-integration-usage",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": "gpt-4o-mini",
|
||||
"choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
|
||||
}
|
||||
return f"data: {json.dumps(chunk)}\n\n".encode()
|
||||
|
||||
|
||||
def _usage_analyst(request: Request) -> Reply:
|
||||
if json.loads(request.body).get("stream"):
|
||||
return Reply(
|
||||
chunks=(
|
||||
_streamed_chunk({"role": "assistant", "content": ANSWER}, None),
|
||||
_streamed_chunk({}, "stop"),
|
||||
b"data: [DONE]\n\n",
|
||||
),
|
||||
content_type="text/event-stream",
|
||||
)
|
||||
return Reply(body=json.dumps(_usage_tool_call()).encode())
|
||||
|
||||
|
||||
def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]:
|
||||
return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY)
|
||||
|
||||
|
|
@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session(
|
|||
)
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
question: Final = f"what did we spend {uuid.uuid4().hex}"
|
||||
owner: Final = f"integration-{uuid.uuid4().hex}"
|
||||
ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}"
|
||||
with (
|
||||
scratch_database() as scratch_url,
|
||||
wire_server(_usage_analyst) as wire,
|
||||
owned_proxy(
|
||||
gateway,
|
||||
tmp_path,
|
||||
{
|
||||
"DATABASE_URL": scratch_url,
|
||||
"OPENAI_API_BASE": f"{wire.url}/v1",
|
||||
"OPENAI_BASE_URL": f"{wire.url}/v1",
|
||||
"OPENAI_API_KEY": "integration-provider-key",
|
||||
},
|
||||
remove_environment=("DATABASE_URL_READ_REPLICA",),
|
||||
) as candidate,
|
||||
):
|
||||
candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False})
|
||||
with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url):
|
||||
answer: Final = candidate.request(
|
||||
"POST",
|
||||
"/usage/ai/chat",
|
||||
{"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"},
|
||||
)
|
||||
assert answer.status_code == 200, answer.text
|
||||
tool_call: Final = {
|
||||
"type": "tool_call",
|
||||
"tool_name": "get_usage_data",
|
||||
"tool_label": "global usage data",
|
||||
"arguments": {"start_date": DAY, "end_date": DAY},
|
||||
}
|
||||
events: Final = [
|
||||
json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ")
|
||||
]
|
||||
assert events == [
|
||||
{"type": "status", "message": "Thinking..."},
|
||||
{**tool_call, "status": "running"},
|
||||
{**tool_call, "status": "complete"},
|
||||
{"type": "status", "message": "Analyzing results..."},
|
||||
{"type": "chunk", "content": ANSWER},
|
||||
{"type": "done"},
|
||||
], answer.text
|
||||
asked, analysed = wire.drain()
|
||||
assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"]
|
||||
assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question}
|
||||
assert json.loads(analysed.body)["messages"][-1] == {
|
||||
"role": "tool",
|
||||
"tool_call_id": TOOL_CALL,
|
||||
"content": SUMMARY_OF_ONE_SEEDED_ROW,
|
||||
}
|
||||
assert owner not in analysed.body.decode()
|
||||
assert ownerless_key not in analysed.body.decode()
|
||||
|
||||
|
||||
@pytest.mark.timeout(300)
|
||||
def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None:
|
||||
released: Final = threading.Event()
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
|
|||
|
||||
import pytest
|
||||
import requests
|
||||
from requests_mock import Adapter
|
||||
|
||||
pytest.importorskip("opentelemetry")
|
||||
|
||||
|
|
@ -22,16 +23,18 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa:
|
|||
)
|
||||
from opentelemetry import baggage # noqa: E402
|
||||
from opentelemetry.context import attach, detach # noqa: E402
|
||||
from opentelemetry._logs.severity import SeverityNumber # noqa: E402
|
||||
from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402
|
||||
from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402
|
||||
from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above
|
||||
from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above
|
||||
from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above
|
||||
from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above
|
||||
from opentelemetry.sdk.metrics import MeterProvider # noqa: E402
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402
|
||||
from opentelemetry.sdk.trace import TracerProvider # noqa: E402
|
||||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above
|
||||
from opentelemetry.sdk.trace.export import ( # noqa: E402
|
||||
BatchSpanProcessor,
|
||||
ConsoleSpanExporter,
|
||||
SimpleSpanProcessor,
|
||||
SpanExportResult,
|
||||
)
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402
|
||||
InMemorySpanExporter,
|
||||
|
|
@ -46,6 +49,7 @@ import litellm # noqa: E402
|
|||
from tests.unit.integrations.conftest import TlsSink # noqa: E402
|
||||
from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402
|
||||
from litellm.integrations.otel.plumbing import providers # noqa: E402
|
||||
from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above
|
||||
from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402
|
||||
from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402
|
||||
from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402
|
||||
|
|
@ -733,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict]
|
|||
class Collector(BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
body = self.rfile.read(int(self.headers["Content-Length"]))
|
||||
received.append({"path": self.path, "headers": dict(self.headers), "body": body})
|
||||
received.append(
|
||||
{"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body}
|
||||
)
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
|
||||
|
|
@ -793,6 +799,95 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint():
|
|||
assert child["parentSpanId"] == format(parent_id, "016x")
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("statuses", "expected"),
|
||||
[
|
||||
((200,), SpanExportResult.SUCCESS),
|
||||
((400,), SpanExportResult.FAILURE),
|
||||
((503, 200), SpanExportResult.SUCCESS),
|
||||
],
|
||||
)
|
||||
def test_http_json_exporter_preserves_http_failure_and_retry_results(
|
||||
statuses: tuple[int, ...], expected: SpanExportResult
|
||||
) -> None:
|
||||
adapter: Final = Adapter()
|
||||
endpoint: Final = "https://collector.test/v1/traces"
|
||||
adapter.register_uri(
|
||||
"POST",
|
||||
endpoint,
|
||||
[{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses],
|
||||
)
|
||||
with requests.Session() as session:
|
||||
session.mount("https://", adapter)
|
||||
exporter: Final = OTLPJsonSpanExporter(
|
||||
endpoint=endpoint,
|
||||
headers={
|
||||
"CONTENT-TYPE": "application/x-protobuf",
|
||||
"x-collector-token": "secret",
|
||||
},
|
||||
session=session,
|
||||
)
|
||||
try:
|
||||
assert exporter.export(()) is expected
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
assert len(adapter.request_history) == len(statuses)
|
||||
for request in adapter.request_history:
|
||||
assert request.headers["Content-Type"] == "application/json"
|
||||
assert request.headers["x-collector-token"] == "secret"
|
||||
assert request.json() == {"resourceSpans": []}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("size_limit", "expected", "request_count"),
|
||||
[
|
||||
(0, SpanExportResult.SUCCESS, 1),
|
||||
(len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1),
|
||||
(len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0),
|
||||
],
|
||||
)
|
||||
def test_http_json_exporter_checks_json_size_before_sending(
|
||||
size_limit: int, expected: SpanExportResult, request_count: int
|
||||
) -> None:
|
||||
adapter: Final = Adapter()
|
||||
endpoint: Final = "https://collector.test/v1/traces"
|
||||
adapter.register_uri("POST", endpoint, status_code=200)
|
||||
with requests.Session() as session:
|
||||
session.mount("https://", adapter)
|
||||
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit)
|
||||
try:
|
||||
assert exporter.export(()) is expected
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
assert len(adapter.request_history) == request_count
|
||||
|
||||
|
||||
def test_http_json_exporter_rejects_export_after_shutdown() -> None:
|
||||
adapter: Final = Adapter()
|
||||
endpoint: Final = "https://collector.test/v1/traces"
|
||||
adapter.register_uri("POST", endpoint, status_code=200)
|
||||
with requests.Session() as session:
|
||||
session.mount("https://", adapter)
|
||||
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
|
||||
exporter.shutdown()
|
||||
assert exporter.export(()) is SpanExportResult.FAILURE
|
||||
assert adapter.request_history == []
|
||||
|
||||
|
||||
def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None:
|
||||
adapter: Final = Adapter()
|
||||
endpoint: Final = "https://collector.test/v1/traces"
|
||||
adapter.register_uri("POST", endpoint, status_code=200)
|
||||
with requests.Session() as session:
|
||||
session.mount("https://", adapter)
|
||||
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
|
||||
try:
|
||||
assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
assert adapter.request_history == []
|
||||
|
||||
|
||||
def test_http_protobuf_exporter_still_posts_protobuf():
|
||||
(_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector(
|
||||
"http/protobuf"
|
||||
|
|
@ -1630,9 +1725,13 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey
|
|||
severity_number=SeverityNumber.INFO,
|
||||
body="v2-tls-test",
|
||||
)
|
||||
log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test"))
|
||||
log_data: Final = ReadableLogRecord(
|
||||
log_record=record,
|
||||
resource=Resource({}),
|
||||
instrumentation_scope=InstrumentationScope("v2-tls-test"),
|
||||
)
|
||||
result = exporter.export([log_data])
|
||||
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
|
||||
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"
|
||||
assert tls_sink.received.get(timeout=5) == "/v1/logs"
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
|
|
|
|||
|
|
@ -382,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch):
|
|||
assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n"
|
||||
for path, response in responses.items():
|
||||
context = server_spans[path].context
|
||||
assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-01"]
|
||||
assert response.headers.get_list("traceresponse") == [
|
||||
f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path", ["/relayed", "/relayed/health"])
|
||||
|
|
@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo
|
|||
|
||||
headers = contextvars.Context().run(trace_response_headers, span)
|
||||
|
||||
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-01"
|
||||
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"
|
||||
|
|
|
|||
|
|
@ -2,13 +2,19 @@
|
|||
event loop: the preset does no network I/O, and a custom exporter mints the JWT
|
||||
lazily on its first export (in the BatchSpanProcessor worker thread)."""
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import requests_mock
|
||||
import respx
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
|
||||
from litellm.integrations.otel.plumbing import providers
|
||||
from litellm.integrations.otel.model.config import ExporterSpec
|
||||
from litellm.integrations.otel.presets import agentops as agentops_mod
|
||||
from litellm.integrations.otel.presets.agentops import (
|
||||
_AGENTOPS_AUTH_ENDPOINT,
|
||||
_AGENTOPS_ENDPOINT,
|
||||
_AGENTOPS_EXPORTER_KIND,
|
||||
_build_agentops_exporter,
|
||||
|
|
@ -75,50 +81,44 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch):
|
|||
)
|
||||
|
||||
|
||||
def test_agentops_exporter_mints_jwt_lazily(monkeypatch):
|
||||
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
|
||||
monkeypatch.setattr(
|
||||
agentops_mod, "_fetch_agentops_jwt", lambda _k: {"token": "jwt-xyz"}
|
||||
)
|
||||
spec = ExporterSpec(
|
||||
kind=_AGENTOPS_EXPORTER_KIND,
|
||||
endpoint=_AGENTOPS_ENDPOINT,
|
||||
options={"api_key": "ak"},
|
||||
)
|
||||
exporter = _build_agentops_exporter(spec)
|
||||
|
||||
# No auth header until the first export triggers the (off-loop) fetch.
|
||||
assert "Authorization" not in exporter._session.headers
|
||||
exporter._ensure_authenticated()
|
||||
assert exporter._session.headers["Authorization"] == "Bearer jwt-xyz"
|
||||
|
||||
# Cached: a second resolution does not re-fetch.
|
||||
calls = []
|
||||
monkeypatch.setattr(
|
||||
agentops_mod,
|
||||
"_fetch_agentops_jwt",
|
||||
lambda k: calls.append(k) or {"token": "again"},
|
||||
)
|
||||
exporter._ensure_authenticated()
|
||||
assert calls == []
|
||||
|
||||
|
||||
def test_agentops_exporter_tolerates_fetch_failure(monkeypatch):
|
||||
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
|
||||
|
||||
def _raise(_k):
|
||||
raise RuntimeError("auth down")
|
||||
|
||||
monkeypatch.setattr(agentops_mod, "_fetch_agentops_jwt", _raise)
|
||||
exporter = _build_agentops_exporter(
|
||||
def _agentops_exporter():
|
||||
return _build_agentops_exporter(
|
||||
ExporterSpec(
|
||||
kind=_AGENTOPS_EXPORTER_KIND,
|
||||
endpoint=_AGENTOPS_ENDPOINT,
|
||||
options={"api_key": "ak"},
|
||||
)
|
||||
)
|
||||
exporter._ensure_authenticated() # must not raise
|
||||
assert "Authorization" not in exporter._session.headers
|
||||
|
||||
|
||||
def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch():
|
||||
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
|
||||
with respx.mock() as auth, requests_mock.Mocker() as collector:
|
||||
token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"})
|
||||
collector.post(_AGENTOPS_ENDPOINT, status_code=200)
|
||||
exporter = _agentops_exporter()
|
||||
assert token_route.call_count == 0, "building the exporter must not fetch the JWT"
|
||||
|
||||
results = [exporter.export(()), exporter.export(())]
|
||||
|
||||
assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS]
|
||||
assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}]
|
||||
assert [request.headers.get("Authorization") for request in collector.request_history] == [
|
||||
"Bearer jwt-xyz",
|
||||
"Bearer jwt-xyz",
|
||||
]
|
||||
|
||||
|
||||
def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails():
|
||||
pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter")
|
||||
with respx.mock() as auth, requests_mock.Mocker() as collector:
|
||||
auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down"))
|
||||
collector.post(_AGENTOPS_ENDPOINT, status_code=200)
|
||||
|
||||
result = _agentops_exporter().export(())
|
||||
|
||||
assert result is SpanExportResult.SUCCESS
|
||||
assert [request.headers.get("Authorization") for request in collector.request_history] == [None]
|
||||
|
||||
|
||||
def test_fetch_jwt_uses_owned_client_not_shared_pool(monkeypatch):
|
||||
|
|
|
|||
|
|
@ -513,6 +513,22 @@ class TestLangfuseOtelIntegration:
|
|||
# Endpoint assertion removed as side effect is gone
|
||||
|
||||
|
||||
def _exported_auth_headers(exporter):
|
||||
import requests_mock
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
|
||||
with requests_mock.Mocker() as collector:
|
||||
collector.post(requests_mock.ANY, status_code=200)
|
||||
assert exporter.export(()) is SpanExportResult.SUCCESS
|
||||
return [
|
||||
(
|
||||
request.url,
|
||||
{name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")},
|
||||
)
|
||||
for request in collector.request_history
|
||||
]
|
||||
|
||||
|
||||
class TestLangfuseOtelKeyDynamicConfig:
|
||||
"""Key/team-scoped Langfuse credentials must define the full export target
|
||||
(OTLP endpoint + auth), not just auth headers on the init-time exporter."""
|
||||
|
|
@ -617,15 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig:
|
|||
|
||||
exporter = span_processors[0].span_exporter
|
||||
assert isinstance(exporter, OTLPSpanExporter)
|
||||
assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces"
|
||||
|
||||
import base64
|
||||
|
||||
expected_auth = base64.b64encode(b"key_public:key_secret").decode()
|
||||
assert exporter._headers == {
|
||||
"Authorization": f"Basic {expected_auth}",
|
||||
"x-langfuse-ingestion-version": "4",
|
||||
}
|
||||
assert _exported_auth_headers(exporter) == [
|
||||
(
|
||||
"https://langfuse.example.com/api/public/otel/v1/traces",
|
||||
{"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"},
|
||||
)
|
||||
]
|
||||
|
||||
def test_key_dynamic_params_reuse_cached_provider(self):
|
||||
with self._clean_env():
|
||||
|
|
@ -678,10 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig:
|
|||
provider = next(iter(logger._tracer_provider_cache.values())).provider
|
||||
exporter = provider._active_span_processor._span_processors[0].span_exporter
|
||||
assert isinstance(exporter, OTLPSpanExporter)
|
||||
assert exporter._headers == {
|
||||
"Authorization": f"Basic {secret}",
|
||||
"x-langfuse-ingestion-version": "4",
|
||||
}
|
||||
assert _exported_auth_headers(exporter) == [
|
||||
(
|
||||
"https://langfuse.example.com/api/public/otel/v1/traces",
|
||||
{"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
class TestLangfuseOtelResponsesAPI:
|
||||
|
|
|
|||
|
|
@ -19,14 +19,16 @@ import pytest
|
|||
|
||||
# Adds the grandparent directory to sys.path to allow importing project modules
|
||||
from opentelemetry import trace
|
||||
from opentelemetry._logs import LogRecord
|
||||
from opentelemetry._logs.severity import SeverityNumber
|
||||
from opentelemetry.sdk._logs import LogData, LogRecord
|
||||
from opentelemetry.sdk._logs import ReadableLogRecord
|
||||
from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor
|
||||
from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogRecordExportResult, SimpleLogRecordProcessor
|
||||
from opentelemetry.sdk.metrics import MeterProvider
|
||||
from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData
|
||||
from opentelemetry.sdk.resources import Resource
|
||||
from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExportResult
|
||||
from opentelemetry.sdk.util.instrumentation import InstrumentationScope
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from parameterized import parameterized
|
||||
|
|
@ -2321,9 +2323,13 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat
|
|||
severity_number=SeverityNumber.INFO,
|
||||
body="tls-export-test",
|
||||
)
|
||||
log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test"))
|
||||
log_data: Final = ReadableLogRecord(
|
||||
log_record=record,
|
||||
resource=Resource({}),
|
||||
instrumentation_scope=InstrumentationScope("tls-export-test"),
|
||||
)
|
||||
result: Final = exporter.export([log_data])
|
||||
assert result is LogExportResult.SUCCESS, f"log export failed: {result}"
|
||||
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"
|
||||
assert tls_sink.received.get(timeout=5) == "/v1/logs"
|
||||
finally:
|
||||
exporter.shutdown()
|
||||
|
|
@ -6639,7 +6645,7 @@ class TestOpenTelemetryProviderlessCallAttributes(unittest.TestCase):
|
|||
time.sleep(self.POLL_INTERVAL)
|
||||
return None
|
||||
|
||||
def _emitted_log_records(self, semconv_opt_in: str) -> tuple[LogData, ...]:
|
||||
def _emitted_log_records(self, semconv_opt_in: str) -> tuple[ReadableLogRecord, ...]:
|
||||
log_exporter = InMemoryLogExporter()
|
||||
logger_provider = OTLoggerProvider()
|
||||
logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter))
|
||||
|
|
@ -6713,7 +6719,12 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
|
|||
must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated
|
||||
until pods were OOMKilled)."""
|
||||
|
||||
BSP_THREAD_NAME = "OtelBatchSpanProcessor"
|
||||
def setUp(self) -> None:
|
||||
processor: Final = BatchSpanProcessor(InMemorySpanExporter())
|
||||
try:
|
||||
self.bsp_thread_name = processor._batch_processor._worker_thread.name
|
||||
finally:
|
||||
processor.shutdown()
|
||||
|
||||
def _logger(self, cap=3, exporter="console"):
|
||||
logger = OpenTelemetry(
|
||||
|
|
@ -6730,7 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
|
|||
logger._tracer_provider_cache.clear()
|
||||
|
||||
def _live_exporter_threads(self):
|
||||
return [t for t in threading.enumerate() if t.name == self.BSP_THREAD_NAME]
|
||||
return [t for t in threading.enumerate() if t.name == self.bsp_thread_name]
|
||||
|
||||
def _wait_for_exporter_threads(self, expected, timeout=10.0):
|
||||
"""Dropped providers are shut down off-thread, so poll instead of sleeping."""
|
||||
|
|
@ -6751,8 +6762,6 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
|
|||
logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"})
|
||||
|
||||
self.assertEqual(len(logger._tracer_provider_cache), 3)
|
||||
# Guards the thread-name constant: a rename upstream would make this read 0 and the
|
||||
# bound assertion below would pass while measuring nothing.
|
||||
self.assertGreaterEqual(len(self._live_exporter_threads()), 1)
|
||||
self.assertLessEqual(self._wait_for_exporter_threads(before + 3) - before, 3)
|
||||
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from litellm import CustomLLM
|
|||
from litellm.litellm_core_utils import get_llm_provider_logic
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import (
|
||||
get_llm_provider,
|
||||
get_provider_affinity_header,
|
||||
inferred_provider,
|
||||
is_registered_custom_provider,
|
||||
)
|
||||
|
|
@ -706,3 +707,11 @@ class TestClaudeModelPatternMatching:
|
|||
set_fallback_generalizations([])
|
||||
with pytest.raises(litellm.BadRequestError):
|
||||
litellm.get_llm_provider(model="claude-opus-4-9")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
|
||||
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
|
||||
api_base: Final = "https://opencode.ai/zen/go/v1"
|
||||
model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base)
|
||||
assert (model, provider, destination) == ("test-model", prefix, api_base)
|
||||
assert get_provider_affinity_header(destination) == "x-opencode-session"
|
||||
|
|
|
|||
|
|
@ -399,9 +399,10 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in():
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refetch_respects_local_env_override(monkeypatch):
|
||||
"""LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
|
||||
async def test_refetch_respects_local_env_override(monkeypatch, flag):
|
||||
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP."""
|
||||
monkeypatch.setenv(flag, "True")
|
||||
|
||||
def _fail(request):
|
||||
raise AssertionError("no HTTP request should be made when local map is forced")
|
||||
|
|
@ -708,9 +709,10 @@ def test_boot_load_falls_back_when_urlsplit_rejects_the_url():
|
|||
assert "invalid URL" in (source["fallback_reason"] or "")
|
||||
|
||||
|
||||
def test_boot_load_respects_local_env_override(monkeypatch):
|
||||
"""LITELLM_LOCAL_MODEL_COST_MAP=True still short-circuits to the backup with zero HTTP."""
|
||||
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
|
||||
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
|
||||
def test_boot_load_respects_local_env_override(monkeypatch, flag):
|
||||
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP."""
|
||||
monkeypatch.setenv(flag, "True")
|
||||
|
||||
def _fail(request):
|
||||
raise AssertionError("no HTTP request should be made when local map is forced")
|
||||
|
|
|
|||
|
|
@ -2047,7 +2047,9 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
|
|||
) -> None:
|
||||
"""The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key,
|
||||
without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS."""
|
||||
import requests_mock
|
||||
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
|
||||
from opentelemetry.sdk.trace.export import SpanExportResult
|
||||
|
||||
from litellm.integrations.opentelemetry import OpenTelemetry
|
||||
from litellm.litellm_core_utils import litellm_logging as logging_module
|
||||
|
|
@ -2069,8 +2071,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
|
|||
assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace"
|
||||
exporter: Final = logger._get_span_processor().span_exporter
|
||||
assert isinstance(exporter, OTLPSpanExporter)
|
||||
assert exporter._endpoint == expected_endpoint
|
||||
assert exporter._headers == {"x-api-key": api_key}
|
||||
with requests_mock.Mocker() as collector:
|
||||
collector.post(expected_endpoint, status_code=200)
|
||||
assert exporter.export(()) is SpanExportResult.SUCCESS
|
||||
assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [
|
||||
(expected_endpoint, api_key)
|
||||
]
|
||||
assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ
|
||||
finally:
|
||||
logging_module._in_memory_loggers.clear()
|
||||
|
|
|
|||
17
tests/unit/litellm_core_utils/test_offline_mode.py
Normal file
17
tests/unit/litellm_core_utils/test_offline_mode.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
import pytest
|
||||
|
||||
from litellm.litellm_core_utils.offline_mode import is_offline_mode
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[("true", True), ("True", True), (" TRUE ", True), ("false", False), ("", False), ("1", False), ("yes", False)],
|
||||
)
|
||||
def test_is_offline_mode_reads_true_case_insensitively(monkeypatch, value, expected):
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", value)
|
||||
assert is_offline_mode() is expected
|
||||
|
||||
|
||||
def test_is_offline_mode_is_off_when_unset(monkeypatch):
|
||||
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
|
||||
assert is_offline_mode() is False
|
||||
|
|
@ -1173,6 +1173,27 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch:
|
|||
assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["llama-2-7b", "meta-llama/llama-3-8b-instruct"])
|
||||
def test_offline_mode_counts_downloaded_tokenizer_models_with_tiktoken(monkeypatch: pytest.MonkeyPatch, model: str):
|
||||
_load_huggingface_tokenizer.cache_clear()
|
||||
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
|
||||
|
||||
with patch("litellm.utils.tokenizer_dispatch.from_pretrained") as from_pretrained:
|
||||
result: Final = _select_tokenizer_helper(model)
|
||||
|
||||
from_pretrained.assert_not_called()
|
||||
assert result["type"] == "openai_tokenizer"
|
||||
|
||||
|
||||
def test_offline_mode_keeps_deepseek_v4_on_its_bundled_tokenizer(monkeypatch: pytest.MonkeyPatch):
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
|
||||
|
||||
assert litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) == _deepseek_token_count(
|
||||
_CODE_SAMPLE
|
||||
)
|
||||
|
||||
|
||||
def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer():
|
||||
_load_huggingface_tokenizer.cache_clear()
|
||||
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,10 @@
|
|||
import litellm
|
||||
import json
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import litellm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -134,6 +138,34 @@ async def test_aws_polly_tts_with_native_voice():
|
|||
assert request_body.get("TextType", "text") == "text"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
@pytest.mark.parametrize("engine", ["standard", "neural", "long-form", "generative"])
|
||||
async def test_aws_polly_tts_sends_engine_from_model(engine: str) -> None:
|
||||
"""
|
||||
The engine in the model name must reach Polly. speech() strips the
|
||||
provider prefix before the config reads it, which used to turn every
|
||||
engine into the neural default.
|
||||
"""
|
||||
mock_httpx_response = MagicMock(spec=httpx.Response)
|
||||
mock_httpx_response.content = b"fake_audio_data"
|
||||
mock_httpx_response.status_code = 200
|
||||
mock_httpx_response.headers = {"content-type": "audio/mpeg"}
|
||||
|
||||
with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post:
|
||||
mock_post.return_value = mock_httpx_response
|
||||
|
||||
await litellm.aspeech(
|
||||
model=f"aws_polly/{engine}",
|
||||
voice="Joanna",
|
||||
input="Hello",
|
||||
aws_region_name="us-east-1",
|
||||
)
|
||||
|
||||
request_body = json.loads(mock_post.call_args.kwargs["data"])
|
||||
assert request_body["Engine"] == engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("fake_provider_credentials")
|
||||
async def test_aws_polly_tts_with_openai_voice_mapping():
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import json
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig
|
||||
|
||||
|
||||
|
|
@ -17,3 +19,17 @@ def test_installed_botocore_signs_the_speech_request() -> None:
|
|||
authorization: Final = headers["Authorization"]
|
||||
assert "/us-west-2/polly/aws4_request" in authorization
|
||||
assert json.loads(body) == {"Text": "ping", "VoiceId": "Joanna"}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected"),
|
||||
[
|
||||
("standard", "standard"),
|
||||
("Long-Form", "long-form"),
|
||||
("aws_polly/generative", "generative"),
|
||||
("aws_polly", "neural"),
|
||||
("unknown", "neural"),
|
||||
],
|
||||
)
|
||||
def test_extract_engine_from_model(model: str, expected: str) -> None:
|
||||
assert AWSPollyTextToSpeechConfig()._extract_engine_from_model(model) == expected
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
from tests.unit.proxy.conftest import httpx_transport
|
||||
|
||||
|
||||
@pytest.mark.serial
|
||||
|
|
@ -100,6 +101,69 @@ def test_get_complete_url():
|
|||
assert result == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("base_path", "expected_path"),
|
||||
[
|
||||
("/foundry/openai/openai/v1", "/foundry/openai/openai/v1/responses?api-version=v1"),
|
||||
("/foundry/openai/openai/v1/", "/foundry/openai/openai/v1/responses?api-version=v1"),
|
||||
("/foundry/openai/openai/v1/responses", "/foundry/openai/openai/v1/responses?api-version=v1"),
|
||||
("/foundry/openai/openai/v1/responses/", "/foundry/openai/openai/v1/responses?api-version=v1"),
|
||||
(
|
||||
"/openai/v1?redirect=/openai/v1/responses",
|
||||
"/openai/v1/responses?redirect=%2Fopenai%2Fv1%2Fresponses&api-version=v1",
|
||||
),
|
||||
(
|
||||
"/openai/v1?tenant=example&api-version=preview",
|
||||
"/openai/v1/responses?tenant=example&api-version=preview",
|
||||
),
|
||||
(
|
||||
"/openai/v1/responses?api-version=2025-04-01-preview",
|
||||
"/openai/v1/responses?api-version=2025-04-01-preview",
|
||||
),
|
||||
(
|
||||
"/openai/v1/responses?redirect=/openai/responses",
|
||||
"/openai/v1/responses?redirect=%2Fopenai%2Fresponses&api-version=v1",
|
||||
),
|
||||
("/foo%2Fbar/openai/v1?tenant=example", "/foo%2Fbar/openai/v1/responses?tenant=example&api-version=v1"),
|
||||
(
|
||||
"/foo%3Fbar/openai/v1/responses?tenant=example",
|
||||
"/foo%3Fbar/openai/v1/responses?tenant=example&api-version=v1",
|
||||
),
|
||||
("/openai", "/openai/v1/responses?api-version=v1"),
|
||||
("/foundry/openai", "/foundry/openai/v1/responses?api-version=v1"),
|
||||
],
|
||||
)
|
||||
def test_explicit_v1_api_base_preserves_the_gateway_prefix_and_query(base_path: str, expected_path: str) -> None:
|
||||
result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url(
|
||||
api_base=f"https://gateway.example.com{base_path}", litellm_params={"api_version": "v1"}
|
||||
)
|
||||
|
||||
assert result == f"https://gateway.example.com{expected_path}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("litellm_params", "expected_version"),
|
||||
[
|
||||
({}, AZURE_DEFAULT_RESPONSES_API_VERSION),
|
||||
({"api_version": "v1"}, "v1"),
|
||||
({"api_version": "latest"}, "latest"),
|
||||
({"api_version": "preview"}, "preview"),
|
||||
({"api_version": "2025-04-01-preview"}, "2025-04-01-preview"),
|
||||
({"api_version": "2024-10-21"}, AZURE_DEFAULT_RESPONSES_API_VERSION),
|
||||
({"api_version": "2024-10-21", "responses_api_version": "preview"}, "preview"),
|
||||
],
|
||||
)
|
||||
def test_explicit_v1_api_base_keeps_responses_api_version_selection(
|
||||
litellm_params: dict[str, str], expected_version: str
|
||||
) -> None:
|
||||
result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url(
|
||||
api_base="https://gateway.example.com/foundry/openai/openai/v1",
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
assert result == f"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version={expected_version}"
|
||||
|
||||
|
||||
@pytest.mark.serial
|
||||
def test_response_id_path_requests_encode_response_id():
|
||||
config = AzureOpenAIResponsesAPIConfig()
|
||||
|
|
@ -947,6 +1011,119 @@ def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url(
|
|||
assert route.calls[0].request.url == _unset_api_version_responses_url()
|
||||
|
||||
|
||||
_COMPLETED_RESPONSE_SSE: Final = "".join(
|
||||
f"data: {json.dumps(event)}\n\n"
|
||||
for event in (
|
||||
{
|
||||
"type": "response.output_text.delta",
|
||||
"sequence_number": 0,
|
||||
"item_id": "msg_1",
|
||||
"output_index": 0,
|
||||
"content_index": 0,
|
||||
"delta": "Done.",
|
||||
},
|
||||
{"type": "response.completed", "sequence_number": 1, "response": _COMPLETED_RESPONSE},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def gateway_responses(respx_mock: respx.MockRouter, stream: bool) -> respx.Route:
|
||||
body: Final = _COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE)
|
||||
content_type: Final = "text/event-stream" if stream else "application/json"
|
||||
return respx_mock.post(
|
||||
"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version=v1"
|
||||
).mock(return_value=httpx.Response(200, text=body, headers={"content-type": content_type}))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_explicit_v1_base_sends_responses_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None:
|
||||
response: Final = litellm.responses(
|
||||
model="azure/gpt-6.1-sol",
|
||||
input="hi",
|
||||
api_base="https://gateway.example.com/foundry/openai/openai/v1",
|
||||
api_key="azure-test-key",
|
||||
api_version="v1",
|
||||
stream=stream,
|
||||
)
|
||||
text: Final = (
|
||||
"".join(event.delta for event in response if event.type == "response.output_text.delta")
|
||||
if stream
|
||||
else response.output[0].content[0].text
|
||||
)
|
||||
|
||||
assert text == "Done."
|
||||
assert gateway_responses.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
def test_explicit_v1_base_sends_bridged_chat_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None:
|
||||
response: Final = litellm.completion(
|
||||
model="azure/responses/gpt-6.1-sol",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://gateway.example.com/foundry/openai/openai/v1",
|
||||
api_key="azure-test-key",
|
||||
api_version="v1",
|
||||
stream=stream,
|
||||
)
|
||||
text: Final = (
|
||||
"".join(chunk.choices[0].delta.content or "" for chunk in response)
|
||||
if stream
|
||||
else response.choices[0].message.content
|
||||
)
|
||||
|
||||
assert text == "Done."
|
||||
assert gateway_responses.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_explicit_v1_base_sends_async_responses_to_the_gateway(
|
||||
gateway_responses: respx.Route, stream: bool
|
||||
) -> None:
|
||||
response: Final = await litellm.aresponses(
|
||||
model="azure/gpt-6.1-sol",
|
||||
input="hi",
|
||||
api_base="https://gateway.example.com/foundry/openai/openai/v1",
|
||||
api_key="azure-test-key",
|
||||
api_version="v1",
|
||||
stream=stream,
|
||||
)
|
||||
text: Final = (
|
||||
"".join([event.delta async for event in response if event.type == "response.output_text.delta"])
|
||||
if stream
|
||||
else response.output[0].content[0].text
|
||||
)
|
||||
|
||||
assert text == "Done."
|
||||
assert gateway_responses.call_count == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures(httpx_transport.__name__)
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
async def test_explicit_v1_base_sends_async_bridged_chat_to_the_gateway(
|
||||
gateway_responses: respx.Route, stream: bool
|
||||
) -> None:
|
||||
response: Final = await litellm.acompletion(
|
||||
model="azure/responses/gpt-6.1-sol",
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base="https://gateway.example.com/foundry/openai/openai/v1",
|
||||
api_key="azure-test-key",
|
||||
api_version="v1",
|
||||
stream=stream,
|
||||
)
|
||||
text: Final = (
|
||||
"".join([chunk.choices[0].delta.content or "" async for chunk in response])
|
||||
if stream
|
||||
else response.choices[0].message.content
|
||||
)
|
||||
|
||||
assert text == "Done."
|
||||
assert gateway_responses.call_count == 1
|
||||
|
||||
|
||||
_LIFECYCLE_BODIES: Final = {
|
||||
"get": _COMPLETED_RESPONSE,
|
||||
"delete": {"id": "resp_dated", "object": "response", "deleted": True},
|
||||
|
|
|
|||
|
|
@ -879,6 +879,32 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages():
|
|||
}
|
||||
|
||||
|
||||
def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text():
|
||||
messages: Final = [
|
||||
{"role": "user", "content": "load the WebFetch tool"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}],
|
||||
},
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "call_1",
|
||||
"content": [{"type": "tool_reference", "tool_name": "WebFetch"}],
|
||||
},
|
||||
]
|
||||
|
||||
request: Final = AzureAIStudioConfig().transform_request(
|
||||
model="fw-glm-5.2",
|
||||
messages=messages,
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base, expected_url",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import asyncio
|
||||
import base64
|
||||
import contextlib
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
|
|
@ -18,6 +19,9 @@ import tiktoken
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.llms.bedrock.count_tokens.kimi_k3 import (
|
||||
HUGGING_FACE_SOURCE,
|
||||
KIMI_K3_RANKS_FILE,
|
||||
KIMI_K3_REVISION,
|
||||
KimiK3Encodings,
|
||||
KimiK3TokenizerSource,
|
||||
kimi_k3_anthropic_tokens,
|
||||
|
|
@ -522,3 +526,40 @@ async def test_tokenizer_load_still_running_after_the_wait_is_logged(
|
|||
|
||||
assert loaded is None
|
||||
assert any("Kimi K3 tokenizer still loading" in record.getMessage() for record in caplog.records)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("offline_mode", "disable_download", "asks_hub"),
|
||||
[("true", None, False), ("false", True, False), ("false", None, True)],
|
||||
)
|
||||
def test_offline_mode_and_disabled_downloads_never_ask_hugging_face_for_the_tokenizer(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path, offline_mode: str, disable_download: bool | None, asks_hub: bool
|
||||
):
|
||||
import huggingface_hub.constants
|
||||
import huggingface_hub.utils._detect_agent
|
||||
|
||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path / "hub"))
|
||||
monkeypatch.setattr(huggingface_hub.constants, "AGENT_HARNESSES_PATH", str(tmp_path / "agent_harnesses.json"))
|
||||
monkeypatch.setattr(huggingface_hub.utils._detect_agent, "_registry", None)
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", offline_mode)
|
||||
monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", disable_download)
|
||||
with respx.mock(assert_all_called=False) as router:
|
||||
hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503))
|
||||
with contextlib.suppress(Exception):
|
||||
HUGGING_FACE_SOURCE.ranks_path()
|
||||
assert hub.called is asks_hub
|
||||
|
||||
|
||||
def test_offline_mode_reads_the_tokenizer_from_the_hugging_face_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path):
|
||||
import huggingface_hub.constants
|
||||
|
||||
cached: Final = tmp_path / "models--moonshotai--Kimi-K3" / "snapshots" / KIMI_K3_REVISION / KIMI_K3_RANKS_FILE
|
||||
cached.parent.mkdir(parents=True)
|
||||
cached.write_bytes(b"cached ranks")
|
||||
monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path))
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
|
||||
with respx.mock(assert_all_called=False) as router:
|
||||
hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503))
|
||||
path: Final = HUGGING_FACE_SOURCE.ranks_path()
|
||||
assert Path(path).read_bytes() == b"cached ranks"
|
||||
assert not hub.called
|
||||
|
|
|
|||
|
|
@ -5542,3 +5542,85 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_
|
|||
sent: Final = received[0]
|
||||
assert "safeguards" not in json.loads(sent.content)
|
||||
assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"])
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "explicit_header", "expected_session"),
|
||||
[
|
||||
("https://opencode.ai/inference/go/anthropic", None, "conversation-123"),
|
||||
("https://opencode.ai/inference/go/anthropic", "caller-override", "caller-override"),
|
||||
("https://other.example", None, None),
|
||||
],
|
||||
)
|
||||
async def test_messages_opencode_session_header(
|
||||
stream: bool,
|
||||
environment_base: str | None,
|
||||
api_base: str,
|
||||
explicit_header: str | None,
|
||||
expected_session: str | None,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False)
|
||||
monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False)
|
||||
if environment_base:
|
||||
monkeypatch.setenv(environment_base, "https://opencode.ai/inference/go/anthropic")
|
||||
received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
|
||||
payload: Final = {
|
||||
"id": "msg_test",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-haiku-5-5",
|
||||
"content": [{"type": "text", "text": "OK"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 1},
|
||||
}
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
received.put_nowait(request)
|
||||
return (
|
||||
httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
text=f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': payload})}\n\n"
|
||||
'event: message_stop\ndata: {"type":"message_stop"}\n\n',
|
||||
)
|
||||
if stream
|
||||
else httpx.Response(200, json=payload)
|
||||
)
|
||||
|
||||
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
|
||||
try:
|
||||
result: Final = await litellm.anthropic.messages.acreate(
|
||||
model="anthropic/claude-haiku-5-5",
|
||||
messages=[{"role": "user", "content": "hello"}],
|
||||
max_tokens=128000,
|
||||
api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base,
|
||||
api_key="test-key",
|
||||
stream=stream,
|
||||
litellm_session_id="conversation-123",
|
||||
client=upstream,
|
||||
extra_headers={"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"},
|
||||
)
|
||||
if stream:
|
||||
chunks: Final = b"".join([chunk async for chunk in result])
|
||||
assert b'"text": "OK"' in chunks
|
||||
assert b"message_stop" in chunks
|
||||
else:
|
||||
assert result["content"] == [{"type": "text", "text": "OK"}]
|
||||
finally:
|
||||
await upstream.close()
|
||||
sent: Final = received.get_nowait()
|
||||
assert str(sent.url) == f"{api_base}/v1/messages"
|
||||
assert sent.headers.get("x-opencode-session") == expected_session
|
||||
assert sent.headers["x-api-key"] == "test-key"
|
||||
assert json.loads(sent.content) == {
|
||||
"model": "claude-haiku-5-5",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 128000,
|
||||
"stream": stream,
|
||||
}
|
||||
assert received.empty()
|
||||
|
|
|
|||
90
tests/unit/llms/opencode/test_transformation.py
Normal file
90
tests/unit/llms/opencode/test_transformation.py
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[
|
||||
"https://opencode.ai/zen/v1",
|
||||
"https://opencode.ai/zen/go/v1/",
|
||||
"https://opencode.ai/inference/openai/v1",
|
||||
"https://opencode.ai/inference/go/openai/v1",
|
||||
"https://opencode.ai/inference/go/anthropic/v1",
|
||||
"https://opencode.ai/inference/anthropic/v1",
|
||||
],
|
||||
)
|
||||
def test_opencode_defaults_affinity_header_to_the_callers_session(api_base: str) -> None:
|
||||
headers: Final = {"Authorization": "Bearer test-key"}
|
||||
params: Final = {"metadata": {"session_id": "conversation-123"}}
|
||||
assert add_provider_affinity_header(
|
||||
headers, params, OpenCodeProviderAdapter().get_provider_affinity_header(api_base)
|
||||
) == {
|
||||
**headers,
|
||||
"x-opencode-session": "conversation-123",
|
||||
}
|
||||
assert headers == {"Authorization": "Bearer test-key"}
|
||||
assert params == {"metadata": {"session_id": "conversation-123"}}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"api_base",
|
||||
[
|
||||
None,
|
||||
"https://api.openai.com/v1",
|
||||
"https://opencode.ai.example/zen/go/v1",
|
||||
"https://opencode.ai@example.com/zen/go/v1",
|
||||
"https://example.com/opencode.ai/zen/go/v1",
|
||||
"https://example.com/v1?upstream=https://opencode.ai/zen/go/v1",
|
||||
"https://opencode.ai/console/v1",
|
||||
"http://opencode.ai/zen/go/v1",
|
||||
],
|
||||
)
|
||||
def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: str | None) -> None:
|
||||
headers: Final = {"Authorization": "Bearer test-key", "x-existing": "unchanged"}
|
||||
assert (
|
||||
add_provider_affinity_header(
|
||||
headers,
|
||||
{"litellm_session_id": "conversation-123"},
|
||||
OpenCodeProviderAdapter().get_provider_affinity_header(api_base),
|
||||
)
|
||||
== headers
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"params",
|
||||
[
|
||||
{},
|
||||
{"litellm_trace_id": "request-123"},
|
||||
{"metadata": {"trace_id": "request-123"}},
|
||||
{
|
||||
"litellm_session_id": "request-123",
|
||||
"litellm_metadata": {"session_id": "request-123", SESSION_ID_GENERATED_METADATA_KEY: True},
|
||||
},
|
||||
],
|
||||
)
|
||||
def test_opencode_does_not_invent_a_conversation_id(params: dict[str, object]) -> None:
|
||||
assert (
|
||||
add_provider_affinity_header(
|
||||
{}, params, OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1")
|
||||
)
|
||||
== {}
|
||||
)
|
||||
|
||||
|
||||
def test_opencode_preserves_explicit_header_and_affinity_configuration() -> None:
|
||||
assert add_provider_affinity_header(
|
||||
{"X-OpenCode-Session": "explicit"},
|
||||
{"litellm_session_id": "conversation-123"},
|
||||
OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"),
|
||||
) == {"X-OpenCode-Session": "explicit"}
|
||||
assert add_provider_affinity_header(
|
||||
{},
|
||||
{"litellm_session_id": "conversation-123", "provider_affinity_header": "x-custom-session"},
|
||||
OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"),
|
||||
) == {"x-custom-session": "conversation-123"}
|
||||
|
|
@ -180,6 +180,18 @@ async def test_breach_check_skipped_when_disabled():
|
|||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_breach_check_skipped_in_offline_mode(monkeypatch):
|
||||
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
|
||||
password = "correct horse battery staple"
|
||||
result = await validate_password_not_breached(
|
||||
password=password,
|
||||
general_settings={},
|
||||
client=_client_returning(f"{_sha1_upper(password)[5:]}:42"),
|
||||
)
|
||||
assert result is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rejects_breached_password():
|
||||
password = "correct horse battery staple"
|
||||
|
|
|
|||
|
|
@ -11688,6 +11688,124 @@ def _alias_request(route: str, data: dict, content_type: str = "application/json
|
|||
return request
|
||||
|
||||
|
||||
def _viewer_inference_key(monkeypatch: pytest.MonkeyPatch, team_id: str | None, *, spend: float = 0.25) -> str:
|
||||
from litellm.proxy import proxy_server
|
||||
from litellm.proxy.utils import ProxyLogging, hash_token
|
||||
|
||||
key: Final = "sk-viewer-inference"
|
||||
hashed_key: Final = hash_token(key)
|
||||
cache: Final = UserApiKeyCache()
|
||||
cache.set_cache(
|
||||
key=hashed_key,
|
||||
value=UserAPIKeyAuth(
|
||||
token=hashed_key,
|
||||
user_id="viewer",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
team_id=team_id,
|
||||
models=["gpt-6.1-sol"],
|
||||
max_budget=1.0,
|
||||
spend=spend,
|
||||
),
|
||||
)
|
||||
cache.set_cache(
|
||||
key="viewer",
|
||||
value=LiteLLM_UserTable(
|
||||
user_id="viewer",
|
||||
user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
user_email="viewer@example.com",
|
||||
models=["gpt-6.1-sol"],
|
||||
),
|
||||
)
|
||||
if team_id is not None:
|
||||
cache.set_cache(
|
||||
key=team_membership_reservation_cache_key(user_id="viewer", team_id=team_id),
|
||||
value=NO_TEAM_MEMBERSHIP_SENTINEL,
|
||||
)
|
||||
for name, value in {
|
||||
**_proxy_attrs_for_centralized_checks(),
|
||||
"prisma_client": MagicMock(),
|
||||
"user_api_key_cache": cache,
|
||||
"spend_counter_cache": DualCache(),
|
||||
"proxy_logging_obj": ProxyLogging(user_api_key_cache=cache),
|
||||
"llm_model_list": [],
|
||||
}.items():
|
||||
monkeypatch.setattr(proxy_server, name, value)
|
||||
return key
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
|
||||
@pytest.mark.parametrize("route", ["/chat/completions", "/v1/chat/completions"])
|
||||
async def test_viewer_inference_uses_the_normal_key_authorization_path(
|
||||
monkeypatch: pytest.MonkeyPatch, team_id: str | None, route: str
|
||||
) -> None:
|
||||
key: Final = _viewer_inference_key(monkeypatch, team_id)
|
||||
request: Final = _alias_request(route, {"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "usage"}]})
|
||||
|
||||
result: Final = await user_api_key_auth(request=request, api_key=f"Bearer {key}")
|
||||
|
||||
assert (result.user_id, result.user_role, result.team_id, result.models, result.max_budget) == (
|
||||
"viewer",
|
||||
LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY,
|
||||
team_id,
|
||||
["gpt-6.1-sol"],
|
||||
1.0,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
|
||||
async def test_viewer_inference_rejects_a_model_outside_the_key_allowlist(
|
||||
monkeypatch: pytest.MonkeyPatch, team_id: str | None
|
||||
) -> None:
|
||||
key: Final = _viewer_inference_key(monkeypatch, team_id)
|
||||
request: Final = _alias_request("/chat/completions", {"model": "blocked-model"})
|
||||
|
||||
with pytest.raises(ProxyException) as failure:
|
||||
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
|
||||
|
||||
assert (failure.value.code, failure.value.type, failure.value.param) == (
|
||||
"403",
|
||||
ProxyErrorTypes.key_model_access_denied,
|
||||
"model",
|
||||
)
|
||||
assert failure.value.message == (
|
||||
"The requested model 'blocked-model' is not available for this API key, or the model name is invalid. "
|
||||
"Check the models available to you and try again."
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
|
||||
@pytest.mark.parametrize("spend", [1.0, 1.25], ids=["at-limit", "over-limit"])
|
||||
async def test_viewer_inference_rejects_an_exhausted_key_budget(
|
||||
monkeypatch: pytest.MonkeyPatch, team_id: str | None, spend: float
|
||||
) -> None:
|
||||
key: Final = _viewer_inference_key(monkeypatch, team_id, spend=spend)
|
||||
request: Final = _alias_request("/chat/completions", {"model": "gpt-6.1-sol"})
|
||||
|
||||
with pytest.raises(ProxyException) as failure:
|
||||
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
|
||||
|
||||
assert failure.value.type == ProxyErrorTypes.budget_exceeded
|
||||
assert f"Current cost: {spend}, Max budget: 1.0" in failure.value.message
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"])
|
||||
async def test_viewer_inference_access_does_not_allow_management_writes(
|
||||
monkeypatch: pytest.MonkeyPatch, team_id: str | None
|
||||
) -> None:
|
||||
key: Final = _viewer_inference_key(monkeypatch, team_id)
|
||||
request: Final = _alias_request("/team/new", {"team_alias": "forbidden"})
|
||||
|
||||
with pytest.raises(ProxyException) as failure:
|
||||
await user_api_key_auth(request=request, api_key=f"Bearer {key}")
|
||||
|
||||
assert failure.value.code == "403"
|
||||
assert "proxy_admin_viewer" in failure.value.message
|
||||
|
||||
|
||||
async def _enforce_alias_access(token: UserAPIKeyAuth, data: dict, route: str, request, router: litellm.Router):
|
||||
from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access
|
||||
|
||||
|
|
@ -12480,11 +12598,14 @@ async def test_lens_oauth2_validates_identity_without_inference_user_field(
|
|||
patch("litellm.proxy.proxy_server.master_key", "sk-fixture-master"),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", None),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()),
|
||||
patch.dict(os.environ, {
|
||||
"OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect",
|
||||
"OAUTH_CLIENT_ID": "fixture-client",
|
||||
"OAUTH_CLIENT_SECRET": "fixture-client-secret",
|
||||
}),
|
||||
patch.dict(
|
||||
os.environ,
|
||||
{
|
||||
"OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect",
|
||||
"OAUTH_CLIENT_ID": "fixture-client",
|
||||
"OAUTH_CLIENT_SECRET": "fixture-client-secret",
|
||||
},
|
||||
),
|
||||
patch("httpx.AsyncClient.send", side_effect=token_info) as network,
|
||||
):
|
||||
if active:
|
||||
|
|
@ -12611,6 +12732,4 @@ def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery(
|
|||
) -> None:
|
||||
from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks
|
||||
|
||||
assert (
|
||||
_should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected
|
||||
)
|
||||
assert _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import (
|
|||
build_window_spend_transaction,
|
||||
)
|
||||
from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL
|
||||
from litellm.responses.streaming_iterator import drain_logging_tasks
|
||||
from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation
|
||||
from tests.unit.proxy.db.fake_prisma_engine import engine_call
|
||||
|
||||
|
|
@ -2158,46 +2159,122 @@ async def test_commit_spend_updates_to_db_increments_key_total_spend_alongside_s
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_database_creates_single_task():
|
||||
async def test_shutdown_drain_finishes_the_rollups_update_database_started():
|
||||
"""
|
||||
Test that update_database() fires exactly 1 asyncio.create_task() call
|
||||
(the batched task) instead of the previous 11.
|
||||
Regression for LIT-6027. The key, team and daily rollups ran in an unreferenced asyncio task,
|
||||
so the shutdown sequence (flush, drain_logging_tasks, flush) never waited for them and spend
|
||||
logs landed without matching daily rows. Once update_database returns, drain_logging_tasks
|
||||
must leave every rollup for that request in its queue.
|
||||
"""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
|
||||
# Mock all helpers so nothing real runs
|
||||
db_writer: Final = DBSpendUpdateWriter()
|
||||
db_writer._insert_spend_log_to_db = AsyncMock()
|
||||
db_writer._batch_database_updates = AsyncMock()
|
||||
|
||||
prisma: Final = MagicMock()
|
||||
prisma.get_request_status = MagicMock(return_value="success")
|
||||
fake_payload: Final = {
|
||||
"request_id": "req-lit-6027",
|
||||
"api_key": "hashed-key",
|
||||
"user": "test-user",
|
||||
"team_id": "test-team",
|
||||
"agent_id": None,
|
||||
"startTime": "2024-01-01T00:00:00",
|
||||
"endTime": "2024-01-01T00:00:01",
|
||||
"model": "gpt-4o-mini",
|
||||
"custom_llm_provider": "openai",
|
||||
"model_group": "gpt-4o-mini",
|
||||
"prompt_tokens": 10,
|
||||
"completion_tokens": 5,
|
||||
"spend": 0.0,
|
||||
"metadata": '{"usage_object": {}}',
|
||||
}
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.disable_spend_logs", False),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", prisma),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
|
||||
patch("litellm.proxy.db.db_spend_update_writer.asyncio.create_task") as mock_create_task,
|
||||
patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload),
|
||||
):
|
||||
await db_writer.update_database(
|
||||
token="test-token",
|
||||
user_id="test-user",
|
||||
end_user_id="test-end-user",
|
||||
end_user_id=None,
|
||||
team_id="test-team",
|
||||
org_id=None,
|
||||
kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"},
|
||||
completion_response=MagicMock(),
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
team_id="test-team",
|
||||
org_id="test-org",
|
||||
completion_response=MagicMock(),
|
||||
response_cost=0.1,
|
||||
kwargs={"model": "gpt-4", "custom_llm_provider": "openai"},
|
||||
)
|
||||
await drain_logging_tasks()
|
||||
|
||||
# Exactly 1 create_task call (the batch), not 11
|
||||
assert mock_create_task.call_count == 1
|
||||
spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
|
||||
assert spend["key_list_transactions"] == {"test-token": 0.1}
|
||||
assert spend["team_list_transactions"] == {"test-team": 0.1}
|
||||
daily_user: Final = (
|
||||
await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
)
|
||||
daily_team: Final = (
|
||||
await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
)
|
||||
assert daily_user is not None and daily_team is not None
|
||||
(user_row,) = daily_user.values()
|
||||
(team_row,) = daily_team.values()
|
||||
assert (user_row["api_requests"], user_row["spend"]) == (1, 0.1)
|
||||
assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_database_updates_isolation_on_failure():
|
||||
async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue():
|
||||
"""
|
||||
Regression for LIT-6027. A rollup enqueue that is still in flight when the shutdown drain
|
||||
starts (here: blocked on a full spend queue) must be finished by the time drain_logging_tasks
|
||||
returns, so the second shutdown flush commits it.
|
||||
"""
|
||||
enqueue_started: Final = asyncio.Event()
|
||||
release_enqueue: Final = asyncio.Event()
|
||||
|
||||
class _BlockedSpendQueue(SpendUpdateQueue):
|
||||
async def add_update(self, update: SpendUpdateQueueItem):
|
||||
enqueue_started.set()
|
||||
await release_enqueue.wait()
|
||||
await super().add_update(update)
|
||||
|
||||
db_writer: Final = DBSpendUpdateWriter()
|
||||
db_writer.spend_update_queue = _BlockedSpendQueue()
|
||||
db_writer._insert_spend_log_to_db = AsyncMock()
|
||||
with (
|
||||
patch("litellm.proxy.proxy_server.disable_spend_logs", True),
|
||||
patch("litellm.proxy.proxy_server.prisma_client", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()),
|
||||
patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"),
|
||||
):
|
||||
await db_writer.update_database(
|
||||
token="t1",
|
||||
user_id=None,
|
||||
end_user_id=None,
|
||||
team_id="team-1",
|
||||
org_id=None,
|
||||
kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"},
|
||||
completion_response=MagicMock(),
|
||||
start_time=datetime.now(),
|
||||
end_time=datetime.now(),
|
||||
response_cost=0.1,
|
||||
)
|
||||
await enqueue_started.wait()
|
||||
drain: Final = asyncio.create_task(drain_logging_tasks())
|
||||
release_enqueue.set()
|
||||
await drain
|
||||
spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions()
|
||||
|
||||
assert spend["key_list_transactions"] == {"t1": 0.1}
|
||||
assert spend["team_list_transactions"] == {"team-1": 0.1}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_batch_database_updates_isolation_on_failure(caplog: pytest.LogCaptureFixture):
|
||||
"""
|
||||
Test that if one helper inside _batch_database_updates raises,
|
||||
all other helpers still execute.
|
||||
all other helpers still execute and the failure is reported at ERROR with the request id.
|
||||
"""
|
||||
db_writer = DBSpendUpdateWriter()
|
||||
|
||||
|
|
@ -2217,6 +2294,7 @@ async def test_batch_database_updates_isolation_on_failure():
|
|||
db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock()
|
||||
db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock()
|
||||
|
||||
caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name)
|
||||
await db_writer._batch_database_updates(
|
||||
response_cost=0.1,
|
||||
user_id="u1",
|
||||
|
|
@ -2226,7 +2304,7 @@ async def test_batch_database_updates_isolation_on_failure():
|
|||
end_user_id="eu1",
|
||||
prisma_client=MagicMock(),
|
||||
litellm_proxy_budget_name="budget",
|
||||
payload={"key": "value"},
|
||||
payload={"request_id": "req-iso", "key": "value"},
|
||||
)
|
||||
|
||||
# _update_key_db raised, but all others should still have been called
|
||||
|
|
@ -2242,6 +2320,48 @@ async def test_batch_database_updates_isolation_on_failure():
|
|||
db_writer.add_spend_log_transaction_to_daily_team_transaction.assert_awaited_once()
|
||||
db_writer.add_spend_log_transaction_to_daily_org_transaction.assert_awaited_once()
|
||||
db_writer.add_spend_log_transaction_to_daily_tag_transaction.assert_awaited_once()
|
||||
assert any(
|
||||
"_update_key_db" in record.getMessage() and "req-iso" in record.getMessage()
|
||||
for record in caplog.records
|
||||
if record.levelno >= logging.ERROR
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_metadata_keeps_the_daily_rollup_row_and_is_reported(caplog: pytest.LogCaptureFixture):
|
||||
"""
|
||||
Regression for LIT-6027. A spend log whose metadata is not JSON used to vanish from every
|
||||
daily rollup while the spend log itself persisted, with nothing above debug in the logs.
|
||||
"""
|
||||
writer: Final = DBSpendUpdateWriter()
|
||||
prisma: Final = MagicMock()
|
||||
prisma.get_request_status = MagicMock(return_value="success")
|
||||
payload: Final = {
|
||||
"request_id": "req-bad-metadata",
|
||||
"user": "test-user",
|
||||
"startTime": "2024-01-01T00:00:00",
|
||||
"endTime": "2024-01-01T00:00:01",
|
||||
"api_key": "test-key",
|
||||
"model": "gpt-4o-mini",
|
||||
"custom_llm_provider": "openai",
|
||||
"model_group": "gpt-4o-mini",
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"spend": 0.05,
|
||||
"metadata": "{not-json",
|
||||
}
|
||||
caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name)
|
||||
|
||||
await writer.add_spend_log_transaction_to_daily_user_transaction( # pyright: ignore[reportUnknownMemberType] # prisma_client param is untyped on the writer
|
||||
payload=payload, prisma_client=prisma
|
||||
)
|
||||
|
||||
daily: Final = await writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions()
|
||||
(row,) = daily.values()
|
||||
assert (row["api_requests"], row["spend"], row["prompt_tokens"]) == (1, 0.05, 100)
|
||||
assert any(
|
||||
"req-bad-metadata" in record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -2775,12 +2895,12 @@ async def test_update_database_does_not_deepcopy_on_request_path():
|
|||
"""
|
||||
Regression for LIT-4088: copy.deepcopy must not run while the caller awaits
|
||||
update_database(). The deepcopy used to isolate the daily-spend helpers is
|
||||
relocated into the _batch_database_updates background task, and the spend-log
|
||||
relocated into the _batch_database_updates logging task, and the spend-log
|
||||
insert receives the payload directly (all consumers are read-only).
|
||||
|
||||
Asserts:
|
||||
- zero copy.deepcopy calls happen on the awaited request path
|
||||
- the batch background task still hands the daily helpers an isolated copy
|
||||
- the tracked rollup task still hands the daily helpers an isolated copy
|
||||
(mutating the original after the task ran does not bleed into it)
|
||||
- the spend-log insert receives the payload on the request path with the
|
||||
correct content
|
||||
|
|
@ -2856,16 +2976,13 @@ async def test_update_database_does_not_deepcopy_on_request_path():
|
|||
response_cost=0.1,
|
||||
)
|
||||
|
||||
# Request path is clean: nothing was deepcopied while the caller awaited.
|
||||
assert len(deepcopy_calls) == 0
|
||||
|
||||
# The spend-log insert ran inline on the request path with the real payload.
|
||||
assert captured_spend_log["ref"] is fake_payload
|
||||
assert captured_spend_log["model_at_call"] == "gpt-4"
|
||||
assert fake_payload["spend"] == 0.1
|
||||
|
||||
# Now let the batch background task run; the deepcopy happens here.
|
||||
await asyncio.sleep(0)
|
||||
await drain_logging_tasks()
|
||||
|
||||
assert len(deepcopy_calls) >= 1
|
||||
assert len(captured_batch_payloads) == 1
|
||||
|
|
|
|||
|
|
@ -52,7 +52,7 @@ app.include_router(router)
|
|||
client = TestClient(app)
|
||||
|
||||
BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets"
|
||||
SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"]
|
||||
SORTABLE = ["budget_duration", "budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"]
|
||||
|
||||
|
||||
def _row(budget_id: str, **overrides: Any) -> dict[str, Any]:
|
||||
|
|
@ -172,22 +172,12 @@ def test_appends_the_tiebreaker_to_an_explicit_sort(query_raw, as_proxy_admin):
|
|||
|
||||
_get("sort=-max_budget")
|
||||
|
||||
assert 'ORDER BY "max_budget" DESC NULLS LAST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0]
|
||||
assert 'ORDER BY "max_budget" DESC NULLS FIRST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0]
|
||||
|
||||
|
||||
def test_refuses_to_sort_on_budget_duration(query_raw, as_proxy_admin):
|
||||
"""The column holds "7d"/"30d", so a lexicographic ORDER BY would put "30d"
|
||||
before "7d" and silently mis-order the page."""
|
||||
_serve(query_raw, [])
|
||||
|
||||
def test_accepts_duration_sorting(query_raw, as_proxy_admin):
|
||||
response = _get("sort=budget_duration")
|
||||
|
||||
assert response.status_code == 400
|
||||
assert response.headers["content-type"].startswith("application/problem+json")
|
||||
body = response.json()
|
||||
assert "budget_duration" in body["detail"]
|
||||
assert body["allowed"] == SORTABLE
|
||||
query_raw.assert_not_called()
|
||||
assert response.status_code == 200
|
||||
|
||||
|
||||
def test_the_advertised_sort_fields_are_the_ones_that_work(query_raw, as_proxy_admin):
|
||||
|
|
@ -346,7 +336,7 @@ def test_q_does_not_search_any_other_column(query_raw, as_proxy_admin):
|
|||
|
||||
|
||||
def test_is_null_selects_the_unlimited_budgets(query_raw, as_proxy_admin):
|
||||
""""Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap."""
|
||||
""" "Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap."""
|
||||
_serve(query_raw, [_row("b-unlimited", max_budget=None)])
|
||||
|
||||
body = _get("filter[max_budget][is_null]=true").json()
|
||||
|
|
|
|||
|
|
@ -5,14 +5,20 @@ Tests _test_guardrail_definitions and _compute_overall_action directly
|
|||
without needing a running proxy.
|
||||
"""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi import Request
|
||||
|
||||
import litellm
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.policy_endpoints.endpoints import (
|
||||
GuardrailTestResultEntry,
|
||||
_compute_overall_action,
|
||||
_test_guardrail_definitions,
|
||||
get_policy_templates,
|
||||
list_policies,
|
||||
)
|
||||
from litellm.proxy.policy_engine.policy_registry import get_policy_registry
|
||||
|
|
@ -368,3 +374,19 @@ async def test_list_policies_reports_each_loaded_policy_with_its_resolved_guardr
|
|||
|
||||
assert response == PolicyListResponse(policies=expected_policies, total_count=0)
|
||||
assert list(response.policies) == list(expected_policies)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flag", ["LITELLM_OFFLINE_MODE", "LITELLM_LOCAL_POLICY_TEMPLATES"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_policy_templates_serve_the_bundled_copy_without_fetching(monkeypatch: pytest.MonkeyPatch, flag: str):
|
||||
monkeypatch.setenv(flag, "true")
|
||||
bundled = json.loads((Path(litellm.__file__).parent / "policy_templates_backup.json").read_text(encoding="utf-8"))
|
||||
|
||||
with patch(
|
||||
"litellm.llms.custom_httpx.http_handler.get_async_httpx_client",
|
||||
side_effect=AssertionError("policy templates were fetched"),
|
||||
) as get_client:
|
||||
response = await get_policy_templates(request=_policy_list_request(), user_api_key_dict=UserAPIKeyAuth())
|
||||
|
||||
get_client.assert_not_called()
|
||||
assert response == bundled
|
||||
|
|
|
|||
|
|
@ -9198,58 +9198,6 @@ async def test_saml_callback_blocked_when_admin_ui_disabled():
|
|||
assert "Admin UI is Disabled" in response.body.decode()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_saml_callback_enforces_free_sso_user_limit_after_validation():
|
||||
"""An IdP-initiated assertion must not bypass the >5 free-SSO-user Enterprise gate
|
||||
that /sso/key/generate enforces; the ACS re-checks it after validating the assertion,
|
||||
so the entitlement DB query never runs on unvalidated input."""
|
||||
from litellm.proxy._types import ProxyException
|
||||
from litellm.proxy.management_endpoints.types import CustomOpenID
|
||||
from litellm.proxy.management_endpoints.ui_sso import saml_callback
|
||||
|
||||
call_order: list[str] = []
|
||||
|
||||
async def _fake_handle_acs(**kwargs):
|
||||
call_order.append("validate")
|
||||
return (CustomOpenID(
|
||||
id="dana@litellm.ai",
|
||||
email="dana@litellm.ai",
|
||||
first_name=None,
|
||||
last_name=None,
|
||||
display_name="dana",
|
||||
picture=None,
|
||||
provider="saml",
|
||||
team_ids=[],
|
||||
user_role=None,
|
||||
), None)
|
||||
|
||||
async def _fake_count_billable_users():
|
||||
call_order.append("count")
|
||||
return 6
|
||||
|
||||
async def _stream():
|
||||
yield b"SAMLResponse=signed-response"
|
||||
|
||||
request_double = SimpleNamespace(cookies={}, headers={}, stream=_stream)
|
||||
|
||||
with patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}), patch(
|
||||
"litellm.proxy.proxy_server.premium_user", False
|
||||
), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch(
|
||||
"litellm.proxy.proxy_server.master_key", MASTER_KEY
|
||||
), patch(
|
||||
"litellm.proxy.management_endpoints.sso.saml_sso.SAMLAuthHandler.handle_acs",
|
||||
new=_fake_handle_acs,
|
||||
), patch(
|
||||
"litellm.repositories.user_repository.UserRepository.count_billable_users",
|
||||
new=AsyncMock(side_effect=_fake_count_billable_users),
|
||||
):
|
||||
with pytest.raises(ProxyException) as exc:
|
||||
await saml_callback(request_double)
|
||||
|
||||
assert str(exc.value.code) == "403"
|
||||
assert call_order == ["validate", "count"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cli_poll_key_tolerates_missing_user_row():
|
||||
"""The CLI poll must still mint the JWT when the user lookup raises,
|
||||
|
|
|
|||
|
|
@ -1,657 +0,0 @@
|
|||
"""
|
||||
Tests for AI Usage Chat module.
|
||||
"""
|
||||
|
||||
import json
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
TOOL_HANDLERS,
|
||||
TOOLS_ADMIN,
|
||||
TOOLS_BASE,
|
||||
_build_system_prompt,
|
||||
_summarise_entity_data,
|
||||
_summarise_usage_data,
|
||||
stream_usage_ai_chat,
|
||||
)
|
||||
|
||||
|
||||
SAMPLE_AGGREGATED_RESPONSE = {
|
||||
"results": [
|
||||
{
|
||||
"date": "2025-01-15",
|
||||
"metrics": {
|
||||
"spend": 50.25,
|
||||
"prompt_tokens": 20000,
|
||||
"completion_tokens": 10000,
|
||||
"total_tokens": 30000,
|
||||
"api_requests": 500,
|
||||
"successful_requests": 480,
|
||||
"failed_requests": 20,
|
||||
"cache_read_input_tokens": 0,
|
||||
"cache_creation_input_tokens": 0,
|
||||
},
|
||||
"breakdown": {
|
||||
"models": {
|
||||
"gpt-4": {
|
||||
"metrics": {
|
||||
"spend": 40.0,
|
||||
"api_requests": 300,
|
||||
"total_tokens": 25000,
|
||||
},
|
||||
"metadata": {},
|
||||
"api_key_breakdown": {},
|
||||
},
|
||||
},
|
||||
"providers": {
|
||||
"openai": {
|
||||
"metrics": {"spend": 50.25, "api_requests": 500},
|
||||
"metadata": {},
|
||||
"api_key_breakdown": {},
|
||||
},
|
||||
},
|
||||
"api_keys": {
|
||||
"sk-test123": {
|
||||
"metrics": {"spend": 50.25},
|
||||
"metadata": {"key_alias": "Production Key"},
|
||||
},
|
||||
},
|
||||
"model_groups": {},
|
||||
"mcp_servers": {},
|
||||
"entities": {},
|
||||
},
|
||||
},
|
||||
],
|
||||
"metadata": {
|
||||
"total_spend": 50.25,
|
||||
"total_api_requests": 500,
|
||||
"total_successful_requests": 480,
|
||||
"total_failed_requests": 20,
|
||||
"total_tokens": 30000,
|
||||
},
|
||||
}
|
||||
|
||||
SAMPLE_TEAM_RESPONSE = {
|
||||
"results": [
|
||||
{
|
||||
"date": "2025-01-15",
|
||||
"metrics": {"spend": 100.0, "api_requests": 1000, "total_tokens": 50000},
|
||||
"breakdown": {
|
||||
"entities": {
|
||||
"team-1": {
|
||||
"metrics": {
|
||||
"spend": 60.0,
|
||||
"api_requests": 600,
|
||||
"total_tokens": 30000,
|
||||
},
|
||||
"metadata": {"alias": "Engineering"},
|
||||
"api_key_breakdown": {},
|
||||
},
|
||||
"team-2": {
|
||||
"metrics": {
|
||||
"spend": 40.0,
|
||||
"api_requests": 400,
|
||||
"total_tokens": 20000,
|
||||
},
|
||||
"metadata": {"alias": "Marketing"},
|
||||
"api_key_breakdown": {},
|
||||
},
|
||||
},
|
||||
"models": {},
|
||||
"providers": {},
|
||||
"api_keys": {},
|
||||
"model_groups": {},
|
||||
"mcp_servers": {},
|
||||
},
|
||||
},
|
||||
],
|
||||
"metadata": {"total_spend": 100.0, "total_api_requests": 1000},
|
||||
}
|
||||
|
||||
|
||||
class TestToolSchemas:
|
||||
def test_admin_tools_include_all(self):
|
||||
assert len(TOOLS_ADMIN) == 3
|
||||
names = {t["function"]["name"] for t in TOOLS_ADMIN}
|
||||
assert "get_usage_data" in names
|
||||
assert "get_team_usage_data" in names
|
||||
assert "get_tag_usage_data" in names
|
||||
|
||||
def test_base_tools_restricted_to_usage_only(self):
|
||||
assert len(TOOLS_BASE) == 1
|
||||
assert TOOLS_BASE[0]["function"]["name"] == "get_usage_data"
|
||||
|
||||
def test_admin_prompt_mentions_all_tools(self):
|
||||
prompt = _build_system_prompt(is_admin=True)
|
||||
assert "get_usage_data" in prompt
|
||||
assert "get_team_usage_data" in prompt
|
||||
assert "get_tag_usage_data" in prompt
|
||||
|
||||
def test_non_admin_prompt_only_mentions_usage_tool(self):
|
||||
prompt = _build_system_prompt(is_admin=False)
|
||||
assert "get_usage_data" in prompt
|
||||
assert "get_team_usage_data" not in prompt
|
||||
assert "get_tag_usage_data" not in prompt
|
||||
|
||||
def test_system_prompt_includes_todays_date(self):
|
||||
from datetime import date
|
||||
|
||||
prompt = _build_system_prompt(is_admin=True)
|
||||
assert date.today().isoformat() in prompt
|
||||
|
||||
|
||||
class TestSummariseUsageData:
|
||||
def test_summarise_includes_totals(self):
|
||||
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
|
||||
assert "$50.25" in summary
|
||||
assert "500" in summary
|
||||
|
||||
def test_summarise_includes_models(self):
|
||||
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
|
||||
assert "gpt-4" in summary
|
||||
|
||||
def test_summarise_includes_providers(self):
|
||||
summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE)
|
||||
assert "openai" in summary
|
||||
|
||||
def test_summarise_handles_empty_data(self):
|
||||
empty = {"results": [], "metadata": {}}
|
||||
summary = _summarise_usage_data(empty)
|
||||
assert "no data" in summary.lower()
|
||||
|
||||
|
||||
class TestSummariseEntityData:
|
||||
def test_team_summary_includes_teams(self):
|
||||
summary = _summarise_entity_data(SAMPLE_TEAM_RESPONSE, "Team")
|
||||
assert "Engineering" in summary
|
||||
assert "Marketing" in summary
|
||||
assert "$60.0" in summary
|
||||
assert "$40.0" in summary
|
||||
|
||||
def test_team_summary_empty(self):
|
||||
empty = {"results": [], "metadata": {}}
|
||||
summary = _summarise_entity_data(empty, "Team")
|
||||
assert "No Team usage data" in summary
|
||||
|
||||
|
||||
class TestStreamUsageAiChat:
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_emits_status_events(self):
|
||||
mock_tool_call = MagicMock()
|
||||
mock_tool_call.id = "call_123"
|
||||
mock_tool_call.function.name = "get_usage_data"
|
||||
mock_tool_call.function.arguments = json.dumps(
|
||||
{
|
||||
"start_date": "2025-01-01",
|
||||
"end_date": "2025-01-31",
|
||||
}
|
||||
)
|
||||
|
||||
mock_first_response = MagicMock()
|
||||
mock_first_response.choices = [MagicMock()]
|
||||
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
|
||||
mock_first_response.choices[0].message.model_dump.return_value = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_123",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_usage_data",
|
||||
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
async def mock_stream():
|
||||
chunk = MagicMock()
|
||||
chunk.choices = [MagicMock()]
|
||||
chunk.choices[0].delta.content = "Total spend is $50.25"
|
||||
yield chunk
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_usage_data",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_fetch,
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[
|
||||
mock_first_response,
|
||||
mock_stream(),
|
||||
]
|
||||
)
|
||||
mock_fetch.return_value = SAMPLE_AGGREGATED_RESPONSE
|
||||
|
||||
events = []
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "What is my total spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="user-123",
|
||||
is_admin=True,
|
||||
):
|
||||
events.append(json.loads(event.replace("data: ", "").strip()))
|
||||
|
||||
status_events = [e for e in events if e["type"] == "status"]
|
||||
tool_call_events = [e for e in events if e["type"] == "tool_call"]
|
||||
chunk_events = [e for e in events if e["type"] == "chunk"]
|
||||
done_events = [e for e in events if e["type"] == "done"]
|
||||
|
||||
assert len(status_events) >= 1
|
||||
assert "Thinking" in status_events[0]["message"]
|
||||
assert len(tool_call_events) >= 1
|
||||
assert tool_call_events[0]["tool_name"] == "get_usage_data"
|
||||
assert tool_call_events[0]["status"] in ("running", "complete")
|
||||
assert len(chunk_events) >= 1
|
||||
assert len(done_events) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_handles_team_tool(self):
|
||||
mock_tool_call = MagicMock()
|
||||
mock_tool_call.id = "call_team"
|
||||
mock_tool_call.function.name = "get_team_usage_data"
|
||||
mock_tool_call.function.arguments = json.dumps(
|
||||
{
|
||||
"start_date": "2025-01-01",
|
||||
"end_date": "2025-01-31",
|
||||
}
|
||||
)
|
||||
|
||||
mock_first_response = MagicMock()
|
||||
mock_first_response.choices = [MagicMock()]
|
||||
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
|
||||
mock_first_response.choices[0].message.model_dump.return_value = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_team",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_team_usage_data",
|
||||
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
async def mock_stream():
|
||||
chunk = MagicMock()
|
||||
chunk.choices = [MagicMock()]
|
||||
chunk.choices[0].delta.content = "Engineering is the top team."
|
||||
yield chunk
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_team_usage_data",
|
||||
new_callable=AsyncMock,
|
||||
) as mock_fetch,
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[
|
||||
mock_first_response,
|
||||
mock_stream(),
|
||||
]
|
||||
)
|
||||
mock_fetch.return_value = SAMPLE_TEAM_RESPONSE
|
||||
|
||||
events = []
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "Which team spends the most?"}],
|
||||
model="gpt-4o-mini",
|
||||
is_admin=True,
|
||||
):
|
||||
events.append(json.loads(event.replace("data: ", "").strip()))
|
||||
|
||||
chunk_events = [e for e in events if e["type"] == "chunk"]
|
||||
assert len(chunk_events) >= 1
|
||||
assert "Engineering" in chunk_events[0]["content"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"reason",
|
||||
[
|
||||
"end_date must be on or after start_date",
|
||||
"start_date and end_date must be valid YYYY-MM-DD dates",
|
||||
],
|
||||
ids=["reversed", "invalid-format"],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"tool_name", ["get_usage_data", "get_team_usage_data", "get_tag_usage_data"]
|
||||
)
|
||||
async def test_invalid_tool_dates_are_reported_to_the_model(
|
||||
self, tool_name: str, reason: str
|
||||
) -> None:
|
||||
fetch: Final = AsyncMock(
|
||||
side_effect=HTTPException(status_code=400, detail={"error": reason})
|
||||
)
|
||||
|
||||
tool_message, statuses = await _run_one_tool(
|
||||
tool_name, "2025-01-31", "2025-01-01", fetch
|
||||
)
|
||||
|
||||
fetch.assert_awaited_once()
|
||||
assert tool_message == f"Invalid date range: {reason}"
|
||||
assert statuses == ["running", "complete"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_dates_the_query_layer_accepts_are_queried_as_sent(
|
||||
self,
|
||||
) -> None:
|
||||
fetch: Final = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE)
|
||||
|
||||
tool_message, statuses = await _run_one_tool(
|
||||
"get_usage_data", "2025/01/01", "2025-01-31T00:00:00", fetch
|
||||
)
|
||||
|
||||
fetch.assert_awaited_once_with(
|
||||
start_date="2025/01/01", end_date="2025-01-31T00:00:00"
|
||||
)
|
||||
assert tool_message.startswith("Total Spend: $")
|
||||
assert statuses == ["running", "complete"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"error",
|
||||
[
|
||||
HTTPException(status_code=500, detail={"error": "db down"}),
|
||||
HTTPException(status_code=400, detail="page and page_size must be >= 1"),
|
||||
],
|
||||
ids=["server-error", "other-400"],
|
||||
)
|
||||
async def test_other_tool_errors_stay_a_generic_fetch_error(
|
||||
self, error: HTTPException
|
||||
) -> None:
|
||||
tool_message, statuses = await _run_one_tool(
|
||||
"get_team_usage_data",
|
||||
"2025-01-01",
|
||||
"2025-01-31",
|
||||
AsyncMock(side_effect=error),
|
||||
)
|
||||
|
||||
assert tool_message == "Error fetching team usage data. Please try again."
|
||||
assert statuses == ["running", "error"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stream_handles_error(self):
|
||||
with patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm:
|
||||
mock_litellm.acompletion = AsyncMock(side_effect=Exception("LLM error"))
|
||||
|
||||
events = []
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "test"}],
|
||||
):
|
||||
events.append(json.loads(event.replace("data: ", "").strip()))
|
||||
|
||||
error_events = [e for e in events if e["type"] == "error"]
|
||||
assert len(error_events) == 1
|
||||
assert "internal error" in error_events[0]["message"].lower()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_admin_enforces_user_id(self):
|
||||
mock_tool_call = MagicMock()
|
||||
mock_tool_call.id = "call_456"
|
||||
mock_tool_call.function.name = "get_usage_data"
|
||||
mock_tool_call.function.arguments = json.dumps(
|
||||
{
|
||||
"start_date": "2025-01-01",
|
||||
"end_date": "2025-01-31",
|
||||
"user_id": "other-user",
|
||||
}
|
||||
)
|
||||
|
||||
mock_first_response = MagicMock()
|
||||
mock_first_response.choices = [MagicMock()]
|
||||
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
|
||||
mock_first_response.choices[0].message.model_dump.return_value = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_456",
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "get_usage_data",
|
||||
"arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31","user_id":"other-user"}',
|
||||
},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
async def mock_stream():
|
||||
chunk = MagicMock()
|
||||
chunk.choices = [MagicMock()]
|
||||
chunk.choices[0].delta.content = "Data."
|
||||
yield chunk
|
||||
|
||||
mock_fetch = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE)
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch.dict(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS",
|
||||
{
|
||||
"get_usage_data": {
|
||||
"fetch": mock_fetch,
|
||||
"summarise": _summarise_usage_data,
|
||||
"label": "global usage data",
|
||||
}
|
||||
},
|
||||
),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[
|
||||
mock_first_response,
|
||||
mock_stream(),
|
||||
]
|
||||
)
|
||||
|
||||
events = []
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "Show data"}],
|
||||
model="gpt-4o-mini",
|
||||
user_id="my-user-id",
|
||||
is_admin=False,
|
||||
):
|
||||
events.append(event)
|
||||
|
||||
mock_fetch.assert_called_once_with(
|
||||
start_date="2025-01-01",
|
||||
end_date="2025-01-31",
|
||||
user_id="my-user-id",
|
||||
)
|
||||
|
||||
|
||||
class TestUsageAiChatServiceAccountGuard:
|
||||
"""
|
||||
Security regression: a non-admin caller with user_id=None (service-account
|
||||
key) must be rejected at the endpoint boundary, before any tool dispatch.
|
||||
"""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_non_admin_with_user_id_none_is_rejected(self):
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
|
||||
ChatMessage,
|
||||
UsageAIChatRequest,
|
||||
usage_ai_chat,
|
||||
)
|
||||
|
||||
service_account_key = UserAPIKeyAuth(
|
||||
user_id=None,
|
||||
user_role=LitellmUserRoles.INTERNAL_USER,
|
||||
)
|
||||
request = MagicMock()
|
||||
body = UsageAIChatRequest(
|
||||
messages=[ChatMessage(role="user", content="hi")],
|
||||
model="gpt-4o-mini",
|
||||
)
|
||||
|
||||
with pytest.raises(HTTPException) as exc_info:
|
||||
await usage_ai_chat(
|
||||
data=body,
|
||||
request=request,
|
||||
user_api_key_dict=service_account_key,
|
||||
)
|
||||
|
||||
assert exc_info.value.status_code == 403
|
||||
assert "Service-account keys" in str(exc_info.value.detail)
|
||||
|
||||
def test_resolve_fetch_kwargs_tripwire_fires_on_none_user_id(self):
|
||||
"""
|
||||
Defense-in-depth: if a future endpoint forgets the entry guard and
|
||||
a non-admin caller with user_id=None reaches _resolve_fetch_kwargs,
|
||||
the tripwire must fire rather than issuing an unscoped query.
|
||||
"""
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
|
||||
_resolve_fetch_kwargs,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info:
|
||||
_resolve_fetch_kwargs(
|
||||
fn_name="get_usage_data",
|
||||
fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"},
|
||||
user_id=None,
|
||||
is_admin=False,
|
||||
)
|
||||
assert "Endpoint-level guard missing" in str(exc_info.value)
|
||||
|
||||
|
||||
class TestUsageAiChatKeepalive:
|
||||
async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]:
|
||||
import asyncio
|
||||
|
||||
import litellm
|
||||
from fastapi.responses import StreamingResponse
|
||||
from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import (
|
||||
ChatMessage,
|
||||
UsageAIChatRequest,
|
||||
usage_ai_chat,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval)
|
||||
|
||||
async def slow_acompletion(**kwargs):
|
||||
await asyncio.sleep(delay)
|
||||
response = MagicMock()
|
||||
response.choices = [MagicMock()]
|
||||
response.choices[0].message.tool_calls = None
|
||||
response.choices[0].message.content = "Total spend is $50.25"
|
||||
return response
|
||||
|
||||
with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion",
|
||||
new=AsyncMock(side_effect=slow_acompletion),
|
||||
):
|
||||
response = await usage_ai_chat(
|
||||
data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"),
|
||||
request=MagicMock(),
|
||||
user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN),
|
||||
)
|
||||
assert isinstance(response, StreamingResponse)
|
||||
chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator]
|
||||
return chunks, dict(response.headers)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch):
|
||||
chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05)
|
||||
|
||||
assert headers["content-type"].startswith("text/event-stream")
|
||||
assert headers["cache-control"] == "no-cache"
|
||||
assert headers["x-accel-buffering"] == "no"
|
||||
assert chunks[0].startswith(b'data: {"type": "status"')
|
||||
assert chunks[1] == b": ping\n\n"
|
||||
assert chunks.count(b": ping\n\n") >= 3
|
||||
assert b'"content": "Total spend is $50.25"' in b"".join(chunks)
|
||||
assert chunks[-1] == b'data: {"type": "done"}\n\n'
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch):
|
||||
chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15)
|
||||
|
||||
assert b": ping\n\n" not in chunks
|
||||
assert chunks[0].startswith(b'data: {"type": "status"')
|
||||
assert chunks[-1] == b'data: {"type": "done"}\n\n'
|
||||
|
||||
|
||||
async def _run_one_tool(
|
||||
tool_name: str, start_date: str, end_date: str, fetch: AsyncMock
|
||||
) -> tuple[str, list[object]]:
|
||||
"""Runs one admin chat turn whose model calls `tool_name` with these dates; returns the
|
||||
tool message the follow-up model call carried and the tool statuses streamed."""
|
||||
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
|
||||
mock_tool_call: Final = MagicMock()
|
||||
mock_tool_call.id = "call_dates"
|
||||
mock_tool_call.function.name = tool_name
|
||||
mock_tool_call.function.arguments = arguments
|
||||
|
||||
mock_first_response: Final = MagicMock()
|
||||
mock_first_response.choices = [MagicMock()]
|
||||
mock_first_response.choices[0].message.tool_calls = [mock_tool_call]
|
||||
mock_first_response.choices[0].message.model_dump.return_value = {
|
||||
"role": "assistant",
|
||||
"content": None,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "call_dates",
|
||||
"type": "function",
|
||||
"function": {"name": tool_name, "arguments": arguments},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
async def mock_stream() -> AsyncIterator[MagicMock]:
|
||||
chunk: Final = MagicMock()
|
||||
chunk.choices = [MagicMock()]
|
||||
chunk.choices[0].delta.content = "Done."
|
||||
yield chunk
|
||||
|
||||
with (
|
||||
patch(
|
||||
"litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm"
|
||||
) as mock_litellm,
|
||||
patch.dict(TOOL_HANDLERS[tool_name], {"fetch": fetch}),
|
||||
):
|
||||
mock_litellm.acompletion = AsyncMock(
|
||||
side_effect=[mock_first_response, mock_stream()]
|
||||
)
|
||||
events: Final[tuple[dict[str, object], ...]] = tuple(
|
||||
[
|
||||
json.loads(event.replace("data: ", "").strip())
|
||||
async for event in stream_usage_ai_chat(
|
||||
messages=[{"role": "user", "content": "How much did we spend?"}],
|
||||
model="gpt-4o-mini",
|
||||
is_admin=True,
|
||||
)
|
||||
]
|
||||
)
|
||||
final_messages: Final = mock_litellm.acompletion.call_args_list[1].kwargs[
|
||||
"messages"
|
||||
]
|
||||
|
||||
tool_messages: Final = [m for m in final_messages if m["role"] == "tool"]
|
||||
assert len(tool_messages) == 1, tool_messages
|
||||
assert tool_messages[0]["tool_call_id"] == "call_dates"
|
||||
return str(tool_messages[0]["content"]), [
|
||||
e["status"] for e in events if e["type"] == "tool_call"
|
||||
]
|
||||
|
|
@ -0,0 +1,85 @@
|
|||
import json
|
||||
import re
|
||||
import sqlite3
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from pydantic import JsonValue
|
||||
|
||||
from litellm.proxy.management_helpers.team_list_sorting import UnsupportedTeamFilter, _where_sql, sorted_team_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("where", "expected"),
|
||||
(
|
||||
({}, ("a", "b", "c")),
|
||||
({"OR": []}, ()),
|
||||
({"AND": []}, ("a", "b", "c")),
|
||||
({"team_id": "a"}, ("a",)),
|
||||
({"team_alias": {"contains": "ALPHA", "mode": "insensitive"}}, ("a", "c")),
|
||||
({"team_id": {"startsWith": "b"}}, ("b",)),
|
||||
({"organization_id": {"in": ["org-1"]}}, ("a", "b")),
|
||||
(
|
||||
{
|
||||
"OR": [{"team_id": "b"}, {"team_alias": {"contains": "alpha", "mode": "insensitive"}}],
|
||||
"AND": [{"OR": [{"organization_id": "org-1"}, {"team_id": "c"}]}],
|
||||
"team_id": {"in": ["a", "c"]},
|
||||
},
|
||||
("a", "c"),
|
||||
),
|
||||
({"team_id": "a' OR 1=1 --"}, ()),
|
||||
),
|
||||
)
|
||||
def test_team_filters_preserve_search_and_authorization_intersection(
|
||||
where: dict[str, JsonValue], expected: tuple[str, ...]
|
||||
) -> None:
|
||||
result: Final = _where_sql(where)
|
||||
assert not isinstance(result, UnsupportedTeamFilter)
|
||||
predicate, params = result
|
||||
sqlite_predicate: Final = re.sub(
|
||||
r"= ANY\((\$\d+)::text\[\]\)", r"IN (SELECT value FROM json_each(\1))", predicate
|
||||
).replace(" ILIKE ", " LIKE ")
|
||||
with sqlite3.connect(":memory:") as db:
|
||||
db.execute("CREATE TABLE teams (team_id TEXT, team_alias TEXT, organization_id TEXT)")
|
||||
db.executemany(
|
||||
"INSERT INTO teams VALUES (?, ?, ?)",
|
||||
(("a", "Alpha", "org-1"), ("b", "Beta", "org-1"), ("c", "Alpha", "org-2")),
|
||||
)
|
||||
rows: Final = db.execute(
|
||||
f"SELECT t.team_id FROM teams t WHERE {sqlite_predicate} ORDER BY t.team_id",
|
||||
{
|
||||
str(index): json.dumps(value) if isinstance(value, list) else value
|
||||
for index, value in enumerate(params, 1)
|
||||
},
|
||||
).fetchall()
|
||||
assert tuple(row[0] for row in rows) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("where", ({"unknown": "x"}, {"team_id": {"not": "x"}}, {"AND": [False]}))
|
||||
@pytest.mark.asyncio
|
||||
async def test_unsupported_filter_fails_without_reading_unscoped_teams(where: dict[str, JsonValue]) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.query_raw = AsyncMock()
|
||||
result: Final = await sorted_team_ids(prisma, where=where, sort_by="keys_count", descending=False, skip=0, take=25)
|
||||
assert isinstance(result, UnsupportedTeamFilter)
|
||||
prisma.db.query_raw.assert_not_awaited()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sort_by", ("keys_count", "members", "models", "organization_alias"))
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
@pytest.mark.asyncio
|
||||
async def test_sorted_ids_bind_search_and_pagination_without_interpolating_input(
|
||||
sort_by: str, descending: bool
|
||||
) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
search: Final = "x' OR TRUE --"
|
||||
result: Final = await sorted_team_ids(
|
||||
prisma, where={"team_id": search}, sort_by=sort_by, descending=descending, skip=25, take=25
|
||||
)
|
||||
assert result == ()
|
||||
sql, *bindings = prisma.db.query_raw.await_args.args
|
||||
assert search not in sql
|
||||
assert bindings == [search, 25, 25]
|
||||
assert ("DESC" if descending else "ASC") + " NULLS LAST, t.team_id ASC" in sql
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
from typing import Final, Literal
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
|
||||
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
|
||||
|
||||
|
||||
@pytest.mark.parametrize("sort_by", ("status", "key_count"))
|
||||
@pytest.mark.parametrize("descending", (False, True))
|
||||
@pytest.mark.asyncio
|
||||
async def test_user_sort_binds_scope_search_and_page_and_excludes_ui_sessions(
|
||||
sort_by: Literal["status", "key_count"], descending: bool
|
||||
) -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
search: Final = "x' OR TRUE --"
|
||||
result: Final = await sorted_user_ids(
|
||||
prisma,
|
||||
sort_by=sort_by,
|
||||
descending=descending,
|
||||
skip=25,
|
||||
take=25,
|
||||
role="internal_user",
|
||||
user_ids=" a, b ",
|
||||
user_email=search,
|
||||
team="team-1",
|
||||
sso_user_ids="sso-1, sso-2",
|
||||
organization_ids="org-1, org-2",
|
||||
search=search,
|
||||
)
|
||||
assert result == ()
|
||||
sql, *bindings = prisma.db.query_raw.await_args.args
|
||||
assert search not in sql
|
||||
assert bindings == [
|
||||
"internal_user",
|
||||
("a", "b"),
|
||||
search,
|
||||
"team-1",
|
||||
False,
|
||||
("sso-1", "sso-2"),
|
||||
("org-1", "org-2"),
|
||||
search,
|
||||
25,
|
||||
25,
|
||||
] + ([UI_SESSION_TOKEN_TEAM_ID] if sort_by == "key_count" else [])
|
||||
assert ("DESC" if descending else "ASC") + ", u.user_id ASC" in sql
|
||||
assert ("k.team_id IS DISTINCT FROM $11::text" in sql) == (sort_by == "key_count")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_user_filters_preserve_unfiltered_query() -> None:
|
||||
prisma: Final = MagicMock()
|
||||
prisma.db.query_raw = AsyncMock(return_value=[])
|
||||
result: Final = await sorted_user_ids(
|
||||
prisma,
|
||||
sort_by="status",
|
||||
descending=False,
|
||||
skip=0,
|
||||
take=25,
|
||||
role=None,
|
||||
user_ids=None,
|
||||
user_email=None,
|
||||
team=None,
|
||||
sso_user_ids=None,
|
||||
organization_ids=None,
|
||||
search=None,
|
||||
)
|
||||
assert result == ()
|
||||
assert prisma.db.query_raw.await_args.args[1:] == (None, None, None, None, True, (), (), None, 25, 0)
|
||||
|
|
@ -1430,6 +1430,30 @@ async def test_get_autorouter_presets_fetches_once_per_process(monkeypatch, rese
|
|||
assert calls == ["https://example.test/presets.json"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_AUTOROUTER_PRESETS", "LITELLM_OFFLINE_MODE"])
|
||||
async def test_get_autorouter_presets_local_flags_serve_bundled_without_fetching(
|
||||
monkeypatch, reset_autorouter_presets_cache, flag
|
||||
):
|
||||
from litellm.proxy.public_endpoints.public_endpoints import (
|
||||
_load_bundled_autorouter_presets,
|
||||
get_autorouter_presets,
|
||||
)
|
||||
|
||||
monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False)
|
||||
monkeypatch.setenv(flag, "True")
|
||||
calls = []
|
||||
|
||||
async def fake_fetch(url):
|
||||
calls.append(url)
|
||||
raise AssertionError("no fetch when the bundled catalog is forced")
|
||||
|
||||
presets = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch)
|
||||
|
||||
assert presets == _load_bundled_autorouter_presets()
|
||||
assert calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start(
|
||||
monkeypatch, reset_autorouter_presets_cache
|
||||
|
|
|
|||
|
|
@ -1,13 +1,12 @@
|
|||
from dataclasses import dataclass, field
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
|
||||
from typing_extensions import LiteralString
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
from httpx import ASGITransport, AsyncClient, Response
|
||||
from typing_extensions import LiteralString
|
||||
|
||||
from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth
|
||||
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings
|
||||
|
|
|
|||
|
|
@ -93,6 +93,20 @@ def test_retired_roi_calculator_cannot_be_loaded_or_called(
|
|||
assert response.status_code == 404, f"{method} {path}: {response.text}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("disable_lazy_routes", ("false", "true"))
|
||||
@pytest.mark.parametrize("path", ("/lazy/warm/usage_ai", "/usage/ai/chat"))
|
||||
def test_retired_usage_chat_cannot_be_loaded_or_called(
|
||||
monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str, path: str
|
||||
) -> None:
|
||||
monkeypatch.setenv(FLAG, disable_lazy_routes)
|
||||
app: Final = FastAPI()
|
||||
attach_lazy_features(app)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response: Final = client.post(path)
|
||||
assert response.status_code == 404, f"{path}: {response.text}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ("1", "true", "TRUE", "yes", "on"))
|
||||
def test_flag_registers_every_feature_at_startup(monkeypatch: pytest.MonkeyPatch, value: str) -> None:
|
||||
monkeypatch.setenv(FLAG, value)
|
||||
|
|
|
|||
|
|
@ -106,6 +106,13 @@ class TestGetMetadataVariableName:
|
|||
request = self._make_request("/v1/files")
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"]
|
||||
)
|
||||
def test_returns_litellm_metadata_for_interactions_routes(self, path):
|
||||
request = self._make_request(path)
|
||||
assert get_metadata_variable_name(request) == "litellm_metadata"
|
||||
|
||||
def test_returns_metadata_for_chat_completions(self):
|
||||
request = self._make_request("/chat/completions")
|
||||
assert get_metadata_variable_name(request) == "metadata"
|
||||
|
|
@ -708,6 +715,46 @@ async def test_add_litellm_data_to_request_parses_json_object_string_litellm_met
|
|||
assert updated["litellm_metadata"]["cost_centre"] == "research"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_interactions_spend_logs_metadata_survives_the_router_metadata_bucket():
|
||||
"""GH#47841: on /v1beta/interactions the router plants its deployment fields in
|
||||
`litellm_metadata`. The proxy must put the request's metadata there too: otherwise the
|
||||
spend log reads only the router's bucket and `spend_logs_metadata` from the
|
||||
`x-litellm-spend-logs-metadata` header never reaches LiteLLM_SpendLogs."""
|
||||
request_mock = MagicMock(spec=Request)
|
||||
request_mock.url = MagicMock()
|
||||
request_mock.url.__str__.return_value = "http://localhost/v1beta/interactions"
|
||||
request_mock.url.path = "/v1beta/interactions"
|
||||
request_mock.method = "POST"
|
||||
request_mock.query_params = {}
|
||||
request_mock.headers = {
|
||||
"Content-Type": "application/json",
|
||||
"x-litellm-spend-logs-metadata": '{"cost_centre": "research"}',
|
||||
}
|
||||
request_mock.client = MagicMock()
|
||||
request_mock.client.host = "127.0.0.1"
|
||||
request_mock.state.parent_otel_span = None
|
||||
|
||||
data = await add_litellm_data_to_request(
|
||||
data={"model": "gemini-3.1-flash-lite", "input": "Reply OK.", "store": True},
|
||||
request=request_mock,
|
||||
user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"),
|
||||
proxy_config=MagicMock(),
|
||||
general_settings={},
|
||||
version="test-version",
|
||||
)
|
||||
assert data["litellm_metadata"]["spend_logs_metadata"] == {"cost_centre": "research"}
|
||||
|
||||
# The router then adds its deployment fields to the same bucket, and the spend log
|
||||
# resolves that bucket.
|
||||
data["litellm_metadata"].update({"model_group": "gemini-3.1-flash-lite", "deployment": "gemini/gemini-3.1-flash-lite"})
|
||||
resolved = get_litellm_metadata_from_kwargs(
|
||||
{"litellm_params": {"metadata": data.get("metadata") or {}, "litellm_metadata": data["litellm_metadata"]}}
|
||||
)
|
||||
assert resolved["spend_logs_metadata"] == {"cost_centre": "research"}
|
||||
assert resolved["model_group"] == "gemini-3.1-flash-lite"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_strip():
|
||||
"""Regression: proxy_server_request['body'] used to be snapshotted before
|
||||
|
|
@ -8064,6 +8111,67 @@ def test_client_anthropic_api_headers_stay_off_openai_compatible_providers():
|
|||
assert forwarded == {}
|
||||
|
||||
|
||||
def test_client_anthropic_api_headers_stay_off_azure_ai_outside_native_messages():
|
||||
forwarded = _headers_forwarded_to({"anthropic-beta": "thinking-binding-controls-2026-08-01"}, "azure_ai")
|
||||
|
||||
assert forwarded == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_beta_headers_config")
|
||||
async def test_client_thinking_binding_beta_reaches_azure_ai_foundry_claude():
|
||||
import httpx
|
||||
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler
|
||||
|
||||
data: Final[dict] = {"model": "claude-opus-4-8"} # mutable-ok: the function under test writes into it
|
||||
add_provider_specific_headers_to_request(
|
||||
data=data,
|
||||
headers={
|
||||
"content-type": "application/json",
|
||||
"anthropic-beta": "thinking-binding-controls-2026-08-01",
|
||||
"anthropic-version": "2026-08-01",
|
||||
},
|
||||
)
|
||||
received: Final[list[httpx.Request]] = [] # mutable-ok: the fake Foundry endpoint records what the proxy sent
|
||||
|
||||
def foundry_answers(request: httpx.Request) -> httpx.Response:
|
||||
received.append(request)
|
||||
return httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "msg_01",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-opus-4-8",
|
||||
"content": [{"type": "text", "text": "hello"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 12, "output_tokens": 6},
|
||||
},
|
||||
request=request,
|
||||
)
|
||||
|
||||
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers))
|
||||
try:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
max_tokens=16,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
thinking={"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}},
|
||||
model="azure_ai/claude-opus-4-8",
|
||||
api_key="foundry-key",
|
||||
api_base="https://my-resource.services.ai.azure.com/anthropic",
|
||||
client=upstream,
|
||||
provider_specific_header=data["provider_specific_header"],
|
||||
)
|
||||
finally:
|
||||
await upstream.close()
|
||||
|
||||
assert received[0].headers["anthropic-beta"] == "thinking-binding-controls-2026-08-01"
|
||||
assert received[0].headers["anthropic-version"] == "2026-08-01"
|
||||
assert json.loads(received[0].content)["thinking"]["block_binding"] == {"prefix_mismatch_behavior": "drop_block"}
|
||||
|
||||
|
||||
TYPESAFE_DANGER_ZONE_HEADERS = {
|
||||
"TypeSafe-Danger-Zone-Images": "true",
|
||||
"typesafe-danger-zone-long-context": "true",
|
||||
|
|
|
|||
|
|
@ -401,6 +401,26 @@ class TestLatestReleaseInfoEndpoint:
|
|||
assert response.status_code == 200
|
||||
assert response.json() is None
|
||||
|
||||
@pytest.mark.parametrize("name", ["LITELLM_DISABLE_LATEST_RELEASE_CHECK", "LITELLM_OFFLINE_MODE"])
|
||||
@pytest.mark.parametrize(
|
||||
("flag", "expected_body", "expected_urls"),
|
||||
[
|
||||
("true", None, []),
|
||||
(" TRUE ", None, []),
|
||||
("false", EXPECTED_INFO, [CHANGELOG_FEED_URL, _tag_url("v1.102.0")]),
|
||||
],
|
||||
)
|
||||
def test_disable_flag_skips_every_outbound_request(
|
||||
self, http_client, monkeypatch, name, flag, expected_body, expected_urls
|
||||
):
|
||||
monkeypatch.setenv(name, flag)
|
||||
client = _minor_release()
|
||||
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)
|
||||
response = http_client.get("/get/latest_release_info")
|
||||
assert response.status_code == 200
|
||||
assert response.json() == expected_body
|
||||
assert client.urls == expected_urls
|
||||
|
||||
def test_repeated_requests_reuse_cache(self, http_client):
|
||||
client = _minor_release()
|
||||
_override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN)
|
||||
|
|
|
|||
|
|
@ -12,15 +12,118 @@ import asyncio
|
|||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Mapping
|
||||
from contextlib import AbstractAsyncContextManager
|
||||
from types import SimpleNamespace
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from fastapi import HTTPException
|
||||
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.models.user import LiteLLM_UserTable
|
||||
from litellm.proxy.db.log_db_metrics import record_db_io
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
from litellm.repositories.user_repository import free_sso_user_creation
|
||||
|
||||
|
||||
class _SSOAdmissionUsers:
|
||||
def __init__(self, count: int) -> None:
|
||||
self.rows: Mapping[str, LiteLLM_UserTable] = {
|
||||
f"user-{index}": LiteLLM_UserTable(user_id=f"user-{index}")
|
||||
for index in range(count)
|
||||
}
|
||||
|
||||
async def find_unique(
|
||||
self, where: Mapping[str, object]
|
||||
) -> LiteLLM_UserTable | None:
|
||||
return self.rows.get(str(where["user_id"]))
|
||||
|
||||
async def count(self, where: Mapping[str, object] | None = None) -> int:
|
||||
if where is None:
|
||||
return len(self.rows)
|
||||
return sum((row.metadata or {}).get("scim_active") is False for row in self.rows.values())
|
||||
|
||||
async def upsert(
|
||||
self, where: Mapping[str, object], data: Mapping[str, object]
|
||||
) -> LiteLLM_UserTable:
|
||||
existing: Final = await self.find_unique(where)
|
||||
if existing is not None:
|
||||
return existing
|
||||
created: Final = LiteLLM_UserTable.model_validate(data["create"])
|
||||
self.rows = {**self.rows, created.user_id: created}
|
||||
return created
|
||||
|
||||
|
||||
class _SSOAdmissionTransaction(AbstractAsyncContextManager["_SSOAdmissionTransaction"]):
|
||||
def __init__(self, users: _SSOAdmissionUsers) -> None:
|
||||
self.litellm_usertable: Final = users
|
||||
|
||||
async def __aenter__(self) -> _SSOAdmissionTransaction:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info: object) -> None:
|
||||
return None
|
||||
|
||||
async def execute_raw(self, query: str) -> int:
|
||||
return 0
|
||||
|
||||
async def query_raw(self, query: str) -> object:
|
||||
return ({"locked": 1},)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("existing_count", "scope_id", "user_id", "enabled", "child_task", "refused"),
|
||||
[
|
||||
(4, "new-user", "new-user", True, False, False),
|
||||
(5, "new-user", "new-user", True, False, True),
|
||||
(6, "user-0", "user-0", True, False, False),
|
||||
(5, "sso-user", "new-user", True, False, False),
|
||||
(5, "new-user", "new-user", True, True, False),
|
||||
(5, None, "new-user", True, False, True),
|
||||
(5, None, "new-user", False, False, False),
|
||||
],
|
||||
)
|
||||
async def test_user_insert_enforces_free_sso_admission_only_for_its_owner(
|
||||
prisma_client: PrismaClient,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
existing_count: int,
|
||||
scope_id: str | None,
|
||||
user_id: str,
|
||||
enabled: bool,
|
||||
child_task: bool,
|
||||
refused: bool,
|
||||
) -> None:
|
||||
users: Final = _SSOAdmissionUsers(existing_count)
|
||||
prisma_client.db.litellm_usertable = users
|
||||
prisma_client.db.tx.return_value = _SSOAdmissionTransaction(users)
|
||||
monkeypatch.setattr("asyncio.sleep", AsyncMock(return_value=None))
|
||||
original: Final = users.rows.get(user_id)
|
||||
|
||||
async def attempt() -> None:
|
||||
if refused:
|
||||
with pytest.raises(HTTPException) as error:
|
||||
await prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
|
||||
assert error.value.status_code == 403
|
||||
assert user_id not in users.rows
|
||||
elif child_task:
|
||||
result: Final = await asyncio.create_task(
|
||||
prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
|
||||
)
|
||||
assert result is users.rows[user_id]
|
||||
else:
|
||||
inserted: Final = await prisma_client.insert_data(data={"user_id": user_id}, table_name="user")
|
||||
assert inserted is users.rows[user_id]
|
||||
|
||||
with free_sso_user_creation(scope_id, enabled=enabled):
|
||||
for _ in range(2):
|
||||
await attempt()
|
||||
|
||||
assert len(users.rows) == existing_count + int(not refused and original is None)
|
||||
if original is not None:
|
||||
assert users.rows[user_id] is original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st
|
|||
in expected_responses_api_request/.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
|
|
@ -1231,3 +1232,110 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) ->
|
|||
assert [event.response.output[0].content[0].text for event in response if event.type == "response.completed"] == [
|
||||
"Done."
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("stream", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
("api_base", "explicit_header", "expected_session"),
|
||||
[
|
||||
("https://opencode.ai/inference/go/openai/v1", None, "conversation-123"),
|
||||
("https://opencode.ai/inference/go/openai/v1", "caller-override", "caller-override"),
|
||||
("https://other.example/v1", None, None),
|
||||
],
|
||||
)
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_opencode_session_header(
|
||||
asynchronous: bool, stream: bool, api_base: str, explicit_header: str | None, expected_session: str | None
|
||||
) -> None:
|
||||
received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue()
|
||||
payload: Final = _minimal_responses_api_payload("resp_test", "gpt-6-luna")
|
||||
|
||||
def respond(request: httpx.Request) -> httpx.Response:
|
||||
received.put_nowait(request)
|
||||
return (
|
||||
httpx.Response(
|
||||
200,
|
||||
headers={"content-type": "text/event-stream"},
|
||||
text="".join(
|
||||
f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': payload})}\n\n"
|
||||
for kind in ("response.created", "response.completed")
|
||||
),
|
||||
)
|
||||
if stream
|
||||
else httpx.Response(200, json=payload)
|
||||
)
|
||||
|
||||
args: Final = {
|
||||
"model": "openai/gpt-6-luna",
|
||||
"input": "hello",
|
||||
"api_base": api_base,
|
||||
"api_key": "test-key",
|
||||
"litellm_session_id": "conversation-123",
|
||||
"stream": stream,
|
||||
"extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"},
|
||||
}
|
||||
if asynchronous:
|
||||
async_client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond))
|
||||
try:
|
||||
response: Final = await litellm.aresponses(**args, client=async_client)
|
||||
if stream:
|
||||
events: Final = tuple([event.type async for event in response])
|
||||
assert "response.completed" in events
|
||||
else:
|
||||
assert response.output[0].content[0].text == "Done."
|
||||
finally:
|
||||
await async_client.close()
|
||||
else:
|
||||
sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond)))
|
||||
try:
|
||||
result: Final = litellm.responses(**args, client=sync_client)
|
||||
if stream:
|
||||
assert "response.completed" in tuple(event.type for event in result)
|
||||
else:
|
||||
assert result.output[0].content[0].text == "Done."
|
||||
finally:
|
||||
sync_client.close()
|
||||
sent: Final = received.get_nowait()
|
||||
assert str(sent.url) == f"{api_base}/responses"
|
||||
assert sent.headers.get("x-opencode-session") == expected_session
|
||||
assert sent.headers["authorization"] == "Bearer test-key"
|
||||
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello", "stream": stream}
|
||||
assert received.empty()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("asynchronous", [False, True])
|
||||
@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"])
|
||||
@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"])
|
||||
@pytest.mark.asyncio
|
||||
async def test_responses_opencode_session_uses_effective_destination(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
respx_mock: respx.MockRouter,
|
||||
asynchronous: bool,
|
||||
source: str,
|
||||
explicit_base: str | None,
|
||||
) -> None:
|
||||
default_base: Final = "https://opencode.ai/inference/go/openai/v1"
|
||||
monkeypatch.delenv("OPENAI_BASE_URL", raising=False)
|
||||
monkeypatch.delenv("OPENAI_API_BASE", raising=False)
|
||||
monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None)
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
if source != "litellm.api_base":
|
||||
monkeypatch.setenv(source, default_base)
|
||||
endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/responses").respond(
|
||||
json=_minimal_responses_api_payload("resp_test", "gpt-6-luna")
|
||||
)
|
||||
args: Final = {
|
||||
"model": "openai/gpt-6-luna",
|
||||
"input": "hello",
|
||||
"api_base": explicit_base,
|
||||
"api_key": "test-key",
|
||||
"litellm_session_id": "conversation-123",
|
||||
}
|
||||
response: Final = await litellm.aresponses(**args) if asynchronous else litellm.responses(**args)
|
||||
assert response.output[0].content[0].text == "Done."
|
||||
assert endpoint.call_count == 1
|
||||
sent: Final = endpoint.calls[0].request
|
||||
assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123")
|
||||
assert sent.headers["authorization"] == "Bearer test-key"
|
||||
assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"}
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue