diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a707eb8044c..f6c06f7d83f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri -/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri +/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 2276efc4e34..76af64c8ccd 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -425,6 +425,7 @@ jobs: tests/unit/realtime_api tests/unit/rerank_api tests/unit/search + tests/unit/text_completions tests/unit/vector_stores tests/unit/videos python-version: "3.12" diff --git a/Dockerfile b/Dockerfile index 6bab3d5ad71..713d7f8f5ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -121,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \ FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder COPY --from=uvbin /uv /usr/local/bin/uv RUN apk add --no-cache python-3.13 -ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \ - https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz +ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \ + https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \ uv venv /opt/liteadmin --python python3.13 && \ uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \ diff --git a/README.md b/README.md index 5fd84b529dc..4feb6e00ffa 100644 --- a/README.md +++ b/README.md @@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws ### Verify Docker Image Signatures -All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). +All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065). **Verify using the pinned commit hash (recommended):** -A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key: ```bash cosign verify \ - --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + --key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \ ghcr.io/berriai/litellm: ``` diff --git a/cosign.pub b/cosign.pub index 2c2a555ab09..7603d69641f 100644 --- a/cosign.pub +++ b/cosign.pub @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb -POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg +JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ== -----END PUBLIC KEY----- diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs index b1509313b88..8429c98a88a 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs @@ -235,7 +235,10 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse], - Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[], + Self::ComputerUse20250124 + | Self::ComputerUse20251124 + | Self::Context1m20250807 + | Self::ThinkingBindingControls20260801 => &[], Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse], Self::Effort20251124 => &[BetaProvider::VertexAi], Self::FastMode20260201 | Self::Oauth20250420 => &[ @@ -272,7 +275,6 @@ impl AnthropicBeta { BetaProvider::Databricks, ], Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi], - Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi], Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks], Self::ToolSearchTool20251019 => &[ BetaProvider::Anthropic, diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 236b18fe428..d68de6c9d4d 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -67,7 +67,7 @@ "structured-outputs-2025-11-13": "structured-outputs-2025-11-13", "text_editor_20241022": null, "text_editor_20250124": null, - "thinking-binding-controls-2026-08-01": null, + "thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", "token-efficient-tools-2025-02-19": null, "web-fetch-2025-09-10": "web-fetch-2025-09-10", diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index ea9d0d7e102..fec1d8e4465 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -36,6 +36,7 @@ import httpx from pydantic import TypeAdapter from litellm._logging import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode # Cache for the loaded configuration _BETA_HEADERS_CONFIG: dict | None = None @@ -146,7 +147,7 @@ def get_beta_headers_config(url: str) -> dict: Dict containing the beta headers configuration """ # Check if local-only mode is enabled - if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true": # verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)") return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index cb52ad24c8e..c0d42eeb5c5 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` is unavailable. This MUST be called at app-creation time — once the lifespan - runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add - middleware after an application has started". + runs, the middleware stack is frozen and middleware added later never runs, so + a late call is skipped without registering anything. No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global ``ProxyTracerProvider``; the proxy publishes the real provider as the global @@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No try: if not is_otel_v2_enabled(): return + if app.middleware_stack is not None: + verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started") + return # Lazy: only the V2-enabled path needs the optional # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py index bc6d7d435b0..fc0351d5049 100644 --- a/litellm/integrations/otel/plumbing/otlp_json.py +++ b/litellm/integrations/otel/plumbing/otlp_json.py @@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex). import base64 import json +import logging from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final, TypeAlias @@ -13,9 +14,13 @@ from typing import Final, TypeAlias import requests from google.protobuf.json_format import MessageToDict from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE +_logger: Final = logging.getLogger(__name__) JSON_CONTENT_TYPE: Final = "application/json" _HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"}) @@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter): def __init__( self, endpoint: str | None, - headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict + headers: Mapping[str, str], certificate_file: str | None = None, session: "requests.Session | None" = None, + *, + max_request_size: int | None = None, ) -> None: - super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session) - self._session.headers["Content-Type"] = JSON_CONTENT_TYPE + super().__init__( + endpoint=endpoint, + headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE}, + certificate_file=certificate_file, + session=session, + max_request_size=max_request_size, + ) - def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes: - return encode_spans_json(spans) + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return SpanExportResult.FAILURE + + with self._metrics.export_operation(len(spans)) as result: + try: + serialized_data: Final = encode_spans_json(spans) + except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results + _logger.error("Failed to encode span batch: %s", error) + result.error = error + return SpanExportResult.FAILURE + + if 0 < self._max_request_size < len(serialized_data): + result.error = RequestPayloadTooLargeError( + f"Serialized span request size {len(serialized_data)} bytes " + f"exceeds max_request_size {self._max_request_size} bytes." + ) + _logger.warning("Dropping span batch: %s", result.error) + return SpanExportResult.FAILURE + + export_result: Final = self._client.export(serialized_data) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None + ) + return SpanExportResult.FAILURE + + return SpanExportResult.SUCCESS diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 3befb73b0cc..141162bdfa5 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -76,6 +76,7 @@ def agentops_preset( def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter.""" + import requests from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, ) @@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """ def __init__(self, *, endpoint: str | None, api_key: str | None) -> None: - super().__init__(endpoint=endpoint) + self._agentops_session = requests.Session() + super().__init__(endpoint=endpoint, session=self._agentops_session) self._agentops_api_key = api_key self._auth_resolved = False @@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: try: token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token") if token: - # ``_session`` is the requests.Session the base exporter - # POSTs through; updating its Authorization header is how the - # minted JWT reaches every subsequent export. - self._session.headers["Authorization"] = f"Bearer {token}" + self._agentops_session.headers["Authorization"] = f"Bearer {token}" except Exception as e: verbose_logger.debug("AgentOps JWT fetch failed: %s", e) diff --git a/litellm/litellm_core_utils/get_blog_posts.py b/litellm/litellm_core_utils/get_blog_posts.py index c8bc184f972..c3b4114d5b9 100644 --- a/litellm/litellm_core_utils/get_blog_posts.py +++ b/litellm/litellm_core_utils/get_blog_posts.py @@ -19,6 +19,7 @@ import httpx from pydantic import ConfigDict from litellm import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.types.llms.base import LiteLLMBaseModel BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour @@ -129,7 +130,7 @@ class GetBlogPosts: Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS. Fetches from ``url`` otherwise, falling back to local backup on failure. """ - if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true": return cls.load_local_blog_posts() now: Final = time.time() diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index a67c1bb7df8..3130d49a5e2 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info( if dynamic_api_key is None and api_key is not None: dynamic_api_key = api_key return model, custom_llm_provider, dynamic_api_key, api_base + + +def get_provider_affinity_header(api_base: str | None) -> str | None: + from litellm.llms.opencode.transformation import OpenCodeProviderAdapter + + return OpenCodeProviderAdapter().get_provider_affinity_header(api_base) diff --git a/litellm/litellm_core_utils/get_model_cost_map.py b/litellm/litellm_core_utils/get_model_cost_map.py index a162ad94510..7fc6eab6c60 100644 --- a/litellm/litellm_core_utils/get_model_cost_map.py +++ b/litellm/litellm_core_utils/get_model_cost_map.py @@ -39,6 +39,7 @@ from litellm.constants import ( from litellm.litellm_core_utils.fallback_generalizations import ( set_fallback_generalizations, ) +from litellm.litellm_core_utils.offline_mode import is_offline_mode FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations" _CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]]) @@ -458,7 +459,7 @@ async def refetch_model_cost_map( on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the map they already have. """ - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true": _cost_map_source_info.loaded_at = datetime.now(timezone.utc) _cost_map_source_info.source = "local" _cost_map_source_info.url = None @@ -742,7 +743,7 @@ def get_model_cost_map( _cost_map_source_info.loaded_at = datetime.now(timezone.utc) # Note: can't use get_secret_bool here — this runs during litellm.__init__ # before litellm._key_management_settings is set. - if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): + if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process(): _cost_map_source_info.source = "local" _cost_map_source_info.url = None _cost_map_source_info.is_env_forced = True diff --git a/litellm/litellm_core_utils/offline_mode.py b/litellm/litellm_core_utils/offline_mode.py new file mode 100644 index 00000000000..7fdcae8825f --- /dev/null +++ b/litellm/litellm_core_utils/offline_mode.py @@ -0,0 +1,7 @@ +import os + + +def is_offline_mode() -> bool: + """True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather + than from a destination the operator configured, must check this and use its bundled copy or skip instead""" + return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true" diff --git a/litellm/litellm_core_utils/provider_affinity.py b/litellm/litellm_core_utils/provider_affinity.py index 02c4cd69159..e022839c5c7 100644 --- a/litellm/litellm_core_utils/provider_affinity.py +++ b/litellm/litellm_core_utils/provider_affinity.py @@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None: return getattr(value, key, None) -def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None: +def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None: header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None if header is None: - return None + return default_header if not isinstance(header, str): raise TypeError("provider_affinity_header must be a string") return validate_provider_affinity_header_name(header) @@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None: def add_provider_affinity_header( - headers: Mapping[str, object], litellm_params: object | None + headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None ) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers - header_name: Final = _get_provider_affinity_header_name(litellm_params) + header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header) if header_name is None or any(key.lower() == header_name.lower() for key in headers): return dict(headers) diff --git a/litellm/llms/aws_polly/text_to_speech/transformation.py b/litellm/llms/aws_polly/text_to_speech/transformation.py index 01fec02c310..325c9fd364d 100644 --- a/litellm/llms/aws_polly/text_to_speech/transformation.py +++ b/litellm/llms/aws_polly/text_to_speech/transformation.py @@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM): else: mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT - # Extract engine from model name (e.g., "aws_polly/neural" -> "neural") + # model arrives without the "aws_polly/" prefix, e.g. "standard" engine: Final = self._extract_engine_from_model(model) mapped_params["engine"] = engine @@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM): """ Extract engine from model name. + speech() strips the "aws_polly/" prefix before the config is called, + so the bare engine name is what normally arrives here. + Examples: - - aws_polly/neural -> neural - - aws_polly/standard -> standard + - standard -> standard - aws_polly/long-form -> long-form - aws_polly -> neural (default) """ - if "/" in model: - parts: Final = model.split("/") - if len(parts) >= 2: - engine: Final = parts[1].lower() - if engine in self.VALID_ENGINES: - return engine + engine: Final = model.rsplit("/", 1)[-1].lower() + if engine in self.VALID_ENGINES: + return engine return self.DEFAULT_ENGINE def validate_environment( diff --git a/litellm/llms/azure/responses/transformation.py b/litellm/llms/azure/responses/transformation.py index 86ae8f5c224..6d904503df8 100644 --- a/litellm/llms/azure/responses/transformation.py +++ b/litellm/llms/azure/responses/transformation.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from copy import deepcopy from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal +from urllib.parse import urlsplit import httpx from openai.types.responses import ResponseReasoningItem @@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig): "https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview" """ resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base) + responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params) + parsed_base: Final = urlsplit(resolved_api_base or "") + path: Final = parsed_base.path.rstrip("/") + if path.endswith(("/openai/v1", "/openai/v1/responses")): + responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path + return BaseAzureLLM.get_base_azure_url( + api_base=parsed_base._replace(path=responses_path).geturl(), + litellm_params=responses_params, + route="/openai/v1/responses", + default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, + ) + return BaseAzureLLM.get_base_azure_url( api_base=resolved_api_base, - litellm_params=_with_responses_api_version(resolved_api_base, litellm_params), + litellm_params=responses_params, route="/openai/responses", default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION, ) diff --git a/litellm/llms/azure_ai/chat/transformation.py b/litellm/llms/azure_ai/chat/transformation.py index f1d062b1645..3b541310d28 100644 --- a/litellm/llms/azure_ai/chat/transformation.py +++ b/litellm/llms/azure_ai/chat/transformation.py @@ -12,6 +12,7 @@ from litellm._logging import verbose_logger from litellm.litellm_core_utils.prompt_templates.common_utils import ( audio_or_image_in_message_content, convert_content_list_to_str, + drop_tool_reference_parts_from_tool_messages, filter_value_from_dict, ) from litellm.llms.azure.common_utils import BaseAzureLLM @@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig): Operates on a deep copy so the caller's messages keep their thinking blocks and provider metadata, which a fallback to another provider still needs. """ - stripped_messages: Final = copy.deepcopy(messages) + stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages)) for message in stripped_messages: message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy for field in NON_OPENAI_SPEC_MESSAGE_FIELDS: diff --git a/litellm/llms/bedrock/count_tokens/kimi_k3.py b/litellm/llms/bedrock/count_tokens/kimi_k3.py index 5e3fd56e4ba..4f8b4624cd8 100644 --- a/litellm/llms/bedrock/count_tokens/kimi_k3.py +++ b/litellm/llms/bedrock/count_tokens/kimi_k3.py @@ -27,6 +27,7 @@ from typing_extensions import Self import litellm from litellm._logging import verbose_logger +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.prompt_templates.factory import ( chat_completion_tools_for, is_flat_responses_tool, @@ -67,12 +68,14 @@ class KimiK3TokenizerSource: def _hugging_face_ranks_path() -> str: import huggingface_hub - return huggingface_hub.hf_hub_download( - repo_id=KIMI_K3_REPO, - filename=KIMI_K3_RANKS_FILE, - revision=KIMI_K3_REVISION, - local_files_only=litellm.disable_hf_tokenizer_download is True, - ) + if litellm.disable_hf_tokenizer_download is True or is_offline_mode(): + cached: Final = huggingface_hub.try_to_load_from_cache( + repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION + ) + if not isinstance(cached, str): + raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache") + return cached + return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION) HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 96fb65d41eb..55b2c21718d 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import ( from litellm.litellm_core_utils.core_helpers import as_str_mapping, set_provider_response_headers_in_hidden_params from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.litellm_core_utils.realtime_errors import ( close_after_upstream_handshake_refusal, realtime_error_event, @@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import ( ) from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base +from litellm.llms.anthropic.common_utils import AnthropicModelInfo from litellm.llms.base_llm.anthropic_messages.transformation import ( BaseAnthropicMessagesConfig, ) @@ -2457,13 +2460,13 @@ class BaseLLMHTTPHandler: # Also check for extra_headers in kwargs (from config or direct calls) extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None) # Merge all header sources: forwarded < extra_headers < provider_specific - merged_headers: Final = {} - if forwarded_headers: - merged_headers.update(forwarded_headers) - if extra_headers_from_kwargs: - merged_headers.update(extra_headers_from_kwargs) - if provider_specific_headers: - merged_headers.update(provider_specific_headers) + merged_headers: Final[Mapping[str, object]] = MappingProxyType( + { + **(forwarded_headers or {}), + **(extra_headers_from_kwargs or {}), + **(provider_specific_headers or {}), + } + ) request_params: Final = reduce( delete_nested_value, litellm_params.get("additional_drop_params") or (), @@ -2474,11 +2477,19 @@ class BaseLLMHTTPHandler: and anthropic_messages_provider_config.should_filter_anthropic_beta_headers() ): await aload_beta_headers_config() + affinity_api_base: Final = ( + AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) ( headers, api_base, ) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment( - headers=merged_headers or {}, + headers=( + add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header) + if default_affinity_header is not None + else dict(merged_headers) + ), model=model, messages=messages, optional_params=request_params, diff --git a/litellm/llms/opencode/common_utils.py b/litellm/llms/opencode/common_utils.py new file mode 100644 index 00000000000..65a96411e52 --- /dev/null +++ b/litellm/llms/opencode/common_utils.py @@ -0,0 +1,10 @@ +import re + + +def is_opencode_api_base(api_base: str | None) -> bool: + return bool( + api_base + and re.fullmatch( + r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base + ) + ) diff --git a/litellm/llms/opencode/transformation.py b/litellm/llms/opencode/transformation.py new file mode 100644 index 00000000000..9b62e7aca95 --- /dev/null +++ b/litellm/llms/opencode/transformation.py @@ -0,0 +1,6 @@ +from litellm.llms.opencode.common_utils import is_opencode_api_base + + +class OpenCodeProviderAdapter: + def get_provider_affinity_header(self, api_base: str | None) -> str | None: + return "x-opencode-session" if is_opencode_api_base(api_base) else None diff --git a/litellm/main.py b/litellm/main.py index 6bdad30fdcb..c22cd8eb11c 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import ( with_control_options, ) from litellm.litellm_core_utils.get_llm_provider_logic import ( + get_provider_affinity_header, invalid_provider_routing_hints_message, ) from litellm.litellm_core_utils.get_provider_specific_headers import ( @@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl from litellm.llms.cohere.common_utils import CohereModelInfo from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config +from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.json_loader import JSONProviderRegistry from litellm.llms.vertex_ai.common_utils import ( VertexAIModelRoute, @@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import ( resolve_chat_reasoning, ) from litellm.secret_managers.main import get_secret_bool, get_secret_str +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper from litellm.types.completion import ( CompletionDispatchContext, _CompletionDispatchResult, @@ -169,7 +172,6 @@ from litellm.utils import ( ProviderConfigManager, TextChoices, TextCompletionResponse, - TextCompletionStreamWrapper, TranscriptionResponse, Usage, add_provider_specific_params_to_optional_params, @@ -5996,19 +5998,28 @@ def completion( }, ) litellm_params: Final = with_control_options(requested_litellm_params, control_options) - if litellm_params.get("provider_affinity_header") is not None: + affinity_api_base: Final = ( + str(client.base_url) + if isinstance(client, openai.OpenAI | openai.AsyncOpenAI) + else get_openai_credentials(api_base=api_base).api_base + if custom_llm_provider == "openai" + else api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) + if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None: try: headers = add_provider_affinity_header( headers=headers or litellm.headers or MappingProxyType({}), litellm_params=MappingProxyType( { - "provider_affinity_header": litellm_params["provider_affinity_header"], + "provider_affinity_header": litellm_params.get("provider_affinity_header"), "litellm_session_id": kwargs.get("litellm_session_id"), "session_id": kwargs.get("session_id"), "metadata": metadata, "litellm_metadata": kwargs.get("litellm_metadata"), } ), + default_header=default_affinity_header, ) except ValueError as affinity_error: raise litellm.BadRequestError( diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index cdb9a925403..2e9d6a77a62 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target: _FEATURES: Final[detectors.Detectors] = MappingProxyType( { **detectors.COMMON, + Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity, Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier, Feature.RESPONSE_CACHE: detectors.response_cache, Feature.MODIFY_PARAMS: detectors.modify_params, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 90a0763f786..3edb30b4218 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -48499,6 +48723,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -52906,12 +53141,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53163,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -73330,6 +73569,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/litellm/proxy/_experimental/out/assets/aria-agent.png b/litellm/proxy/_experimental/out/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/litellm/proxy/_experimental/out/assets/aria-agent.png differ diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 03e71f64fe4..aa67d36af4f 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.spend_tracking.vantage_endpoints", path_prefixes=("/vantage",), ), - LazyFeature( - name="usage_ai", - module_path="litellm.proxy.management_endpoints.usage_endpoints", - path_prefixes=("/usage/ai",), - ), LazyFeature( name="prompts", module_path="litellm.proxy.prompts.prompt_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index e5e0027a808..0dd978ee855 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -57898,163 +57898,6 @@ } } }, - "usage_ai": { - "components": { - "schemas": { - "ChatMessage": { - "properties": { - "content": { - "title": "Content", - "type": "string" - }, - "role": { - "enum": [ - "user", - "assistant" - ], - "title": "Role", - "type": "string" - } - }, - "required": [ - "role", - "content" - ], - "title": "ChatMessage", - "type": "object" - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "title": "Detail", - "type": "array" - } - }, - "title": "HTTPValidationError", - "type": "object" - }, - "UsageAIChatRequest": { - "properties": { - "messages": { - "description": "Chat messages (user/assistant history)", - "items": { - "$ref": "#/components/schemas/ChatMessage" - }, - "title": "Messages", - "type": "array" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Model to use for AI chat", - "title": "Model" - } - }, - "required": [ - "messages" - ], - "title": "UsageAIChatRequest", - "type": "object" - }, - "ValidationError": { - "properties": { - "ctx": { - "title": "Context", - "type": "object" - }, - "input": { - "title": "Input" - }, - "loc": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "title": "Location", - "type": "array" - }, - "msg": { - "title": "Message", - "type": "string" - }, - "type": { - "title": "Error Type", - "type": "string" - } - }, - "required": [ - "loc", - "msg", - "type" - ], - "title": "ValidationError", - "type": "object" - } - } - }, - "paths": { - "/usage/ai/chat": { - "post": { - "description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.", - "operationId": "usage_ai_chat_usage_ai_chat_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UsageAIChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Usage Ai Chat", - "tags": [ - "usage_ai" - ] - } - } - } - }, "vantage": { "components": { "schemas": { diff --git a/litellm/proxy/auth/password_policy.py b/litellm/proxy/auth/password_policy.py index 36a569bf18f..d810ba4238b 100644 --- a/litellm/proxy/auth/password_policy.py +++ b/litellm/proxy/auth/password_policy.py @@ -21,6 +21,7 @@ from typing import Final from litellm._logging import verbose_proxy_logger from litellm._version import version from litellm.constants import HIBP_RANGE_API_BASE +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client from litellm.proxy._types import ProxyErrorTypes, ProxyException from litellm.types.llms.custom_http import httpxSpecialProvider @@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool: - return general_settings.get("password_policy_check_breached_passwords", True) is not False + return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False async def is_password_breached( diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index b44ab5d2ca6..b290e0f85b4 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -12,7 +12,6 @@ import json import os import random import time -import traceback from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence from contextlib import asynccontextmanager from contextvars import ContextVar @@ -91,6 +90,7 @@ from litellm.proxy.spend_tracking.savings import ( ) from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error from litellm.repositories.prisma_protocols import BatchTable +from litellm.responses.streaming_iterator import spawn_logging_task from litellm.types.utils import CallTypes if TYPE_CHECKING: @@ -329,6 +329,38 @@ def _merged_spend_restore(first: _SpendRestore, second: _SpendRestore) -> _Spend return {**first, **second} +def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None: + spend_log_error( + "Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s", + step_name, + payload.get("request_id"), + payload.get("api_key"), + exc=exc, + ) + + +_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata + + +def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata: + """Malformed metadata costs the request its metadata-derived fields, never its daily rollup row.""" + raw: Final = payload.get("metadata") + if not isinstance(raw, str): + return _EMPTY_SPEND_LOG_METADATA + try: + parsed: Final[SpendLogsMetadata] = json.loads(raw) + except json.JSONDecodeError as e: + spend_log_error( + "Spend tracking - unreadable spend log metadata, rollups for this request are written without " + "metadata-derived fields. request_id=%s api_key=%s", + payload.get("request_id"), + payload.get("api_key"), + exc=e, + ) + return _EMPTY_SPEND_LOG_METADATA + return parsed + + def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool: """An earlier billed retry attempt's own spend row, not a request the caller sent.""" return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True @@ -663,8 +695,8 @@ class DBSpendUpdateWriter: "disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur." ) - # Single task replaces 11 create_task() calls - asyncio.create_task( + spawn_logging_task( + asyncio.get_running_loop(), self._batch_database_updates( response_cost=response_cost, user_id=user_id, @@ -677,7 +709,8 @@ class DBSpendUpdateWriter: litellm_proxy_budget_name=litellm_proxy_budget_name, payload=payload, request_model_access_groups=get_request_model_access_groups(kwargs), - ) + ), + task_name="spend rollups", ) self._enqueue_tool_registry_upsert( @@ -1102,91 +1135,52 @@ class DBSpendUpdateWriter: project_id: str | None = None, ): """ - Runs all 13 spend-update helpers sequentially inside a single asyncio task. - - Each helper is wrapped in try/except so one failure doesn't prevent the others. - - The deepcopy runs here, off the awaited request path, so the daily spend - helpers get a payload isolated from the spend-log queue entry and the caller. + Runs every spend-update helper for one request. Each helper goes through + _run_spend_update_step so one failure never skips the sibling writes. The deepcopy + isolates the daily spend helpers' payload from the spend-log queue entry and the caller. """ payload_copy: Final = copy.deepcopy(payload) request_tags: Final = payload_copy.get("request_tags") - try: - await self._update_user_db( + await self._run_spend_update_step( + "_update_user_db", + self._update_user_db( response_cost=response_cost, user_id=user_id, prisma_client=prisma_client, litellm_proxy_budget_name=litellm_proxy_budget_name, end_user_id=end_user_id, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_user_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_key_db( - response_cost=response_cost, - hashed_token=hashed_token, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_key_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_team_db( - response_cost=response_cost, - team_id=team_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_team_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_org_db( - response_cost=response_cost, - org_id=org_id, - user_id=user_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_org_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_project_db( - response_cost=response_cost, - project_id=project_id, - prisma_client=prisma_client, - ) - except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _update_project_db failed: %s", - traceback.format_exc(), - ) - - try: - await self._update_tag_db( - response_cost=response_cost, - request_tags=request_tags, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_tag_db failed: %s", - traceback.format_exc(), - ) - + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_key_db", + self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_team_db", + self._update_team_db( + response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_org_db", + self._update_org_db( + response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_project_db", + self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "_update_tag_db", + self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client), + payload=payload_copy, + ) await self._update_model_access_group_db( response_cost=response_cost, request_model_access_groups=request_model_access_groups, @@ -1194,96 +1188,64 @@ class DBSpendUpdateWriter: prisma_client=prisma_client, router=get_llm_router(), ) - - _agent_id_for_spend: Final = payload_copy.get("agent_id") - try: - await self._update_agent_db( - response_cost=response_cost, - agent_id=_agent_id_for_spend, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: _update_agent_db failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_end_user_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_agent_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_team_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s", - traceback.format_exc(), - ) + await self._run_spend_update_step( + "_update_agent_db", + self._update_agent_db( + response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_user_transaction", + self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_end_user_transaction", + self.add_spend_log_transaction_to_daily_end_user_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_agent_transaction", + self.add_spend_log_transaction_to_daily_agent_transaction( + payload=payload_copy, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_team_transaction", + self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) try: self._record_request_error(payload=payload_copy, prisma_client=prisma_client) - except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes - verbose_proxy_logger.debug( - "_batch_database_updates: _record_request_error failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_org_transaction( - payload=payload_copy, - org_id=org_id, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s", - traceback.format_exc(), - ) - - try: - await self.add_spend_log_transaction_to_daily_tag_transaction( - payload=payload_copy, - prisma_client=prisma_client, - ) - except Exception: - verbose_proxy_logger.debug( - "_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s", - traceback.format_exc(), - ) - + except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes + _report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_org_transaction", + self.add_spend_log_transaction_to_daily_org_transaction( + payload=payload_copy, org_id=org_id, prisma_client=prisma_client + ), + payload=payload_copy, + ) + await self._run_spend_update_step( + "add_spend_log_transaction_to_daily_tag_transaction", + self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client), + payload=payload_copy, + ) await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client) + @staticmethod + async def _run_spend_update_step( + step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload + ) -> None: + try: + await step + except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes + _report_spend_update_step_failure(step_name, payload=payload, exc=e) + async def _update_key_db( self, response_cost: float | None, @@ -2857,7 +2819,7 @@ class DBSpendUpdateWriter: date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "") if not date: return - metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + metadata: Final = _parse_spend_log_metadata(payload) internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults request_error_accumulator.record( payload=payload, @@ -2908,7 +2870,7 @@ class DBSpendUpdateWriter: # spend, token and per-entity columns written here stay either way. request_status: Final = prisma_client.get_request_status(payload) verbose_proxy_logger.debug("Logged request status: %s", request_status) - _metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"]) + _metadata: Final = _parse_spend_log_metadata(payload) usage_obj: Final = _metadata.get("usage_object", {}) or {} if isinstance(payload["startTime"], datetime): start_time: Final = payload["startTime"].isoformat() diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 8a45f658f36..8dc7501bc3b 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = ( "/v1/messages", "responses", "files", + "interactions", ) LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset( @@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( LlmProviders.VERTEX_AI.value, ) ) +_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join( + (LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value) +) _ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS)) @@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request( custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers ), ProviderSpecificHeader( - custom_llm_provider=LlmProviders.DATABRICKS.value, + custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers, native_anthropic_messages_only=True, ), diff --git a/litellm/proxy/management_endpoints/internal_user_endpoints.py b/litellm/proxy/management_endpoints/internal_user_endpoints.py index b563dbc5c4c..d6bf9220551 100644 --- a/litellm/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/proxy/management_endpoints/internal_user_endpoints.py @@ -2093,7 +2093,9 @@ async def get_user_key_counts( return result -def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None: +def _validate_sort_params( + sort_by: str | None, sort_order: str +) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None: order_by: Final[dict[str, str]] = {} if sort_by is None: @@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str "spend", "user_alias", "user_role", + "max_budget", + "updated_at", + "sso_user_id", + "status", + "key_count", ] if sort_by not in valid_columns: raise HTTPException( @@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"}, ) + if sort_by in {"status", "key_count"}: + return None + if sort_by in {"max_budget", "updated_at", "sso_user_id"}: + nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last" + return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"}) order_by[sort_by] = sort_order.lower() return order_by @@ -2338,16 +2350,40 @@ async def get_users( # Build order_by conditions - order_by: Final[dict[str, str] | None] = ( + order_by: Final = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) - users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many( - where=where, - skip=skip, - take=page_size, - order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified + from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids + + computed_sort: Final = sort_by in ("status", "key_count") + sorted_ids: Final = ( + await sorted_user_ids( + prisma_client, + sort_by="status" if sort_by == "status" else "key_count", + descending=sort_order.lower() == "desc", + skip=skip, + take=page_size, + role=role, + user_ids=user_ids, + user_email=user_email, + team=team, + sso_user_ids=sso_user_ids, + organization_ids=organization_ids, + search=search, + ) + if computed_sort + else () ) + page_users: Final = await UserRepository(prisma_client).table.find_many( + # bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route + where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where, + skip=0 if computed_sort else skip, + take=page_size, + order=order_by if order_by else {"created_at": "desc"}, + ) + positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)} + users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users # Get total count of user rows total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 2e58cc41f13..0ad4258fb04 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -7415,20 +7415,26 @@ async def key_aliases( ) -def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None: - order_by: Final[dict[str, str]] = {} - +def _validate_sort_params( + sort_by: str | None, sort_order: str +) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None: if sort_by is None: return None # Validate sort_by is a valid column - valid_columns: Final = [ + valid_columns: Final = ( "spend", "max_budget", "created_at", "updated_at", "token", "key_alias", - ] + "expires", + "last_active", + "budget_reset_at", + "total_spend", + "tpm_limit", + "rpm_limit", + ) if sort_by not in valid_columns: raise HTTPException( status_code=400, @@ -7442,9 +7448,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"}, ) - order_by[sort_by] = sort_order.lower() - - return order_by + direction: Final = sort_order.lower() + nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit") + if sort_by not in (*nullable_sort_columns, "total_spend"): + return {sort_by: direction} + return [ + {sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction}, + {"token": "asc"}, + ] def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]: @@ -7686,7 +7697,7 @@ async def list_key_helper( verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size) - order_by: Final[dict[str, str] | None] = ( + order_by: Final = ( _validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None ) diff --git a/litellm/proxy/management_endpoints/management_v1/budgets.py b/litellm/proxy/management_endpoints/management_v1/budgets.py index 1970a0797ef..19a5408496a 100644 --- a/litellm/proxy/management_endpoints/management_v1/budgets.py +++ b/litellm/proxy/management_endpoints/management_v1/budgets.py @@ -3,6 +3,7 @@ from collections.abc import Mapping, Sequence from dataclasses import dataclass from datetime import datetime +from itertools import chain from types import MappingProxyType from typing import Annotated, Final @@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...]) SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields) +# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt. +_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰0𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰" +_DECIMAL_DIGITS: Final = "".join( + chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES) +) +_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES) + + +_BUDGET_DURATION_SECONDS: Final = f"""( + SELECT EXTRACT(EPOCH FROM ( + (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + + (parts[1] || ' ' || CASE parts[2] + WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours' + WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval + - (CURRENT_TIMESTAMP AT TIME ZONE 'UTC') + )) FROM regexp_match( + CASE lower(trim(budget_duration)) + WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h' + WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d' + ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration + ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END, + '^([0-9]+)(mo|[smhdw])' + ) AS duration(parts) +)""" + + @dataclass(frozen=True, slots=True) class PrismaBudgetListExecutor: """The database half of the budgets list. Every caller-supplied value is bound to a @@ -91,10 +118,16 @@ class PrismaBudgetListExecutor: async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]: clauses, params = where_sql(plan.where) + ordering: Final = ", ".join( + order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST") + if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit") + else order_by_sql((key,)) + for key in plan.order + ).replace('"budget_duration"', _BUDGET_DURATION_SECONDS) sql: Final = ( f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}" + (f" WHERE {clauses}" if clauses else "") - + f" ORDER BY {order_by_sql(plan.order)}" + + f" ORDER BY {ordering}" + f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}" ) return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip)) @@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope: return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}") -# budget_duration is deliberately absent from `sortable`: the column holds strings -# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d". BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( { "budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))), @@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType( BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec( resource="budgets", - sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")), + sortable=frozenset( + ("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration") + ), searchable=frozenset(("budget_id",)), filters=BUDGET_FILTERS, default_sort=(SortKey(field="created_at", descending=True),), @@ -154,8 +187,9 @@ async def list_budgets( way to page, sort or filter it. `sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`, - `rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending, - and defaults to `-created_at`. `budget_id` is appended to every sort as the + `rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending, + and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months + measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the tiebreaker. `q` is a case-insensitive substring match on `budget_id`. `page_size` defaults to 50 and is capped at 100. Filters are `filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]` diff --git a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py index 3bef7c107eb..7cd36d51511 100644 --- a/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py +++ b/litellm/proxy/management_endpoints/policy_endpoints/endpoints.py @@ -28,6 +28,7 @@ from litellm.constants import ( MAX_COMPETITOR_NAMES, ) from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.llms.openai.chat.guardrail_translation.handler import ( OpenAIChatCompletionsHandler, ) @@ -657,7 +658,7 @@ async def get_policy_templates( Fetches from GitHub with automatic fallback to local backup on failure. Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only. """ - use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in ( + use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in ( "true", "1", "yes", diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index bc9539f4ac4..361778ab2ea 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -6117,13 +6117,52 @@ async def list_team_v2( "total_pages": 0, } - # Build order_by conditions - valid_sort_columns: Final = ["team_id", "team_alias", "created_at"] - order_by = None - if sort_by and sort_by in valid_sort_columns: - if sort_order.lower() not in ["asc", "desc"]: - sort_order = "asc" - order_by = {sort_by: sort_order.lower()} + from litellm.proxy.management_helpers.team_list_sorting import ( + COMPUTED_TEAM_SORTS, + UnsupportedTeamFilter, + sorted_team_ids, + ) + + valid_sort_columns: Final = ( + "team_id", + "team_alias", + "created_at", + "spend", + "max_budget", + "tpm_limit", + "rpm_limit", + "updated_at", + ) + sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc" + nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit") + order_by: Final = ( + ( + {sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}}, + {"team_id": "asc"}, + ) + if nullable_sort + else ({sort_by: sort_direction}, {"team_id": "asc"}) + if sort_by in ("spend", "updated_at") + else {sort_by: sort_direction} + if sort_by in valid_sort_columns + else None + ) + computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS + sorted_ids: Final = ( + await sorted_team_ids( + prisma_client, + where=where_conditions, + sort_by=sort_by or "", + descending=sort_direction == "desc", + skip=skip, + take=page_size, + ) + if computed_sort + else () + ) + + if isinstance(sorted_ids, UnsupportedTeamFilter): + raise HTTPException(status_code=500, detail="Unsupported team sorting filter") # Get teams with pagination if use_deleted_table: @@ -6138,26 +6177,32 @@ async def list_team_v2( total_count = await _deleted_team_db(prisma_client).count(where=where_conditions) else: teams = await _team_db(prisma_client).find_many( - where=where_conditions, - skip=skip, + # bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route + where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions, + skip=0 if computed_sort else skip, take=page_size, - order=order_by if order_by else {"created_at": "desc"}, # Default sort + order=order_by if order_by else {"created_at": "desc"}, include=_INCLUDE_MODEL_TABLE, ) # Get total count for pagination total_count = await _team_db(prisma_client).count(where=where_conditions) + positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)} + sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams + # Calculate total pages total_pages: Final = -(-total_count // page_size) # Ceiling division # Aggregate virtual-key counts per team for the current page. The deleted # table does not carry keys_count, so it is skipped. - keys_count_by_team: dict[str, int] = {} - if not use_deleted_table: - keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams) + keys_count_by_team: Final = ( + await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {} + ) # Convert Prisma models to response models with members_count and keys_count - team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team) + team_list: Final = _convert_teams_to_response_models( + sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team + ) # Resolve resources inherited from access groups (single batch query) if not use_deleted_table: diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 46d6e25a48f..02494de8c87 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -152,7 +152,7 @@ from litellm.proxy.utils import ( from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository -from litellm.repositories.user_repository import UserRepository +from litellm.repositories.user_repository import UserRepository, free_sso_user_creation from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403 @@ -1085,25 +1085,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token( ) -async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None: - """Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license.""" - if premium_user is True: - return - if prisma_client is None: - raise ProxyException( - message=CommonProxyErrors.db_not_connected_error.value, - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) - billable_users: Final = await UserRepository(prisma_client).count_billable_users() - if billable_users and billable_users > 5: - raise ProxyException( - message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it", - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) +class _SSOUserAdmissionError(ProxyException): + pass @router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False) @@ -1123,8 +1106,6 @@ async def google_login( from litellm.proxy.proxy_server import ( cli_sso_session_cache, general_settings, - premium_user, - prisma_client, saml_sso_cache, user_custom_ui_sso_sign_in_handler, ) @@ -1139,15 +1120,6 @@ async def google_login( if admin_ui_is_disabled and not is_cli_sso_login: return admin_ui_disabled() - ####### Check if user is a Enterprise / Premium User ####### - if ( - microsoft_client_id is not None - or google_client_id is not None - or generic_client_id is not None - or SAMLAuthHandler.is_saml_configured() - ): - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ####### Detect DB + MASTER KEY in .env ####### missing_env_vars: Final = show_missing_vars_in_env() if missing_env_vars is not None: @@ -1972,6 +1944,8 @@ async def get_user_info_from_db( ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e) @@ -2337,7 +2311,6 @@ async def saml_callback(request: Request): general_settings, jwt_handler, master_key, - premium_user, prisma_client, saml_sso_cache, ) @@ -2361,8 +2334,6 @@ async def saml_callback(request: Request): user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data) - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ui_access_mode: Final = general_settings.get("ui_access_mode", None) relay_state: Final = post_data.get("RelayState") cp_return_to: Final[str | None] = ( @@ -2826,12 +2797,20 @@ async def insert_sso_user( if result_openid and hasattr(result_openid, "provider"): new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")} - response: Final = await new_user( - data=new_user_request, - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), - ) + from litellm.proxy.proxy_server import premium_user - return response + with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True): + try: + return await new_user( + data=new_user_request, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + except ProxyException as error: + if premium_user is True: + raise + raise _SSOUserAdmissionError( + message=error.message, type=error.type, param=error.param, code=error.code + ) from error DefaultTeamJoin: TypeAlias = Callable[ @@ -3572,6 +3551,8 @@ class SSOAuthenticationHandler: prisma_client=prisma_client, ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if ( user_info is None diff --git a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py b/litellm/proxy/management_endpoints/usage_endpoints/__init__.py deleted file mode 100644 index 6e68dcd2a2e..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Usage endpoints package. - -Re-exports the router from endpoints module. -""" - -from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401 - router, -) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py deleted file mode 100644 index 7a45d0438de..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ /dev/null @@ -1,627 +0,0 @@ -""" -AI Usage Chat - uses LLM tool calling to answer questions about -usage/spend data by querying the aggregated daily activity endpoints. -""" - -import json -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence -from datetime import date -from typing import Final, Literal, NamedTuple, Protocol, cast, overload - -from fastapi import HTTPException -from typing_extensions import ReadOnly, TypedDict - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL -from litellm.proxy._types import CommonProxyErrors -from litellm.types.proxy.management_endpoints.common_daily_activity import ( - SpendAnalyticsPaginatedResponse, -) -from litellm.types.utils import ChatCompletionMessageToolCall - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -USAGE_AI_TEMPERATURE: Final = 0.2 - -TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend" -TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend" -TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend" - -ENTITY_FIELD_USER: Final = "user_id" -ENTITY_FIELD_TEAM: Final = "team_id" -ENTITY_FIELD_TAG: Final = "tag" - -PAGINATED_PAGE_SIZE: Final = 200 -MAX_CHAT_MESSAGES: Final = 20 -TOP_N_MODELS: Final = 15 -TOP_N_PROVIDERS: Final = 10 -TOP_N_KEYS: Final = 10 - -# --------------------------------------------------------------------------- -# Types -# --------------------------------------------------------------------------- - - -class SSEStatusEvent(TypedDict): - type: Literal["status"] - message: str - - -class SSEToolCallEvent(TypedDict, total=False): - type: Literal["tool_call"] - tool_name: str - tool_label: str - arguments: dict[str, str] - status: Literal["running", "complete", "error"] - error: str - - -class SSEChunkEvent(TypedDict): - type: Literal["chunk"] - content: str - - -class SSEDoneEvent(TypedDict): - type: Literal["done"] - - -class SSEErrorEvent(TypedDict): - type: Literal["error"] - message: str - - -SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent - - -class _EntityEntry(TypedDict, total=False): - metrics: ReadOnly[Mapping[str, float]] - metadata: ReadOnly[Mapping[str, str]] - - -class _DayDump(TypedDict, total=False): - breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]] - - -class _EntityTotal(NamedTuple): - """Running per-entity totals accumulated while summarising a usage dump.""" - - alias: str - spend: float - requests: float - tokens: float - - -class _UsageDump(Protocol): - @overload - def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ... - @overload - def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ... - - -class _ToolFunctionDef(TypedDict): - name: ReadOnly[str] - description: ReadOnly[str] - parameters: ReadOnly[Mapping[str, object]] - - -class _ToolDef(TypedDict): - type: ReadOnly[str] - function: ReadOnly[_ToolFunctionDef] - - -class ToolHandler(TypedDict): - fetch: Callable[..., Awaitable[_UsageDump]] - summarise: Callable[[_UsageDump], str] - label: str - - -# --------------------------------------------------------------------------- -# Tool definitions (OpenAI function-calling schema) -# --------------------------------------------------------------------------- - -_DATE_PARAMS: Final = { - "start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"}, - "end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"}, -} - -_TOOL_USAGE: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_usage_data", - "description": ( - "Fetch aggregated global usage/spend data. Returns daily spend, " - "token counts, request counts, and breakdowns by model, provider, " - "and API key. Use for overall spend, top models, top providers." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "user_id": { - "type": "string", - "description": "Optional user ID filter. Omit for global view.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TEAM: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_team_usage_data", - "description": ( - "Fetch usage/spend data broken down by team. Use for questions " - "like 'which team spends the most' or 'show me team X usage'." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "team_ids": { - "type": "string", - "description": "Optional comma-separated team IDs. Omit for all teams.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TAG: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_tag_usage_data", - "description": ( - "Fetch usage/spend data broken down by tag. Tags are labels " - "attached to requests (features, environments, credentials)." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "tags": { - "type": "string", - "description": "Optional comma-separated tag names. Omit for all tags.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -TOOLS_BASE: Final = [_TOOL_USAGE] -TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG] - - -def get_tools_for_role(is_admin: bool) -> list[_ToolDef]: - """Return the tool list appropriate for the user's role.""" - return TOOLS_ADMIN if is_admin else TOOLS_BASE - - -_SYSTEM_PROMPT_BASE: Final = ( - "You are an AI assistant embedded in the LiteLLM Usage dashboard. " - "You help users understand their LLM API spend and usage data.\n\n" - "ALWAYS call the appropriate tool(s) first to fetch data before answering. " - "You may call multiple tools if the question spans different dimensions.\n\n" - "Guidelines:\n" - "- Be concise and specific. Use exact numbers from the data.\n" - "- Format costs as dollar amounts (e.g. $12.34).\n" - "- When comparing entities, show a ranked list.\n" - "- If data is empty or no results found, say so clearly.\n" - "- Do not hallucinate data — only use what the tools return.\n" - "- Today's date will be provided below. Use it to interpret relative dates " - "like 'this week', 'this month', 'last 7 days', etc." -) - -_TOOL_DESCRIPTIONS_ADMIN: Final = ( - "You have access to these tools:\n" - "- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n" - "- `get_team_usage_data`: Team-level usage breakdown\n" - "- `get_tag_usage_data`: Tag-level usage breakdown\n\n" -) - -_TOOL_DESCRIPTIONS_BASE: Final = ( - "You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n" -) - - -def _build_system_prompt(is_admin: bool) -> str: - """Build role-appropriate system prompt with today's date.""" - tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE - return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}" - - -# keep a public reference for test assertions -SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE - -# --------------------------------------------------------------------------- -# Data fetchers -# --------------------------------------------------------------------------- - - -def _parse_csv_ids(raw: str | None) -> list[str] | None: - if not raw: - return None - return [t.strip() for t in raw.split(",") if t.strip()] - - -async def _query_activity( - table_name: str, - entity_id_field: str, - entity_id: str | list[str] | None, - start_date: str, - end_date: str, - *, - use_aggregated: bool = False, -) -> SpendAnalyticsPaginatedResponse: - """Shared helper that calls the daily activity query layer.""" - from litellm.proxy.management_endpoints.common_daily_activity import ( - daily_activity_repository, - daily_activity_scope, - get_daily_activity, - get_daily_activity_aggregated, - ) - from litellm.proxy.proxy_server import prisma_client - - if use_aggregated: - if prisma_client is None: - raise HTTPException( - status_code=500, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - repository: Final = daily_activity_repository(prisma_client) - scope: Final = daily_activity_scope( - table_name, - entity_id_field, - entity_id, - None, - None, - start_date, - end_date, - None, - None, - ) - return await get_daily_activity_aggregated( - repository, - scope, - ) - return await get_daily_activity( - prisma_client=prisma_client, - table_name=table_name, - entity_id_field=entity_id_field, - entity_id=entity_id, - entity_metadata_field=None, - start_date=start_date, - end_date=end_date, - model=None, - api_key=None, - page=1, - page_size=PAGINATED_PAGE_SIZE, - ) - - -async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_USER_SPEND, - ENTITY_FIELD_USER, - user_id, - start_date, - end_date, - use_aggregated=True, - ) - return resp.model_dump(mode="json") - - -async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TEAM_SPEND, - ENTITY_FIELD_TEAM, - _parse_csv_ids(team_ids), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TAG_SPEND, - ENTITY_FIELD_TAG, - _parse_csv_ids(tags), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -# --------------------------------------------------------------------------- -# Summarisers — convert raw JSON to concise text the LLM can reason over -# --------------------------------------------------------------------------- - - -def _accumulate_breakdown( - results: Sequence[_DayDump], dimension: str, fields: Sequence[str] -) -> dict[str, dict[str, float]]: - """Aggregate a single breakdown dimension across days.""" - totals: Final[dict[str, dict[str, float]]] = {} - for day in results: - for key, entry in day.get("breakdown", {}).get(dimension, {}).items(): - if key not in totals: - totals[key] = {f: 0.0 for f in fields} - m = entry.get("metrics", {}) - for f in fields: - totals[key][f] += m.get(f, 0) - return totals - - -def _ranked_lines( - totals: dict[str, dict[str, float]], - fmt: Callable[[str, dict[str, float]], str], - limit: int, -) -> list[str]: - """Sort by spend descending, format each entry, and truncate.""" - return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]] - - -def _summarise_usage_data(data: _UsageDump) -> str: - meta: Final = data.get("metadata", {}) - results: Final = data.get("results", []) - - header: Final = ( - f"Total Spend: ${meta.get('total_spend', 0):.4f}\n" - f"Total Requests: {meta.get('total_api_requests', 0)}\n" - f"Successful: {meta.get('total_successful_requests', 0)} | " - f"Failed: {meta.get('total_failed_requests', 0)}\n" - f"Total Tokens: {meta.get('total_tokens', 0)}" - ) - - models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"]) - providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"]) - - model_lines: Final = _ranked_lines( - models, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)", - TOP_N_MODELS, - ) - provider_lines: Final = _ranked_lines( - providers, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)", - TOP_N_PROVIDERS, - ) - - sections = [header, ""] - sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""] - sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"]) - return "\n".join(sections) - - -def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str: - """Summarise team/tag entity usage data.""" - results: Final = data.get("results", []) - if not results: - return f"No {entity_label} usage data found for the given date range." - - totals: Final[dict[str, _EntityTotal]] = {} - for day in results: - for eid, entry in day.get("breakdown", {}).get("entities", {}).items(): - previous = totals.get(eid) - m = entry.get("metrics", {}) - totals[eid] = _EntityTotal( - alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid), - spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0), - requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0), - tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0), - ) - - lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""] - for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend): - label = d.alias if d.alias != eid else eid - lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Tool dispatch registry -# --------------------------------------------------------------------------- - -TOOL_HANDLERS: Final[dict[str, ToolHandler]] = { - "get_usage_data": ToolHandler( - fetch=_fetch_usage_data, - summarise=_summarise_usage_data, - label="global usage data", - ), - "get_team_usage_data": ToolHandler( - fetch=_fetch_team_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Team"), - label="team usage data", - ), - "get_tag_usage_data": ToolHandler( - fetch=_fetch_tag_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Tag"), - label="tag usage data", - ), -} - - -# --------------------------------------------------------------------------- -# SSE streaming -# --------------------------------------------------------------------------- - - -def _sse(event: SSEEvent) -> str: - return f"data: {json.dumps(event)}\n\n" - - -def _resolve_fetch_kwargs( - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> dict[str, str]: - """Build keyword arguments for a tool's fetch function.""" - start_date: Final = fn_args.get("start_date", "") - end_date: Final = fn_args.get("end_date", "") - if not start_date or not end_date: - raise ValueError("Missing required start_date or end_date from tool arguments") - kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date} - if fn_name == "get_usage_data": - if not is_admin: - if user_id is None: - # Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py - # should have already rejected this. If we ever reach here it means - # a future caller invoked the helper without scoping — fail loudly - # rather than issuing an unfiltered global query. - raise ValueError( - "Non-admin caller has user_id=None; refusing to issue an " - "unscoped query. Endpoint-level guard missing." - ) - kwargs["user_id"] = user_id - elif fn_args.get("user_id"): - kwargs["user_id"] = fn_args["user_id"] - elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"): - kwargs["team_ids"] = fn_args["team_ids"] - elif fn_name == "get_tag_usage_data" and fn_args.get("tags"): - kwargs["tags"] = fn_args["tags"] - return kwargs - - -async def _execute_tool_call( - handler: ToolHandler, - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> str: - """Run a single tool and return the summarised result text, or the reason the query - layer rejected its dates so the model can correct them.""" - kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin) - try: - raw_data: Final = await handler["fetch"](**kwargs) - except HTTPException as e: - reason: Final = _invalid_date_reason(e) - if reason is None: - raise - return f"Invalid date range: {reason}" - return handler["summarise"](raw_data) - - -def _invalid_date_reason(error: HTTPException) -> str | None: - """The reason in the daily activity query layer's 400 for invalid or reversed dates.""" - detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict - match detail: - case {"error": str() as reason} if error.status_code == 400: - return reason - case _: - return None - - -async def _process_tool_call( - tc: ChatCompletionMessageToolCall, - chat_messages: list[Mapping[str, object]], - user_id: str | None, - is_admin: bool, -) -> AsyncIterator[str]: - """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final = tc.function.name - fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) - - allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None - - if fn_name is None or fn_name not in allowed_names or not handler: - chat_messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": f"Tool not available: {fn_name}", - } - ) - return - - tool_event_base: Final = { - "type": "tool_call", - "tool_name": fn_name, - "tool_label": handler["label"], - "arguments": fn_args, - } - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"})) - - try: - tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin) - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"})) - except Exception as e: - verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e) - tool_result = f"Error fetching {handler['label']}. Please try again." - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"})) - - chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) - - -async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]: - """Stream the final LLM response after tool results are appended.""" - yield _sse({"type": "status", "message": "Analyzing results..."}) - - response: Final = await litellm.acompletion( - model=model, - messages=chat_messages, - stream=True, - temperature=USAGE_AI_TEMPERATURE, - ) - async for chunk in response: - delta = chunk.choices[0].delta.content - if delta: - yield _sse({"type": "chunk", "content": delta}) - - -async def stream_usage_ai_chat( - messages: list[dict[str, str]], - model: str | None = None, - user_id: str | None = None, - is_admin: bool = False, -) -> AsyncGenerator[str, None]: - """Stream SSE events: status → tool_call → chunk → done.""" - resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL - truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages - chat_messages: Final[list[Mapping[str, object]]] = [ - {"role": "system", "content": _build_system_prompt(is_admin)}, - *truncated, - ] - - try: - yield _sse({"type": "status", "message": "Thinking..."}) - tools: Final = get_tools_for_role(is_admin) - response: Final = await litellm.acompletion( - model=resolved_model, - messages=chat_messages, - tools=tools, - temperature=USAGE_AI_TEMPERATURE, - ) - choice: Final = response.choices[0] - - if not choice.message.tool_calls: - if choice.message.content: - yield _sse({"type": "chunk", "content": choice.message.content}) - yield _sse({"type": "done"}) - return - - chat_messages.append(choice.message.model_dump()) - for tc in choice.message.tool_calls: - async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): - yield event - async for event in _stream_final_response(resolved_model, chat_messages): - yield event - yield _sse({"type": "done"}) - - except Exception as e: - verbose_proxy_logger.error("AI usage chat failed: %s", e) - yield _sse( - { - "type": "error", - "message": "An internal error occurred. Please try again.", - } - ) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py deleted file mode 100644 index cff5bef05a0..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -USAGE AI CHAT ENDPOINTS - -/usage/ai/chat - Stream AI chat responses about usage data -""" - -from typing import Final, Literal - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import StreamingResponse -from pydantic import Field - -import litellm -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.sse_keepalive import ( - SSE_COMMENT_PING, - wrap_sse_stream_with_keepalive_pings, -) -from litellm.types.llms.base import LiteLLMBaseModel - -router: Final = APIRouter() - - -class ChatMessage(LiteLLMBaseModel): - role: Literal["user", "assistant"] - content: str - - -class UsageAIChatRequest(LiteLLMBaseModel): - messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)") - model: str | None = Field(default=None, description="Model to use for AI chat") - - -@router.post( - "/usage/ai/chat", - tags=["Budget & Spend Tracking"], - dependencies=[Depends(user_api_key_auth)], -) -async def usage_ai_chat( - data: UsageAIChatRequest, - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - AI chat about usage data. Streams SSE events with the AI response. - The AI agent has access to tools that query aggregated daily activity data. - """ - from litellm.proxy.management_endpoints.common_utils import ( - require_caller_user_id_for_non_admin, - user_api_key_has_admin_view, - ) - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - stream_usage_ai_chat, - ) - - is_admin: Final = user_api_key_has_admin_view(user_api_key_dict) - if is_admin: - user_id = user_api_key_dict.user_id - else: - user_id = require_caller_user_id_for_non_admin(user_api_key_dict) - messages: Final = [{"role": m.role, "content": m.content} for m in data.messages] - - return StreamingResponse( - wrap_sse_stream_with_keepalive_pings( - stream_usage_ai_chat( - messages=messages, - model=data.model, - user_id=user_id, - is_admin=is_admin, - ), - ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, - ping_chunk=SSE_COMMENT_PING, - ), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) diff --git a/litellm/proxy/management_helpers/team_list_sorting.py b/litellm/proxy/management_helpers/team_list_sorting.py new file mode 100644 index 00000000000..caddd8afc08 --- /dev/null +++ b/litellm/proxy/management_helpers/team_list_sorting.py @@ -0,0 +1,109 @@ +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from types import MappingProxyType +from typing import Final + +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter + +from litellm.proxy.utils import PrismaClient + +COMPUTED_TEAM_SORTS: Final = MappingProxyType( + { + "organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') " + 'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), ' + "NULLIF(t.organization_id, ''))", + "members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' " + "THEN jsonb_array_length(t.members_with_roles) ELSE 0 END", + "models": "cardinality(t.models)", + "keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)', + } +) +_WHERE: Final = TypeAdapter(dict[str, JsonValue]) + + +@dataclass(frozen=True, slots=True) +class UnsupportedTeamFilter: + field: str + + +def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter: + if field not in ("team_id", "team_alias", "organization_id"): + return UnsupportedTeamFilter(field) + column: Final = f't."{field}"' + match value: + case str(text): + return f"{column} = ${index}", (text,) + case {"in": list(values)}: + return f"{column} = ANY(${index}::text[])", (values,) + case {"contains": str(text), "mode": "insensitive"}: + return f"{column} ILIKE ${index}", (f"%{text}%",) + case {"startsWith": str(text)}: + return f"{column} LIKE ${index}", (f"{text}%",) + case _: + return UnsupportedTeamFilter(field) + + +def _group_tokens( + field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue] +) -> tuple[str | tuple[str, JsonValue], ...]: + children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value) + if not children: + return ("FALSE" if field == "OR" else "TRUE",) + return ( + "(", + *chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)), + ")", + ) + + +def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter: + pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal + fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying + params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order + while pending: + match pending.pop(): + case str(fragment): + fragments.append(fragment) + case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)): + pending.extend(reversed(_group_tokens(field, value))) + case (field, value): + match _field_sql(field, value, len(params) + 1): + case UnsupportedTeamFilter() as failure: + return failure + case (fragment, values): + fragments.append(fragment) + params.extend(values) + return "".join(fragments), tuple(params) + + +class _TeamId(BaseModel): + model_config = ConfigDict(frozen=True) + team_id: str + + +_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...]) + + +async def sorted_team_ids( + prisma_client: PrismaClient, + *, + where: Mapping[str, object], + sort_by: str, + descending: bool, + skip: int, + take: int, +) -> Sequence[str] | UnsupportedTeamFilter: + result: Final = _where_sql(_WHERE.validate_python(where)) + if isinstance(result, UnsupportedTeamFilter): + return result + predicate, params = result + expression: Final = COMPUTED_TEAM_SORTS[sort_by] + direction: Final = "DESC" if descending else "ASC" + sql: Final = ( + f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} ' + f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC " + f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}" + ) + rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip)) + return tuple(row.team_id for row in rows) diff --git a/litellm/proxy/management_helpers/user_list_sorting.py b/litellm/proxy/management_helpers/user_list_sorting.py new file mode 100644 index 00000000000..d3d2e323cb0 --- /dev/null +++ b/litellm/proxy/management_helpers/user_list_sorting.py @@ -0,0 +1,77 @@ +from collections.abc import Sequence +from typing import Final, Literal + +from pydantic import BaseModel, ConfigDict, TypeAdapter + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.proxy.utils import PrismaClient + + +class _UserId(BaseModel): + model_config = ConfigDict(frozen=True) + user_id: str + + +_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...]) + + +async def sorted_user_ids( + prisma_client: PrismaClient, + *, + sort_by: Literal["status", "key_count"], + descending: bool, + skip: int, + take: int, + role: str | None, + user_ids: str | None, + user_email: str | None, + team: str | None, + sso_user_ids: str | None, + organization_ids: str | None, + search: str | None, +) -> Sequence[str]: + ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip()) + sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip()) + org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip()) + expression: Final = ( + "CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END" + if sort_by == "status" + else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k ' + "WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)" + ) + direction: Final = "DESC" if descending else "ASC" + sql: Final = f""" + SELECT u.user_id FROM "LiteLLM_UserTable" u + WHERE ($1::text IS NULL OR u.user_role = $1) + AND ($2::text[] IS NULL + OR CASE WHEN cardinality($2::text[]) = 1 + THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%' + ELSE u.user_id = ANY($2::text[]) END) + AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%') + AND ($4::text IS NULL OR $4 = ANY(u.teams)) + AND ($5::boolean OR u.sso_user_id = ANY($6::text[])) + AND (cardinality($7::text[]) = 0 OR EXISTS ( + SELECT 1 FROM "LiteLLM_OrganizationMembership" m + WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[]))) + AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%' + OR u.user_email ILIKE '%' || $8 || '%') + ORDER BY {expression} {direction}, u.user_id ASC + LIMIT $9 OFFSET $10 + """ + rows: Final = _USER_IDS.validate_python( + await prisma_client.db.query_raw( + sql, + role or None, + ids if user_ids else None, + user_email, + team, + sso_user_ids is None, + sso_ids, + org_ids, + search or None, + take, + skip, + *((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()), + ) + ) + return tuple(row.user_id for row in rows) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 6ced7827846..351e4c10d87 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import ( get_blog_posts, ) from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.proxy._types import ( CommonProxyErrors, ) @@ -564,7 +565,7 @@ async def _resolve_autorouter_presets( url: str, fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]], ) -> Mapping[str, AutoRouterPresetRecord]: - if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": + if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true": return _load_bundled_autorouter_presets() try: return await fetch(url) diff --git a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py index ef2a5f88e4e..4bb17fd290e 100644 --- a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import os import re import xml.etree.ElementTree as ET from collections import Counter @@ -13,6 +14,7 @@ from pydantic import ValidationError from litellm._logging import verbose_proxy_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.types.llms.base import LiteLLMBaseModel @@ -88,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock: return _latest_release_fetch_lock +def _default_release_check_enabled() -> bool: + return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true" + + def _bucket_for(line: str) -> _Bucket | None: if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None: return None @@ -224,12 +230,16 @@ async def latest_release_info( client: Annotated[_AsyncGetClient, Depends(_default_client)], cache: Annotated[InMemoryCache, Depends(_default_cache)], fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)], + release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)], ) -> LatestReleaseInfo | None: """ Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates. Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request. """ + if not release_check_enabled: + return None result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) if isinstance(result, LatestReleaseUnavailable): verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 7ea4b3029ef..6cf274884c7 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6071,13 +6071,12 @@ class PrismaClient: elif table_name == "user": db_data = self.jsonify_object(data=data) try: - new_user_row: Final = await UserRepository(self).table.upsert( - where={"user_id": data["user_id"]}, - data={ - "create": {**db_data}, - "update": {}, # don't do anything if it already exists - }, - ) + new_user_row: Final = await UserRepository(self).upsert_user(db_data) + if new_user_row is None: + raise HTTPException( + status_code=403, + detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.", + ) except Exception as e: if ( "Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`" diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 5ed1574c6bc..1fce0717ea3 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -2,8 +2,11 @@ User repository for database operations on LiteLLM_UserTable. """ +import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, contextmanager +from contextvars import ContextVar from itertools import chain from typing import TYPE_CHECKING, Final, Protocol @@ -35,6 +38,16 @@ ORDER BY p.user_id """ _PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...]) +_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None) + + +@contextmanager +def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]: + token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None) + try: + yield + finally: + _FREE_SSO_CREATION.reset(token) class _UserDb(Protocol): @@ -42,6 +55,12 @@ class _UserDb(Protocol): def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ... +class _UserTransaction(_UserDb, Protocol): + async def execute_raw(self, query: str) -> int: ... + + async def query_raw(self, query: str) -> object: ... + + class _PrismaClientView(Protocol): @property def db(self) -> _UserDb: ... @@ -49,6 +68,16 @@ class _PrismaClientView(Protocol): @property def writer_db(self) -> _UserDb: ... + def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ... + + +async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int: + from prisma import Json + + total: Final = await table.count() + deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) + return max(0, total - deactivated) + class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): null, or true all count, so seats are counted as total users minus the deactivated ones. """ - from prisma import Json # pyright: ignore[reportUnknownVariableType] + return await _count_billable_users(self.table) - total: Final = await self.count() - deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) - return max(0, total - deactivated) + async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None": + """Return the user row, or None when a new free SSO user would exceed capacity.""" + scope: Final = _FREE_SSO_CREATION.get() + if ( + scope is None + or (scope[0] is not None and scope[0] != data["user_id"]) + or scope[1] is not asyncio.current_task() + ): + return await self.table.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) + + client: Final[_PrismaClientView] = self.prisma_client + async with client.tx() as transaction: + await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED") + await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)") + existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]}) + if existing is not None: + return existing + if await _count_billable_users(transaction.litellm_usertable) >= 5: + return None + return await transaction.litellm_usertable.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) async def create_user( self, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 95062086900..250e4df7137 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import ( from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup from litellm.litellm_core_utils.prompt_templates.common_utils import ( @@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import ( from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.common_utils import get_openai_credentials from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig from litellm.responses.litellm_completion_transformation.compaction_items import ( with_compaction_items_as_summaries, @@ -1467,6 +1469,12 @@ def responses( # get llm provider logic litellm_params: Final = GenericLiteLLMParams(**kwargs) + affinity_api_base: Final = ( + get_openai_credentials(api_base=litellm_params.api_base).api_base + if custom_llm_provider == "openai" + else litellm_params.api_base + ) + default_affinity_header: Final = get_provider_affinity_header(affinity_api_base) try: effective_extra_headers: Final = ( add_provider_affinity_header( @@ -1480,8 +1488,9 @@ def responses( "litellm_metadata": kwargs.get("litellm_metadata"), } ), + default_header=default_affinity_header, ) - if litellm_params.provider_affinity_header is not None + if litellm_params.provider_affinity_header is not None or default_affinity_header is not None else extra_headers ) except ValueError as affinity_error: diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index ebb221e98be..74cd28161a4 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { } MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { + Feature.PROVIDER_AFFINITY, Feature.SAFEGUARDS_CLASSIFIER, Feature.PRE_REQUEST_HOOK, Feature.WEBSEARCH_INTERCEPTION, diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index 691c04f2daa..b53e80805a5 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -5,10 +5,15 @@ from __future__ import annotations from typing import Final import litellm -from litellm.rust_bridge.detectors import overridden, present, streaming +from litellm.llms.opencode.common_utils import is_opencode_api_base +from litellm.rust_bridge.detectors import api_base, overridden, present, streaming from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str +def provider_affinity(request: NativeCall) -> bool: + return is_opencode_api_base(api_base(request)) + + def anthropic_workload_identity(request: NativeCall) -> bool: """Python mints an Anthropic federation token when no static credential is configured.""" from litellm.llms.anthropic.common_utils import AnthropicModelInfo diff --git a/litellm/rust_bridge/responses/detectors.py b/litellm/rust_bridge/responses/detectors.py index 392c11f48a8..763b63d6742 100644 --- a/litellm/rust_bridge/responses/detectors.py +++ b/litellm/rust_bridge/responses/detectors.py @@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay from pydantic import TypeAdapter import litellm +from litellm.llms.opencode.common_utils import is_opencode_api_base from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str @@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool: def provider_affinity(request: NativeCall) -> bool: """Python validates and fills the affinity header from the session ID.""" - return present(request, "provider_affinity_header") + return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request)) def text_format(request: NativeCall) -> bool: diff --git a/litellm/text_completions/__init__.py b/litellm/text_completions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/text_completions/streaming_iterator.py b/litellm/text_completions/streaming_iterator.py new file mode 100644 index 00000000000..ec1b96fd432 --- /dev/null +++ b/litellm/text_completions/streaming_iterator.py @@ -0,0 +1,88 @@ +from typing import Final + +from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse + + +class TextCompletionStreamWrapper: + def __init__( + self, + completion_stream, + model, + stream_options: dict | None = None, + custom_llm_provider: str | None = None, + ): + self.completion_stream = completion_stream + self.model = model + self.stream_options = stream_options + self.custom_llm_provider = custom_llm_provider + + def __iter__(self): + return self + + def __aiter__(self): + return self + + def convert_to_text_completion_object(self, chunk: ModelResponse): + try: + response: Final = TextCompletionResponse() + response["id"] = chunk.get("id", None) + response["object"] = "text_completion" + response["created"] = chunk.get("created", None) + response["model"] = chunk.get("model", None) + text_choices: Final = TextChoices() + if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices + raise Exception + + if chunk["choices"]: + delta: Final = chunk["choices"][0]["delta"] + text_choices["text"] = delta["content"] + text_choices["reasoning_content"] = delta.get("reasoning_content") + text_choices["index"] = chunk["choices"][0]["index"] + text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] + response["choices"] = [text_choices] + else: + response["choices"] = [] + + # only pass usage when stream_options["include_usage"] is True + if self.stream_options and self.stream_options.get("include_usage", False) is True: + response.usage = chunk.get("usage", None) + else: + response.usage = None + + return response + except Exception as e: + raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") + + def __next__(self): + # model_response = ModelResponse(stream=True, model=self.model) + TextCompletionResponse() + try: + for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + raise Exception + processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + return processed_chunk + raise StopIteration + except StopIteration: + raise StopIteration + except Exception as e: + from litellm.litellm_core_utils.exception_mapping_utils import exception_type + + raise exception_type( + model=self.model, + custom_llm_provider=self.custom_llm_provider or "", + original_exception=e, + completion_kwargs={}, + extra_kwargs={}, + ) + + async def __anext__(self): + try: + async for chunk in self.completion_stream: + if chunk == "None" or chunk is None: + raise Exception + processed_chunk = self.convert_to_text_completion_object(chunk=chunk) + return processed_chunk + raise StopIteration + except StopIteration: + raise StopAsyncIteration diff --git a/litellm/utils.py b/litellm/utils.py index 6e80b916dd3..2b914e0c98c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import ( ) from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases +from litellm.litellm_core_utils.offline_mode import is_offline_mode from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload from litellm.litellm_core_utils.tokenizer import strip_special_tokens from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields @@ -218,6 +219,7 @@ def _get_cached_audio_utils(): return _audio_utils_module +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper from litellm.types.llms.openai import ( AllMessageValues, AllPromptValues, @@ -262,13 +264,17 @@ from litellm.types.utils import ( SearchProviders, SelectTokenizerResponse, StreamingChoices, - TextChoices, - TextCompletionResponse, TranscriptionResponse, Usage, credential_api_key_secret_reference, is_litellm_owned_kwarg, ) +from litellm.types.utils import ( + TextChoices as _TextChoices, +) +from litellm.types.utils import ( + TextCompletionResponse as _TextCompletionResponse, +) _CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes} _BACKFILL_MODES: Final = frozenset({"chat", "responses"}) @@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError): claude_json_str = json.dumps(json_data) import importlib.metadata from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable +from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable from typing_extensions import assert_never +TextChoices: TypeAlias = _TextChoices +TextCompletionResponse: TypeAlias = _TextCompletionResponse +TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper + from litellm import utils as litellm_utils # These are lazy loaded via __getattr__ @@ -340,7 +350,8 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded functions to help mypy understand their types # These imports allow mypy to understand the types when these are accessed via __getattr__ - from litellm.litellm_core_utils.exception_mapping_utils import exception_type + exception_type: Callable[..., Exception] + from litellm.litellm_core_utils.get_litellm_params import ( get_base_model_from_litellm_call_metadata, get_litellm_params, @@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None: def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None: kind: Final = huggingface_tokenizer_kind(model) - if kind is None: + if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()): return None backend: Final = _huggingface_tokenizer_backend() key: Final = _huggingface_tokenizer_failure_key(kind, backend) @@ -7945,88 +7956,6 @@ def register_prompt_template( return litellm.custom_prompt_dict -class TextCompletionStreamWrapper: - def __init__( - self, - completion_stream, - model, - stream_options: dict | None = None, - custom_llm_provider: str | None = None, - ): - self.completion_stream = completion_stream - self.model = model - self.stream_options = stream_options - self.custom_llm_provider = custom_llm_provider - - def __iter__(self): - return self - - def __aiter__(self): - return self - - def convert_to_text_completion_object(self, chunk: ModelResponse): - try: - response: Final = TextCompletionResponse() - response["id"] = chunk.get("id", None) - response["object"] = "text_completion" - response["created"] = chunk.get("created", None) - response["model"] = chunk.get("model", None) - text_choices: Final = TextChoices() - if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices - raise Exception - - if chunk["choices"]: - delta: Final = chunk["choices"][0]["delta"] - text_choices["text"] = delta["content"] - text_choices["reasoning_content"] = delta.get("reasoning_content") - text_choices["index"] = chunk["choices"][0]["index"] - text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"] - response["choices"] = [text_choices] - else: - response["choices"] = [] - - # only pass usage when stream_options["include_usage"] is True - if self.stream_options and self.stream_options.get("include_usage", False) is True: - response["usage"] = chunk.get("usage", None) - - return response - except Exception as e: - raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}") - - def __next__(self): - # model_response = ModelResponse(stream=True, model=self.model) - TextCompletionResponse() - try: - for chunk in self.completion_stream: - if chunk == "None" or chunk is None: - raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk - raise StopIteration - except StopIteration: - raise StopIteration - except Exception as e: - exception_type: Final = litellm_utils.exception_type - raise exception_type( - model=self.model, - custom_llm_provider=self.custom_llm_provider or "", - original_exception=e, - completion_kwargs={}, - extra_kwargs={}, - ) - - async def __anext__(self): - try: - async for chunk in self.completion_stream: - if chunk == "None" or chunk is None: - raise Exception - processed_chunk = self.convert_to_text_completion_object(chunk=chunk) - return processed_chunk - raise StopIteration - except StopIteration: - raise StopAsyncIteration - - def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream: return ModelResponseStream( id=model_response.id, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 90a0763f786..3edb30b4218 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -48499,6 +48723,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -52906,12 +53141,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53163,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -73330,6 +73569,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/osv-scanner.toml b/osv-scanner.toml index bb802137f6e..33e2f614edc 100644 --- a/osv-scanner.toml +++ b/osv-scanner.toml @@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v" ignoreUntil = 2026-11-01 reason = "diskcache has no fixed release published; remove this entry once one exists" +[[IgnoredVulns]] +id = "GHSA-v4x8-3wr5-3wq8" +ignoreUntil = 2026-11-20 +reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships" + [[IgnoredVulns]] id = "GHSA-vfj7-8cjw-p6xm" ignoreUntil = 2026-11-03 diff --git a/pyproject.toml b/pyproject.toml index b43f0b254fd..817c50df90b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -137,7 +137,7 @@ semantic-router = [ "semantic-router>=0.1.15,<1.0; python_version < '3.14'", "aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'", ] -mlflow = ["mlflow>=3.11.1,<4.0"] +mlflow = ["mlflow>=3.17.2,<4.0"] grpc = [ # First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff "grpcio==1.84.1", @@ -177,16 +177,16 @@ proxy-runtime = [ "grpcio==1.84.1", "prometheus-client>=0.20.0,<1.0", "langfuse>=4.7,<5.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "ddtrace>=4.8.2,<5.0", "sentry-sdk>=2.21.0,<3.0", "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.16.1,<7.0", + "pypdf>=6.21.2,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -223,10 +223,10 @@ dev = [ "types-PyYAML==6.0.12.20250915", "botocore-stubs==1.43.14", "types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "langfuse>=4.7,<5.0", "fastapi-offline==1.7.6", "fakeredis==2.34.1", @@ -255,10 +255,10 @@ proxy-dev = [ "hypercorn==0.17.3", "prometheus-client==0.20.0", "sentry-sdk==2.21.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", @@ -280,7 +280,7 @@ ci = [ "langchain==1.3.9", "lunary==1.4.36; python_version == '3.10'", "lunary==1.4.37; python_version >= '3.11'", - "logfire==4.6.0", + "logfire==5.2.0", "traceloop-sdk==0.34.0", "detect-secrets==1.5.0", "PyGithub==2.8.1", @@ -359,7 +359,7 @@ constraint-dependencies = [ "packaging>=24.0", "soupsieve>=2.8.4", "httplib2>=0.32.0", - "setuptools>=83.0.0", + "setuptools>=84.1.0", ] override-dependencies = [ # a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0. @@ -370,10 +370,23 @@ override-dependencies = [ default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" + +[tool.uv.exclude-newer-package] # TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window +certifi = "2026-10-20T03:00:00Z" # TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window +grpcio = "2026-10-22T15:00:00Z" # TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window -exclude-newer-package = { certifi = "2026-10-20T03:00:00Z", grpcio = "2026-10-22T15:00:00Z", boto3 = "2026-10-21T19:31:00Z", botocore = "2026-10-21T19:31:00Z" } +boto3 = "2026-10-21T19:31:00Z" +botocore = "2026-10-21T19:31:00Z" +# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window +brotli = "2026-10-22T14:01:00Z" +# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window +mlflow = "2026-10-21T03:00:00Z" +mlflow-skinny = "2026-10-21T03:00:00Z" +mlflow-tracing = "2026-10-21T03:00:00Z" +# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window +pypdf = "2026-10-21T17:00:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/tests/integration/compatibility/test_status_code_followups_usage_chat.py b/tests/integration/compatibility/test_status_code_followups_usage_chat.py deleted file mode 100644 index de9ec16cf30..00000000000 --- a/tests/integration/compatibility/test_status_code_followups_usage_chat.py +++ /dev/null @@ -1,125 +0,0 @@ -"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check -the daily activity routes answer 400 with, instead of a generic fetch error.""" - -from __future__ import annotations - -import json -import uuid -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Final - -import pytest -from integration._support.client import JSON_OBJECT, Gateway, object_value -from integration._support.upstream import delete_scenario, register_scenario -from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway -from pydantic import JsonValue - -pytestmark: Final = pytest.mark.timeout(180) - -_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data") - - -@dataclass(frozen=True, slots=True) -class _UsageChat: - gateway: Gateway - identity: str - - -@pytest.fixture(scope="module") -def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]: - """A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream.""" - identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}" - handle: Final = register_scenario(identity, json_response({})) - try: - with owned_gateway( - tmp_path_factory.mktemp("audit-usage-chat"), - {"model_list": [], "router_settings": {"num_retries": 0}}, - environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity}, - ) as owned: - yield _UsageChat(owned, identity) - finally: - delete_scenario(handle) - - -def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]: - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - return { - "id": "chatcmpl-usage", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}} - for tool in _TOOLS - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - - -def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]: - """Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up - model call carried, keyed by tool, and the tool statuses the SSE stream reported.""" - register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date))) - Upstream(usage_chat.gateway.upstream_url).reset() - with usage_chat.gateway.client.stream( - "POST", - "/usage/ai/chat", - json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]}, - headers={"Authorization": f"Bearer {usage_chat.gateway.key}"}, - timeout=60, - ) as response: - assert response.status_code == 200 - events: Final = tuple( - object_value(JSON_OBJECT.validate_json(line.removeprefix("data: "))) - for line in response.iter_lines() - if line.startswith("data: ") - ) - calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity) - assert len(calls) == 2, calls - messages: Final = object_value(calls[1]["body"]).get("messages") - assert isinstance(messages, list), calls[1] - tool_messages: Final = { - str(message["tool_call_id"]).removeprefix("call-"): str(message["content"]) - for message in messages - if isinstance(message, dict) and message.get("role") == "tool" - } - statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call") - return tool_messages, statuses - - -_REVERSED: Final = "Invalid date range: end_date must be on or after start_date" -_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates" -_NO_TEAM_DATA: Final = "No Team usage data found for the given date range." -_NO_TAG_DATA: Final = "No Tag usage data found for the given date range." - - -@pytest.mark.parametrize( - ("start_date", "end_date", "team_message", "tag_message"), - ( - pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"), - pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"), - pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"), - pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"), - ), -) -def test_usage_chat_tools_answer_with_the_query_layers_date_check( - usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str -) -> None: - """The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query - takes these dates without that check, still answers with its usage summary.""" - tool_messages, statuses = _tool_results(usage_chat, start_date, end_date) - assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages - assert tool_messages["get_team_usage_data"] == team_message, tool_messages - assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages - assert statuses == ("running", "complete") * len(_TOOLS), statuses diff --git a/tests/integration/management/test_budget_list_sorting.py b/tests/integration/management/test_budget_list_sorting.py new file mode 100644 index 00000000000..aae0bd6e274 --- /dev/null +++ b/tests/integration/management/test_budget_list_sorting.py @@ -0,0 +1,94 @@ +from datetime import datetime, timezone +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from +from tests.integration._support.client import Gateway, Scenario + + +@pytest.mark.parametrize("descending", (False, True)) +def test_budget_reset_sort_orders_durations_before_pagination( + descending: bool, + gateway: Gateway, + scenario: Scenario, +) -> None: + prefix: Final = f"sort-{uuid4().hex}" + durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "5d", "𝟝d", "5d", None) + ids: Final = tuple( + scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10) + for index, duration in enumerate(durations) + ) + now: Final = datetime.now(timezone.utc) + expected: Final = tuple( + budget_id + for _, budget_id in sorted( + zip(durations, ids, strict=True), + key=lambda item: ( + item[0] is None, + duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0, + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": "-budget_duration" if descending else "budget_duration", + "page_size": "3", + "page": str(page), + }, + ) + for page in range(1, -(-len(ids) // 3) + 1) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected + filtered: Final = gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": "budget_duration", + "filter[budget_duration][in]": "7d,30d", + }, + ) + assert filtered.status_code == 200, filtered.text + assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"] + + +@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")) +@pytest.mark.parametrize("descending", (False, True)) +def test_budget_limits_sort_unlimited_as_infinity( + field: str, descending: bool, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-unlimited-{uuid4().hex}" + values: Final = (20, None, 0, 3) + ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values)) + expected: Final = tuple( + identity + for value, identity in sorted( + zip(values, ids, strict=True), + key=lambda item: float("inf") if item[0] is None else item[0], + reverse=descending, + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/management/v1/budgets", + params={ + "q": prefix, + "sort": ("-" if descending else "") + field, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected diff --git a/tests/integration/management/test_key_list.py b/tests/integration/management/test_key_list.py index b70a3782eda..4634314817b 100644 --- a/tests/integration/management/test_key_list.py +++ b/tests/integration/management/test_key_list.py @@ -1,11 +1,16 @@ -from typing import FrozenSet +from datetime import datetime, timedelta, timezone +from itertools import chain +from typing import Final, FrozenSet import pytest from litellm.proxy.utils import hash_token -from tests.integration._support.actors import TEAM_ALPHA, Actor +from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources from tests.integration._support.actors import create_scratch_key +from tests.integration._support.client import Gateway, Scenario +from litellm.repositories.table_repositories import DeletedVerificationTokenRepository +from litellm.repositories.verification_token_repository import VerificationTokenRepository pytestmark = pytest.mark.asyncio(loop_scope="session") @@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor }, ) assert resp.status_code == 403, resp.text + + +@pytest.mark.parametrize( + "field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit") +) +@pytest.mark.parametrize("direction", ("asc", "desc")) +@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted")) +async def test_key_list_sorts_across_pages_with_nulls_last( + field: str, + direction: str, + deleted: bool, + gateway: Gateway, + scenario: Scenario, + db: ManagementDatabase, + resources: ScratchResources, +) -> None: + team_admin: Final = scenario.user(user_role="internal_user") + owner: Final = scenario.user(user_role="internal_user") + team_id: Final = scenario.team( + members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}] + ) + team_admin_key: Final = scenario.key(user_id=team_admin) + table: Final = ( + DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table + ) + values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None) + tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values))) + rows: Final = [ + { + "token": token, + "key_alias": resources.prefix, + "team_id": team_id, + "user_id": owner, + field: ( + datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value) + if field in ("expires", "last_active", "budget_reset_at") and value is not None + else value + ), + } + for token, value in zip(tokens, values, strict=True) + ] + await table.create_many(rows) + try: + valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None) + expected: Final = [ + token + for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1])) + ] + [token for value, token in zip(values, tokens, strict=True) if value is None] + for caller in (gateway.key, team_admin_key): + pages: Final = [ + gateway.client.get( + "/key/list", + headers={"Authorization": f"Bearer {caller}"}, + params={ + "key_alias": resources.prefix, + "team_id": team_id, + "include_team_keys": "true", + "sort_by": field, + "sort_order": direction, + "size": "2", + "page": str(page), + **({"status": "deleted"} if deleted else {}), + }, + ) + for page in (1, 2, 3) + ] + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total_count"] == len(tokens) for page in pages) + assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected + finally: + await table.delete_many(where={"token": {"in": tokens}}) diff --git a/tests/integration/management/test_sso_sign_in.py b/tests/integration/management/test_sso_sign_in.py index c0c03571a13..d0b6e2579c8 100644 --- a/tests/integration/management/test_sso_sign_in.py +++ b/tests/integration/management/test_sso_sign_in.py @@ -3,7 +3,9 @@ import contextlib import json import uuid from collections.abc import Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from functools import partial from pathlib import Path from typing import Final from urllib.parse import parse_qs, urlencode, urlparse @@ -15,7 +17,7 @@ from pydantic import JsonValue from tests.integration._support.client import Gateway, gateway_from_environment from tests.integration._support.database import read_rows -from tests.integration._support.process import owned_proxy +from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy from tests.integration._support.wire import Reply, Request, wire_server pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] @@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role( "metadata": {"auth_provider": IDP_PROVIDER}, } ] + + +def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value) + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + with _sso_gateway(tmp_path, {}) as gateway: + for index in range(4): + signed_in: Final = _sign_in(gateway, f"free-sso-{index}") + assert signed_in.status_code == 303, signed_in.text + + contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8)) + with ThreadPoolExecutor(max_workers=len(contenders)) as pool: + responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders)) + + assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403] + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}] + for subject, response in zip(contenders, responses): + if response.status_code == 403: + assert "requires a LiteLLM Enterprise license" in response.text + assert not response.cookies + assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == [] + + existing: Final = _sign_in(gateway, "free-sso-0") + assert existing.status_code == 303, existing.text + + admin_created: Final = gateway.request( + "POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False} + ) + assert admin_created.status_code == 200, admin_created.text + above_capacity: Final = _sign_in(gateway, "free-sso-0") + refused: Final = _sign_in(gateway, "free-sso-extra") + assert (above_capacity.status_code, refused.status_code) == (303, 403) + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}] diff --git a/tests/integration/management/test_team_list_sorting.py b/tests/integration/management/test_team_list_sorting.py new file mode 100644 index 00000000000..c09e51f0dfe --- /dev/null +++ b/tests/integration/management/test_team_list_sorting.py @@ -0,0 +1,244 @@ +import json +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from tests.integration._support.client import Gateway, Scenario +from tests.integration._support.database import write_rows + + +@pytest.mark.parametrize( + "field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count") +) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_team_list_sorts_across_pages_and_respects_membership( + field: str, + direction: str, + gateway: Gateway, + scenario: Scenario, +) -> None: + prefix: Final = f"sort-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + caller: Final = scenario.key(user_id=owner) + values: Final = (20, 3, 3, 100, 0) + teams: Final = tuple( + scenario.team( + team_id=f"{prefix}-{index}", + team_alias=f"{prefix}-{index}", + members_with_roles=[{"user_id": owner, "role": "admin"}], + ) + for index in range(len(values)) + ) + outsider: Final = scenario.team(team_alias=f"{prefix}-outside") + for team_id, value in zip(teams, values, strict=True): + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, ' + "tpm_limit = %s::bigint, rpm_limit = %s::bigint, " + "updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', " + "members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) " + "WHERE team_id = %s", + ( + str(value), + str(value), + str(value), + str(value), + str(value), + json.dumps( + [{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)] + ), + str(value), + team_id, + ), + ) + write_rows( + 'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) ' + "SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n", + (team_id, team_id, str(value)), + ) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s', + (teams[-1],), + ) + try: + expected: Final = tuple( + team_id + for value, team_id in sorted( + zip(values, teams, strict=True), + key=lambda item: ( + (item[0] != 0 if direction == "desc" else item[0] == 0) + if field in ("max_budget", "tpm_limit", "rpm_limit") + else False, + item[0] * (1 if direction == "asc" else -1), + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "search": prefix, + "user_id": owner, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2, 3) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) + assert returned == expected + assert outsider not in returned + assert all(page.json()["total"] == len(teams) for page in pages) + finally: + for team_id in teams: + write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,)) + + +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_team_list_sorts_organization_names_and_missing_values( + direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-{uuid4().hex}" + budget: Final = scenario.budget(max_budget=10) + organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", "")) + teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4)) + try: + for (organization_id, alias), team_id in zip(organizations, teams): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (organization_id, alias, budget), + ) + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s', + (organization_id, team_id), + ) + ascending: Final = (teams[0], teams[2], teams[1]) + ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending)) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + params={ + "search": prefix, + "sort_by": "organization_alias", + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1]) + finally: + for team_id in teams: + write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,)) + for organization_id, _ in organizations: + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,)) + + +@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_computed_team_sorts_preserve_org_and_membership_union( + field: str, direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-scope-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + budget: Final = scenario.budget(max_budget=10) + orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside") + teams: Final = tuple( + scenario.team( + team_id=f"{prefix}-{i}", + team_alias=f"{prefix}-{i}", + members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [], + ) + for i in range(4) + ) + try: + for org, alias in zip(orgs, ("alpha", "zulu"), strict=True): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (org, alias, budget), + ) + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'org_admin')", + (owner, orgs[0]), + ) + for index, team in enumerate(teams): + write_rows( + 'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, ' + "models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s", + ( + orgs[0] if index < 2 else orgs[1], + json.dumps( + [ + {"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} + for i in range(index + 1) + ] + ), + str(index + 1), + team, + ), + ) + write_rows( + 'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) ' + "SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n", + (team, team, str(index + 1)), + ) + caller: Final = scenario.key(user_id=owner) + values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3) + expected: Final = tuple( + team + for value, team in sorted( + zip(values, teams[:3], strict=True), + key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "user_id": owner, + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == 3 for page in pages) + assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected + denied: Final = gateway.request( + "GET", + "/v2/team/list", + key=caller, + params={ + "organization_id": orgs[1], + "sort_by": field, + }, + ) + assert denied.status_code == 403, denied.text + finally: + for team in teams: + write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,)) + write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,)) + for org in orgs: + write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,)) + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,)) diff --git a/tests/integration/management/test_user_list_sorting.py b/tests/integration/management/test_user_list_sorting.py new file mode 100644 index 00000000000..f5578140e1c --- /dev/null +++ b/tests/integration/management/test_user_list_sorting.py @@ -0,0 +1,179 @@ +from itertools import chain +from typing import Final +from uuid import uuid4 + +import pytest + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from tests.integration._support.client import Gateway, Scenario +from tests.integration._support.database import write_rows + + +@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None: + prefix: Final = f"sort-{uuid4().hex}" + ids: Final = tuple( + scenario.user( + user_id=f"{prefix}-{index}", + user_role="internal_user", + user_alias=f"Alias {value if index != 2 else 4:03}", + user_email=f"{prefix}-{index}@example.com", + max_budget=value, + metadata={"scim_active": value != 3}, + ) + for index, value in enumerate((20, 3, 3, 100, 0)) + ) + for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True): + write_rows( + 'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, ' + "updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s", + (f"{prefix}-{value:03}", str(value), user_id), + ) if value != 3 or user_id == ids[1] else write_rows( + "UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s", + (f"{prefix}-004", user_id), + ) + for index in range(value if value < 4 else 1): + scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}") + write_rows( + 'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s', + (ids[-1],), + ) + scenario.key(user_id=ids[-1]) + write_rows( + 'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1]) + ) + values: Final = ( + (0, 1, 1, 0, 0) + if field == "status" + else (1, 3, 3, 1, 0) + if field == "key_count" + else (20, 3, 4, 100, None) + if field == "sso_user_id" + else (20, 3, 4, 100, 0) + if field == "user_alias" + else (20, 3, 3, 100, None) + ) + expected: Final = tuple( + user_id + for _, user_id in sorted( + ((value, user_id) for value, user_id in zip(values, ids, strict=True)), + key=lambda item: ( + (item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None), + (item[0] or 0) * (1 if direction == "asc" else -1), + item[1], + ), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/user/list", + params={ + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2, 3) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == len(ids) for page in pages) + assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected + + +@pytest.mark.parametrize("field", ("status", "key_count")) +def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None: + team: Final = scenario.team() + member: Final = scenario.member(team) + other: Final = scenario.user(user_role="internal_user") + for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}): + baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"}) + response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field}) + assert response["total"] == baseline["total"] + assert sorted(user["user_id"] for user in response["users"]) == sorted( + user["user_id"] for user in baseline["users"] + ) + ordinary_key: Final = scenario.key(user_id=other) + denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field}) + assert denied.status_code == 403, denied.text + + +@pytest.mark.parametrize("field", ("status", "key_count")) +@pytest.mark.parametrize("direction", ("asc", "desc")) +def test_computed_user_sorts_preserve_org_admin_scope( + field: str, direction: str, gateway: Gateway, scenario: Scenario +) -> None: + prefix: Final = f"sort-org-users-{uuid4().hex}" + owner: Final = scenario.user(user_role="internal_user") + members: Final = tuple( + scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1}) + for index in range(4) + ) + budget: Final = scenario.budget(max_budget=10) + orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside") + try: + for org in orgs: + write_rows( + 'INSERT INTO "LiteLLM_OrganizationTable" ' + "(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) " + "VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())", + (org, org, budget), + ) + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'org_admin')", + (owner, orgs[0]), + ) + for index, member in enumerate(members): + write_rows( + 'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) ' + "VALUES (%s, %s, 'internal_user')", + (member, orgs[0] if index < 3 else orgs[1]), + ) + for _ in range(index): + scenario.key(user_id=member) + caller: Final = scenario.key(user_id=owner) + values: Final = (0, 1, 0) if field == "status" else (0, 1, 2) + expected: Final = tuple( + member + for value, member in sorted( + zip(values, members[:3], strict=True), + key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]), + ) + ) + pages: Final = tuple( + gateway.request( + "GET", + "/user/list", + key=caller, + params={ + "search": prefix, + "sort_by": field, + "sort_order": direction, + "page_size": "2", + "page": str(page), + }, + ) + for page in (1, 2) + ) + assert all(page.status_code == 200 for page in pages), [page.text for page in pages] + assert all(page.json()["total"] == 3 for page in pages) + returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages)) + assert returned == expected + denied: Final = gateway.request( + "GET", + "/user/list", + key=caller, + params={ + "organization_ids": orgs[1], + "sort_by": field, + }, + ) + assert denied.status_code == 403, denied.text + finally: + for org in orgs: + write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,)) + write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,)) diff --git a/tests/integration/security/test_offline_mode.py b/tests/integration/security/test_offline_mode.py new file mode 100644 index 00000000000..11012eb14bc --- /dev/null +++ b/tests/integration/security/test_offline_mode.py @@ -0,0 +1,180 @@ +from __future__ import annotations + +import json +import uuid +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, object_value +from integration._support.forward_proxy import refusing_forward_proxy +from integration._support.process import owned_proxy +from pydantic import JsonValue + +_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm" +_CONTROL_HOST: Final = "provider.offline-mode.example" +_REMOTE_URLS: Final = { + "LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json", + "LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json", + "LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json", + "LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml", +} +_SELF_INITIATED_HOSTS: Final = frozenset( + { + "cost-map.offline-mode.example:443", + "beta-headers.offline-mode.example:443", + "autorouter.offline-mode.example:443", + "blog.offline-mode.example:443", + "raw.githubusercontent.com:443", + "docs.litellm.ai:443", + "api.pwnedpasswords.com:443", + "huggingface.co:443", + } +) +_PER_FEATURE_FLAGS: Final = ( + "LITELLM_OFFLINE_MODE", + "LITELLM_LOCAL_MODEL_COST_MAP", + "LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", + "LITELLM_LOCAL_AUTOROUTER_PRESETS", + "LITELLM_LOCAL_POLICY_TEMPLATES", + "LITELLM_LOCAL_BLOG_POSTS", + "LITELLM_DISABLE_LATEST_RELEASE_CHECK", + "LITELLM_LICENSE", +) + + +def _bundled(name: str) -> JsonValue: + return json.loads((_PACKAGE / name).read_text(encoding="utf-8")) + + +def _write_config(directory: Path) -> Path: + config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "offline-mode-control", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"https://{_CONTROL_HOST}/v1", + "api_key": "synthetic-provider-key", + }, + }, + { + "model_name": "offline-mode-kimi", + "litellm_params": { + "model": "bedrock/us.moonshotai.kimi-k3", + "aws_region_name": "us-east-1", + "aws_access_key_id": "synthetic-access-key", + "aws_secret_access_key": "synthetic-secret-key", + }, + }, + ], + "router_settings": {"num_retries": 0}, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return config + + +def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue: + response: Final = proxy.request(method, path, body) + assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}" + return json.loads(response.content) + + +@pytest.mark.timeout(300) +@pytest.mark.parametrize( + ("offline_mode", "expected_hosts", "reload_status"), + [ + pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"), + pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"), + ], +) +def test_offline_mode_stops_every_self_initiated_request( + gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int +) -> None: + with refusing_forward_proxy() as egress: + environment: Final = { + "HTTPS_PROXY": egress.url, + "HTTP_PROXY": egress.url, + "ALL_PROXY": egress.url, + "NO_PROXY": "127.0.0.1,localhost", + "AIOHTTP_TRUST_ENV": "True", + **_REMOTE_URLS, + **({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}), + } + with owned_proxy( + gateway, + tmp_path, + environment, + config=_write_config(tmp_path), + remove_environment=_PER_FEATURE_FLAGS, + workers=2, + ) as isolated: + source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source")) + reloaded: Final = isolated.request("POST", "/reload/model_cost_map") + beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers")) + release: Final = _json(isolated, "GET", "/get/latest_release_info") + presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets")) + templates: Final = _json(isolated, "GET", "/policy/templates") + blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts")) + user: Final = object_value( + _json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"}) + ) + password_set: Final = object_value( + _json( + isolated, + "POST", + "/user/update", + {"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"}, + ) + ) + tokens: Final = object_value( + _json( + isolated, + "POST", + "/utils/token_counter", + {"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]}, + ) + ) + kimi_tokens: Final = object_value( + _json( + isolated, + "POST", + "/utils/token_counter", + {"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]}, + ) + ) + control: Final = isolated.request( + "POST", + "/v1/chat/completions", + {"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]}, + ) + reached: Final = frozenset(egress.targets()) + + assert reached == expected_hosts + assert control.status_code != 200, control.text + assert source["source"] == "local", source + assert source["is_env_forced"] is (offline_mode is not None), source + assert reloaded.status_code == reload_status, reloaded.text + assert beta_headers["status"] == "success", beta_headers + assert release is None + assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json"))) + assert templates == _bundled("policy_templates_backup.json") + assert [object_value(post)["url"] for post in _list(blog["posts"])] == [ + object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"]) + ] + assert password_set["user_id"] == user["user_id"], password_set + assert tokens["tokenizer_type"] == "openai_tokenizer", tokens + assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens + + +def _list(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value diff --git a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py index 780ee43fb3f..0c18f2a4b6f 100644 --- a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py +++ b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py @@ -99,16 +99,9 @@ class _Activity(BaseModel): results: tuple[_Day, ...] -class _UpstreamMessage(BaseModel): - model_config = ConfigDict(frozen=True) - role: str | None = None - - class _UpstreamRequest(BaseModel): model_config = ConfigDict(frozen=True) stream: bool | None = None - tools: tuple[JsonValue, ...] | None = None - messages: tuple[_UpstreamMessage, ...] = () @dataclass(frozen=True, slots=True) @@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]: return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n") -def _usage_tool_call() -> Mapping[str, JsonValue]: - arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)}) - return { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}} - ], - } - - def _response_object() -> bytes: return json.dumps( { @@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply: body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}") if body.stream: return Reply(content_type="text/event-stream", chunks=_stream_frames()) - roles: Final = frozenset(message.role for message in body.messages) - if body.tools and "tool" not in roles: - return Reply(body=_completion(_usage_tool_call(), "tool_calls")) return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop")) @@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None: ) -def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]: - return tuple( - json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ") - ) - - @dataclass(frozen=True, slots=True) class Tenant: label: str @@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]: @pytest.mark.timeout(360) -def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( +def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( gateway: Gateway, tmp_path: Path ) -> None: with _rig(gateway, tmp_path) as (proxy, database_url): - spender: Final = _spender(proxy, database_url, "ai-chat") + spender: Final = _spender(proxy, database_url, "repeated-timeout") with _pinned(proxy) as pinned: with _locked_spend_logs(database_url): - with _recording(database_url) as during_chat: - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - assert len(_lookups(during_chat)) == 1 + with _recording(database_url) as during_timeout: + timed_out: Final = _read(pinned, spender.digest) + assert timed_out.status_code == 200, timed_out.text + assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed + assert _aliases(timed_out, spender.digest) == (None,), timed_out.text + assert len(_lookups(during_timeout)) == 1 cached_miss: Final = _read(pinned, spender.digest) assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text @@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked assert gateway_activity == 200, gateway_activity chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}") assert object_value(chat["usage"]) == dict(USAGE), chat - - -@pytest.mark.timeout(300) -def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery( - gateway: Gateway, tmp_path: Path -) -> None: - with ( - scratch_database() as database_url, - database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url), - wire_server(_respond) as wire, - _proxy(gateway, tmp_path, relayed_url, wire.url) as proxy, - ): - spender: Final = _spender(proxy, database_url, "dropped-ai-chat") - with _pinned(proxy) as pinned: - relay.arm() - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert relay.tripped.is_set(), chat.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - recovered: Final = _named_once_reconnected(pinned, spender) - assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text diff --git a/tests/integration/spend/test_daily_activity_key_owner_traffic.py b/tests/integration/spend/test_daily_activity_key_owner_traffic.py index b8f113aca49..5284ad631de 100644 --- a/tests/integration/spend/test_daily_activity_key_owner_traffic.py +++ b/tests/integration/spend/test_daily_activity_key_owner_traffic.py @@ -6,7 +6,6 @@ from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from hashlib import sha256 -from pathlib import Path from queue import SimpleQueue from typing import Final @@ -31,8 +30,7 @@ from integration._support.daily_activity import ( user_row, user_with_an_email, ) -from integration._support.database import read_rows, scratch_database -from integration._support.process import owned_proxy +from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue @@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons REQUESTS_OF_A_BURST: Final = 21 READS_DURING_A_BURST: Final = 30 TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models") -TOOL_CALL: Final = "call_integration_usage" -ANSWER: Final = "One request cost $0.25" -SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join( - ( - "Total Spend: $0.2500", - "Total Requests: 1", - "Successful: 1 | Failed: 0", - "Total Tokens: 15", - "", - "Top Models by Spend:", - " - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)", - "", - "Top Providers by Spend:", - " - openai: $0.2500 (1 reqs)", - ) -) def _chat_completion() -> dict[str, JsonValue]: @@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply: return Reply(body=json.dumps(body).encode()) -def _usage_tool_call() -> dict[str, JsonValue]: - call: Final[dict[str, JsonValue]] = { - "id": TOOL_CALL, - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": json.dumps({"start_date": DAY, "end_date": DAY}), - }, - } - return { - "id": f"chatcmpl-{uuid.uuid4().hex}", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": None, "tool_calls": [call]}, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - } - - -def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes: - chunk: Final = { - "id": "chatcmpl-integration-usage", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], - } - return f"data: {json.dumps(chunk)}\n\n".encode() - - -def _usage_analyst(request: Request) -> Reply: - if json.loads(request.body).get("stream"): - return Reply( - chunks=( - _streamed_chunk({"role": "assistant", "content": ANSWER}, None), - _streamed_chunk({}, "stop"), - b"data: [DONE]\n\n", - ), - content_type="text/event-stream", - ) - return Reply(body=json.dumps(_usage_tool_call()).encode()) - - def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]: return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY) @@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session( ) -@pytest.mark.timeout(300) -def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner( - gateway: Gateway, tmp_path: Path -) -> None: - question: Final = f"what did we spend {uuid.uuid4().hex}" - owner: Final = f"integration-{uuid.uuid4().hex}" - ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}" - with ( - scratch_database() as scratch_url, - wire_server(_usage_analyst) as wire, - owned_proxy( - gateway, - tmp_path, - { - "DATABASE_URL": scratch_url, - "OPENAI_API_BASE": f"{wire.url}/v1", - "OPENAI_BASE_URL": f"{wire.url}/v1", - "OPENAI_API_KEY": "integration-provider-key", - }, - remove_environment=("DATABASE_URL_READ_REPLICA",), - ) as candidate, - ): - candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False}) - with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url): - answer: Final = candidate.request( - "POST", - "/usage/ai/chat", - {"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"}, - ) - assert answer.status_code == 200, answer.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": DAY, "end_date": DAY}, - } - events: Final = [ - json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ") - ] - assert events == [ - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": ANSWER}, - {"type": "done"}, - ], answer.text - asked, analysed = wire.drain() - assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"] - assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question} - assert json.loads(analysed.body)["messages"][-1] == { - "role": "tool", - "tool_call_id": TOOL_CALL, - "content": SUMMARY_OF_ONE_SEEDED_ROW, - } - assert owner not in analysed.body.decode() - assert ownerless_key not in analysed.body.decode() - - @pytest.mark.timeout(300) def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None: released: Final = threading.Event() diff --git a/tests/unit/integrations/otel/test_otel_v2_components.py b/tests/unit/integrations/otel/test_otel_v2_components.py index 3f4389765d3..8af13fd6e7e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_components.py +++ b/tests/unit/integrations/otel/test_otel_v2_components.py @@ -14,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer import pytest import requests +from requests_mock import Adapter pytest.importorskip("opentelemetry") @@ -22,16 +23,18 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa: ) from opentelemetry import baggage # noqa: E402 from opentelemetry.context import attach, detach # noqa: E402 -from opentelemetry._logs.severity import SeverityNumber # noqa: E402 -from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402 -from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402 +from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above +from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 -from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above from opentelemetry.sdk.trace.export import ( # noqa: E402 BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor, + SpanExportResult, ) from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, @@ -46,6 +49,7 @@ import litellm # noqa: E402 from tests.unit.integrations.conftest import TlsSink # noqa: E402 from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402 from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402 @@ -733,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict] class Collector(BaseHTTPRequestHandler): def do_POST(self): body = self.rfile.read(int(self.headers["Content-Length"])) - received.append({"path": self.path, "headers": dict(self.headers), "body": body}) + received.append( + {"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body} + ) self.send_response(200) self.end_headers() @@ -793,6 +799,95 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint(): assert child["parentSpanId"] == format(parent_id, "016x") +@pytest.mark.parametrize( + ("statuses", "expected"), + [ + ((200,), SpanExportResult.SUCCESS), + ((400,), SpanExportResult.FAILURE), + ((503, 200), SpanExportResult.SUCCESS), + ], +) +def test_http_json_exporter_preserves_http_failure_and_retry_results( + statuses: tuple[int, ...], expected: SpanExportResult +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri( + "POST", + endpoint, + [{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses], + ) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter( + endpoint=endpoint, + headers={ + "CONTENT-TYPE": "application/x-protobuf", + "x-collector-token": "secret", + }, + session=session, + ) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == len(statuses) + for request in adapter.request_history: + assert request.headers["Content-Type"] == "application/json" + assert request.headers["x-collector-token"] == "secret" + assert request.json() == {"resourceSpans": []} + + +@pytest.mark.parametrize( + ("size_limit", "expected", "request_count"), + [ + (0, SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0), + ], +) +def test_http_json_exporter_checks_json_size_before_sending( + size_limit: int, expected: SpanExportResult, request_count: int +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == request_count + + +def test_http_json_exporter_rejects_export_after_shutdown() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + exporter.shutdown() + assert exporter.export(()) is SpanExportResult.FAILURE + assert adapter.request_history == [] + + +def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + try: + assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE + finally: + exporter.shutdown() + assert adapter.request_history == [] + + def test_http_protobuf_exporter_still_posts_protobuf(): (_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector( "http/protobuf" @@ -1630,9 +1725,13 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey severity_number=SeverityNumber.INFO, body="v2-tls-test", ) - log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("v2-tls-test"), + ) result = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() diff --git a/tests/unit/integrations/otel/test_otel_v2_mount.py b/tests/unit/integrations/otel/test_otel_v2_mount.py index 3a27c3ec4e4..61f8b9ae0d9 100644 --- a/tests/unit/integrations/otel/test_otel_v2_mount.py +++ b/tests/unit/integrations/otel/test_otel_v2_mount.py @@ -382,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch): assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n" for path, response in responses.items(): context = server_spans[path].context - assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-01"] + assert response.headers.get_list("traceresponse") == [ + f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" + ] @pytest.mark.parametrize("path", ["/relayed", "/relayed/health"]) @@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo headers = contextvars.Context().run(trace_response_headers, span) - assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-01" + assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" diff --git a/tests/unit/integrations/otel/test_otel_v2_presets.py b/tests/unit/integrations/otel/test_otel_v2_presets.py index 64366626784..570c27a24e1 100644 --- a/tests/unit/integrations/otel/test_otel_v2_presets.py +++ b/tests/unit/integrations/otel/test_otel_v2_presets.py @@ -2,13 +2,19 @@ event loop: the preset does no network I/O, and a custom exporter mints the JWT lazily on its first export (in the BatchSpanProcessor worker thread).""" +import json + import httpx import pytest +import requests_mock +import respx +from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.otel.plumbing import providers from litellm.integrations.otel.model.config import ExporterSpec from litellm.integrations.otel.presets import agentops as agentops_mod from litellm.integrations.otel.presets.agentops import ( + _AGENTOPS_AUTH_ENDPOINT, _AGENTOPS_ENDPOINT, _AGENTOPS_EXPORTER_KIND, _build_agentops_exporter, @@ -75,50 +81,44 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): ) -def test_agentops_exporter_mints_jwt_lazily(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - monkeypatch.setattr( - agentops_mod, "_fetch_agentops_jwt", lambda _k: {"token": "jwt-xyz"} - ) - spec = ExporterSpec( - kind=_AGENTOPS_EXPORTER_KIND, - endpoint=_AGENTOPS_ENDPOINT, - options={"api_key": "ak"}, - ) - exporter = _build_agentops_exporter(spec) - - # No auth header until the first export triggers the (off-loop) fetch. - assert "Authorization" not in exporter._session.headers - exporter._ensure_authenticated() - assert exporter._session.headers["Authorization"] == "Bearer jwt-xyz" - - # Cached: a second resolution does not re-fetch. - calls = [] - monkeypatch.setattr( - agentops_mod, - "_fetch_agentops_jwt", - lambda k: calls.append(k) or {"token": "again"}, - ) - exporter._ensure_authenticated() - assert calls == [] - - -def test_agentops_exporter_tolerates_fetch_failure(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - - def _raise(_k): - raise RuntimeError("auth down") - - monkeypatch.setattr(agentops_mod, "_fetch_agentops_jwt", _raise) - exporter = _build_agentops_exporter( +def _agentops_exporter(): + return _build_agentops_exporter( ExporterSpec( kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, options={"api_key": "ak"}, ) ) - exporter._ensure_authenticated() # must not raise - assert "Authorization" not in exporter._session.headers + + +def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"}) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + exporter = _agentops_exporter() + assert token_route.call_count == 0, "building the exporter must not fetch the JWT" + + results = [exporter.export(()), exporter.export(())] + + assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS] + assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}] + assert [request.headers.get("Authorization") for request in collector.request_history] == [ + "Bearer jwt-xyz", + "Bearer jwt-xyz", + ] + + +def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down")) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + + result = _agentops_exporter().export(()) + + assert result is SpanExportResult.SUCCESS + assert [request.headers.get("Authorization") for request in collector.request_history] == [None] def test_fetch_jwt_uses_owned_client_not_shared_pool(monkeypatch): diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 9f536ab0dac..c7608159515 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -513,6 +513,22 @@ class TestLangfuseOtelIntegration: # Endpoint assertion removed as side effect is gone +def _exported_auth_headers(exporter): + import requests_mock + from opentelemetry.sdk.trace.export import SpanExportResult + + with requests_mock.Mocker() as collector: + collector.post(requests_mock.ANY, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + return [ + ( + request.url, + {name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")}, + ) + for request in collector.request_history + ] + + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target (OTLP endpoint + auth), not just auth headers on the init-time exporter.""" @@ -617,15 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig: exporter = span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces" - import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert exporter._headers == { - "Authorization": f"Basic {expected_auth}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"}, + ) + ] def test_key_dynamic_params_reuse_cached_provider(self): with self._clean_env(): @@ -678,10 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig: provider = next(iter(logger._tracer_provider_cache.values())).provider exporter = provider._active_span_processor._span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._headers == { - "Authorization": f"Basic {secret}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"}, + ) + ] class TestLangfuseOtelResponsesAPI: diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index d14703e48a1..f908c685382 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -19,14 +19,16 @@ import pytest # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace +from opentelemetry._logs import LogRecord from opentelemetry._logs.severity import SeverityNumber -from opentelemetry.sdk._logs import LogData, LogRecord +from opentelemetry.sdk._logs import ReadableLogRecord from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider -from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor +from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogRecordExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExportResult from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from parameterized import parameterized @@ -2321,9 +2323,13 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat severity_number=SeverityNumber.INFO, body="tls-export-test", ) - log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("tls-export-test"), + ) result: Final = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() @@ -6639,7 +6645,7 @@ class TestOpenTelemetryProviderlessCallAttributes(unittest.TestCase): time.sleep(self.POLL_INTERVAL) return None - def _emitted_log_records(self, semconv_opt_in: str) -> tuple[LogData, ...]: + def _emitted_log_records(self, semconv_opt_in: str) -> tuple[ReadableLogRecord, ...]: log_exporter = InMemoryLogExporter() logger_provider = OTLoggerProvider() logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) @@ -6713,7 +6719,12 @@ class TestDynamicTracerProviderCache(unittest.TestCase): must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated until pods were OOMKilled).""" - BSP_THREAD_NAME = "OtelBatchSpanProcessor" + def setUp(self) -> None: + processor: Final = BatchSpanProcessor(InMemorySpanExporter()) + try: + self.bsp_thread_name = processor._batch_processor._worker_thread.name + finally: + processor.shutdown() def _logger(self, cap=3, exporter="console"): logger = OpenTelemetry( @@ -6730,7 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._tracer_provider_cache.clear() def _live_exporter_threads(self): - return [t for t in threading.enumerate() if t.name == self.BSP_THREAD_NAME] + return [t for t in threading.enumerate() if t.name == self.bsp_thread_name] def _wait_for_exporter_threads(self, expected, timeout=10.0): """Dropped providers are shut down off-thread, so poll instead of sleeping.""" @@ -6751,8 +6762,6 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"}) self.assertEqual(len(logger._tracer_provider_cache), 3) - # Guards the thread-name constant: a rename upstream would make this read 0 and the - # bound assertion below would pass while measuring nothing. self.assertGreaterEqual(len(self._live_exporter_threads()), 1) self.assertLessEqual(self._wait_for_exporter_threads(before + 3) - before, 3) diff --git a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py index d7474936425..57e66805678 100644 --- a/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py +++ b/tests/unit/litellm_core_utils/test_get_llm_provider_logic.py @@ -8,6 +8,7 @@ from litellm import CustomLLM from litellm.litellm_core_utils import get_llm_provider_logic from litellm.litellm_core_utils.get_llm_provider_logic import ( get_llm_provider, + get_provider_affinity_header, inferred_provider, is_registered_custom_provider, ) @@ -706,3 +707,11 @@ class TestClaudeModelPatternMatching: set_fallback_generalizations([]) with pytest.raises(litellm.BadRequestError): litellm.get_llm_provider(model="claude-opus-4-9") + + +@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"]) +def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None: + api_base: Final = "https://opencode.ai/zen/go/v1" + model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base) + assert (model, provider, destination) == ("test-model", prefix, api_base) + assert get_provider_affinity_header(destination) == "x-opencode-session" diff --git a/tests/unit/litellm_core_utils/test_get_model_cost_map.py b/tests/unit/litellm_core_utils/test_get_model_cost_map.py index 84e785ac99a..d44169ae28a 100644 --- a/tests/unit/litellm_core_utils/test_get_model_cost_map.py +++ b/tests/unit/litellm_core_utils/test_get_model_cost_map.py @@ -399,9 +399,10 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in(): @pytest.mark.asyncio -async def test_refetch_respects_local_env_override(monkeypatch): - """LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) +async def test_refetch_respects_local_env_override(monkeypatch, flag): + """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP.""" + monkeypatch.setenv(flag, "True") def _fail(request): raise AssertionError("no HTTP request should be made when local map is forced") @@ -708,9 +709,10 @@ def test_boot_load_falls_back_when_urlsplit_rejects_the_url(): assert "invalid URL" in (source["fallback_reason"] or "") -def test_boot_load_respects_local_env_override(monkeypatch): - """LITELLM_LOCAL_MODEL_COST_MAP=True still short-circuits to the backup with zero HTTP.""" - monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"]) +def test_boot_load_respects_local_env_override(monkeypatch, flag): + """LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP.""" + monkeypatch.setenv(flag, "True") def _fail(request): raise AssertionError("no HTTP request should be made when local map is forced") diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index bbfae2229ce..8509b1d6f7d 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2047,7 +2047,9 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( ) -> None: """The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key, without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS.""" + import requests_mock from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils import litellm_logging as logging_module @@ -2069,8 +2071,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace" exporter: Final = logger._get_span_processor().span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == expected_endpoint - assert exporter._headers == {"x-api-key": api_key} + with requests_mock.Mocker() as collector: + collector.post(expected_endpoint, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [ + (expected_endpoint, api_key) + ] assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ finally: logging_module._in_memory_loggers.clear() diff --git a/tests/unit/litellm_core_utils/test_offline_mode.py b/tests/unit/litellm_core_utils/test_offline_mode.py new file mode 100644 index 00000000000..f5a4786837d --- /dev/null +++ b/tests/unit/litellm_core_utils/test_offline_mode.py @@ -0,0 +1,17 @@ +import pytest + +from litellm.litellm_core_utils.offline_mode import is_offline_mode + + +@pytest.mark.parametrize( + ("value", "expected"), + [("true", True), ("True", True), (" TRUE ", True), ("false", False), ("", False), ("1", False), ("yes", False)], +) +def test_is_offline_mode_reads_true_case_insensitively(monkeypatch, value, expected): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", value) + assert is_offline_mode() is expected + + +def test_is_offline_mode_is_off_when_unset(monkeypatch): + monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False) + assert is_offline_mode() is False diff --git a/tests/unit/litellm_core_utils/test_token_counter.py b/tests/unit/litellm_core_utils/test_token_counter.py index 194d7995735..88af656ad2e 100644 --- a/tests/unit/litellm_core_utils/test_token_counter.py +++ b/tests/unit/litellm_core_utils/test_token_counter.py @@ -1173,6 +1173,27 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch: assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE) +@pytest.mark.parametrize("model", ["llama-2-7b", "meta-llama/llama-3-8b-instruct"]) +def test_offline_mode_counts_downloaded_tokenizer_models_with_tiktoken(monkeypatch: pytest.MonkeyPatch, model: str): + _load_huggingface_tokenizer.cache_clear() + _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + + with patch("litellm.utils.tokenizer_dispatch.from_pretrained") as from_pretrained: + result: Final = _select_tokenizer_helper(model) + + from_pretrained.assert_not_called() + assert result["type"] == "openai_tokenizer" + + +def test_offline_mode_keeps_deepseek_v4_on_its_bundled_tokenizer(monkeypatch: pytest.MonkeyPatch): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + + assert litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) == _deepseek_token_count( + _CODE_SAMPLE + ) + + def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer(): _load_huggingface_tokenizer.cache_clear() _HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache() diff --git a/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py b/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py index 3826b22af34..ace82e517ad 100644 --- a/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py +++ b/tests/unit/llms/aws_polly/text_to_speech/test_aws_polly_text_to_speech_transformation.py @@ -1,6 +1,10 @@ -import litellm +import json +from unittest.mock import MagicMock, patch + +import httpx import pytest -from unittest.mock import MagicMock + +import litellm @pytest.mark.asyncio @@ -134,6 +138,34 @@ async def test_aws_polly_tts_with_native_voice(): assert request_body.get("TextType", "text") == "text" +@pytest.mark.asyncio +@pytest.mark.usefixtures("fake_provider_credentials") +@pytest.mark.parametrize("engine", ["standard", "neural", "long-form", "generative"]) +async def test_aws_polly_tts_sends_engine_from_model(engine: str) -> None: + """ + The engine in the model name must reach Polly. speech() strips the + provider prefix before the config reads it, which used to turn every + engine into the neural default. + """ + mock_httpx_response = MagicMock(spec=httpx.Response) + mock_httpx_response.content = b"fake_audio_data" + mock_httpx_response.status_code = 200 + mock_httpx_response.headers = {"content-type": "audio/mpeg"} + + with patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post") as mock_post: + mock_post.return_value = mock_httpx_response + + await litellm.aspeech( + model=f"aws_polly/{engine}", + voice="Joanna", + input="Hello", + aws_region_name="us-east-1", + ) + + request_body = json.loads(mock_post.call_args.kwargs["data"]) + assert request_body["Engine"] == engine + + @pytest.mark.asyncio @pytest.mark.usefixtures("fake_provider_credentials") async def test_aws_polly_tts_with_openai_voice_mapping(): diff --git a/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py b/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py index 4778b6296fc..5f195b7a478 100644 --- a/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py +++ b/tests/unit/llms/aws_polly/text_to_speech/test_transformation.py @@ -1,6 +1,8 @@ import json from typing import Final +import pytest + from litellm.llms.aws_polly.text_to_speech.transformation import AWSPollyTextToSpeechConfig @@ -17,3 +19,17 @@ def test_installed_botocore_signs_the_speech_request() -> None: authorization: Final = headers["Authorization"] assert "/us-west-2/polly/aws4_request" in authorization assert json.loads(body) == {"Text": "ping", "VoiceId": "Joanna"} + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + ("standard", "standard"), + ("Long-Form", "long-form"), + ("aws_polly/generative", "generative"), + ("aws_polly", "neural"), + ("unknown", "neural"), + ], +) +def test_extract_engine_from_model(model: str, expected: str) -> None: + assert AWSPollyTextToSpeechConfig()._extract_engine_from_model(model) == expected diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 7ed98f6c705..d46242fc5f7 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -21,6 +21,7 @@ from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager +from tests.unit.proxy.conftest import httpx_transport @pytest.mark.serial @@ -100,6 +101,69 @@ def test_get_complete_url(): assert result == expected +@pytest.mark.parametrize( + ("base_path", "expected_path"), + [ + ("/foundry/openai/openai/v1", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/responses", "/foundry/openai/openai/v1/responses?api-version=v1"), + ("/foundry/openai/openai/v1/responses/", "/foundry/openai/openai/v1/responses?api-version=v1"), + ( + "/openai/v1?redirect=/openai/v1/responses", + "/openai/v1/responses?redirect=%2Fopenai%2Fv1%2Fresponses&api-version=v1", + ), + ( + "/openai/v1?tenant=example&api-version=preview", + "/openai/v1/responses?tenant=example&api-version=preview", + ), + ( + "/openai/v1/responses?api-version=2025-04-01-preview", + "/openai/v1/responses?api-version=2025-04-01-preview", + ), + ( + "/openai/v1/responses?redirect=/openai/responses", + "/openai/v1/responses?redirect=%2Fopenai%2Fresponses&api-version=v1", + ), + ("/foo%2Fbar/openai/v1?tenant=example", "/foo%2Fbar/openai/v1/responses?tenant=example&api-version=v1"), + ( + "/foo%3Fbar/openai/v1/responses?tenant=example", + "/foo%3Fbar/openai/v1/responses?tenant=example&api-version=v1", + ), + ("/openai", "/openai/v1/responses?api-version=v1"), + ("/foundry/openai", "/foundry/openai/v1/responses?api-version=v1"), + ], +) +def test_explicit_v1_api_base_preserves_the_gateway_prefix_and_query(base_path: str, expected_path: str) -> None: + result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url( + api_base=f"https://gateway.example.com{base_path}", litellm_params={"api_version": "v1"} + ) + + assert result == f"https://gateway.example.com{expected_path}" + + +@pytest.mark.parametrize( + ("litellm_params", "expected_version"), + [ + ({}, AZURE_DEFAULT_RESPONSES_API_VERSION), + ({"api_version": "v1"}, "v1"), + ({"api_version": "latest"}, "latest"), + ({"api_version": "preview"}, "preview"), + ({"api_version": "2025-04-01-preview"}, "2025-04-01-preview"), + ({"api_version": "2024-10-21"}, AZURE_DEFAULT_RESPONSES_API_VERSION), + ({"api_version": "2024-10-21", "responses_api_version": "preview"}, "preview"), + ], +) +def test_explicit_v1_api_base_keeps_responses_api_version_selection( + litellm_params: dict[str, str], expected_version: str +) -> None: + result: Final = AzureOpenAIResponsesAPIConfig().get_complete_url( + api_base="https://gateway.example.com/foundry/openai/openai/v1", + litellm_params=litellm_params, + ) + + assert result == f"https://gateway.example.com/foundry/openai/openai/v1/responses?api-version={expected_version}" + + @pytest.mark.serial def test_response_id_path_requests_encode_response_id(): config = AzureOpenAIResponsesAPIConfig() @@ -947,6 +1011,119 @@ def test_dated_ga_api_version_sends_responses_to_the_unset_api_version_url( assert route.calls[0].request.url == _unset_api_version_responses_url() +_COMPLETED_RESPONSE_SSE: Final = "".join( + f"data: {json.dumps(event)}\n\n" + for event in ( + { + "type": "response.output_text.delta", + "sequence_number": 0, + "item_id": "msg_1", + "output_index": 0, + "content_index": 0, + "delta": "Done.", + }, + {"type": "response.completed", "sequence_number": 1, "response": _COMPLETED_RESPONSE}, + ) +) + + +@pytest.fixture +def gateway_responses(respx_mock: respx.MockRouter, stream: bool) -> respx.Route: + body: Final = _COMPLETED_RESPONSE_SSE if stream else json.dumps(_COMPLETED_RESPONSE) + content_type: Final = "text/event-stream" if stream else "application/json" + return respx_mock.post( + "https://gateway.example.com/foundry/openai/openai/v1/responses?api-version=v1" + ).mock(return_value=httpx.Response(200, text=body, headers={"content-type": content_type})) + + +@pytest.mark.parametrize("stream", [False, True]) +def test_explicit_v1_base_sends_responses_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None: + response: Final = litellm.responses( + model="azure/gpt-6.1-sol", + input="hi", + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join(event.delta for event in response if event.type == "response.output_text.delta") + if stream + else response.output[0].content[0].text + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.parametrize("stream", [False, True]) +def test_explicit_v1_base_sends_bridged_chat_to_the_gateway(gateway_responses: respx.Route, stream: bool) -> None: + response: Final = litellm.completion( + model="azure/responses/gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join(chunk.choices[0].delta.content or "" for chunk in response) + if stream + else response.choices[0].message.content + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +async def test_explicit_v1_base_sends_async_responses_to_the_gateway( + gateway_responses: respx.Route, stream: bool +) -> None: + response: Final = await litellm.aresponses( + model="azure/gpt-6.1-sol", + input="hi", + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join([event.delta async for event in response if event.type == "response.output_text.delta"]) + if stream + else response.output[0].content[0].text + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +@pytest.mark.parametrize("stream", [False, True]) +async def test_explicit_v1_base_sends_async_bridged_chat_to_the_gateway( + gateway_responses: respx.Route, stream: bool +) -> None: + response: Final = await litellm.acompletion( + model="azure/responses/gpt-6.1-sol", + messages=[{"role": "user", "content": "hi"}], + api_base="https://gateway.example.com/foundry/openai/openai/v1", + api_key="azure-test-key", + api_version="v1", + stream=stream, + ) + text: Final = ( + "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + if stream + else response.choices[0].message.content + ) + + assert text == "Done." + assert gateway_responses.call_count == 1 + + _LIFECYCLE_BODIES: Final = { "get": _COMPLETED_RESPONSE, "delete": {"id": "resp_dated", "object": "response", "deleted": True}, diff --git a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py index 5186541a62b..ccac6ffc309 100644 --- a/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py +++ b/tests/unit/llms/azure_ai/chat/test_azure_ai_transformation.py @@ -879,6 +879,32 @@ def test_azure_ai_stripping_does_not_mutate_caller_messages(): } +def test_azure_ai_reference_only_tool_result_is_sent_as_empty_text(): + messages: Final = [ + {"role": "user", "content": "load the WebFetch tool"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ToolSearch", "arguments": "{}"}}], + }, + { + "role": "tool", + "tool_call_id": "call_1", + "content": [{"type": "tool_reference", "tool_name": "WebFetch"}], + }, + ] + + request: Final = AzureAIStudioConfig().transform_request( + model="fw-glm-5.2", + messages=messages, + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert request["messages"][2] == {"role": "tool", "tool_call_id": "call_1", "content": ""} + + @pytest.mark.parametrize( "api_base, expected_url", [ diff --git a/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py b/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py index ffddb09cca7..8cf69ba4cd3 100644 --- a/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py +++ b/tests/unit/llms/bedrock/count_tokens/test_bedrock_kimi_k3.py @@ -1,5 +1,6 @@ import asyncio import base64 +import contextlib import hashlib import json import logging @@ -18,6 +19,9 @@ import tiktoken import litellm from litellm._logging import verbose_logger from litellm.llms.bedrock.count_tokens.kimi_k3 import ( + HUGGING_FACE_SOURCE, + KIMI_K3_RANKS_FILE, + KIMI_K3_REVISION, KimiK3Encodings, KimiK3TokenizerSource, kimi_k3_anthropic_tokens, @@ -522,3 +526,40 @@ async def test_tokenizer_load_still_running_after_the_wait_is_logged( assert loaded is None assert any("Kimi K3 tokenizer still loading" in record.getMessage() for record in caplog.records) + + +@pytest.mark.parametrize( + ("offline_mode", "disable_download", "asks_hub"), + [("true", None, False), ("false", True, False), ("false", None, True)], +) +def test_offline_mode_and_disabled_downloads_never_ask_hugging_face_for_the_tokenizer( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, offline_mode: str, disable_download: bool | None, asks_hub: bool +): + import huggingface_hub.constants + import huggingface_hub.utils._detect_agent + + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path / "hub")) + monkeypatch.setattr(huggingface_hub.constants, "AGENT_HARNESSES_PATH", str(tmp_path / "agent_harnesses.json")) + monkeypatch.setattr(huggingface_hub.utils._detect_agent, "_registry", None) + monkeypatch.setenv("LITELLM_OFFLINE_MODE", offline_mode) + monkeypatch.setattr(litellm, "disable_hf_tokenizer_download", disable_download) + with respx.mock(assert_all_called=False) as router: + hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503)) + with contextlib.suppress(Exception): + HUGGING_FACE_SOURCE.ranks_path() + assert hub.called is asks_hub + + +def test_offline_mode_reads_the_tokenizer_from_the_hugging_face_cache(monkeypatch: pytest.MonkeyPatch, tmp_path: Path): + import huggingface_hub.constants + + cached: Final = tmp_path / "models--moonshotai--Kimi-K3" / "snapshots" / KIMI_K3_REVISION / KIMI_K3_RANKS_FILE + cached.parent.mkdir(parents=True) + cached.write_bytes(b"cached ranks") + monkeypatch.setattr(huggingface_hub.constants, "HF_HUB_CACHE", str(tmp_path)) + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + with respx.mock(assert_all_called=False) as router: + hub: Final = router.route(host="huggingface.co").mock(return_value=httpx.Response(503)) + path: Final = HUGGING_FACE_SOURCE.ranks_path() + assert Path(path).read_bytes() == b"cached ranks" + assert not hub.called diff --git a/tests/unit/llms/custom_httpx/test_llm_http_handler.py b/tests/unit/llms/custom_httpx/test_llm_http_handler.py index 63cc45b29df..8b32ab2e6b8 100644 --- a/tests/unit/llms/custom_httpx/test_llm_http_handler.py +++ b/tests/unit/llms/custom_httpx/test_llm_http_handler.py @@ -5542,3 +5542,85 @@ async def test_anthropic_messages_dropped_safeguards_send_neither_the_field_nor_ sent: Final = received[0] assert "safeguards" not in json.loads(sent.content) assert "dangerous-tool-use-2026-09-03" not in sent.headers.get("anthropic-beta", ""), dict(sent.headers) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize("environment_base", [None, "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"]) +@pytest.mark.parametrize( + ("api_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/inference/go/anthropic", None, "conversation-123"), + ("https://opencode.ai/inference/go/anthropic", "caller-override", "caller-override"), + ("https://other.example", None, None), + ], +) +async def test_messages_opencode_session_header( + stream: bool, + environment_base: str | None, + api_base: str, + explicit_header: str | None, + expected_session: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + if environment_base: + monkeypatch.setenv(environment_base, "https://opencode.ai/inference/go/anthropic") + received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + payload: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-haiku-5-5", + "content": [{"type": "text", "text": "OK"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + } + + def respond(request: httpx.Request) -> httpx.Response: + received.put_nowait(request) + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"event: message_start\ndata: {json.dumps({'type': 'message_start', 'message': payload})}\n\n" + 'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + if stream + else httpx.Response(200, json=payload) + ) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + result: Final = await litellm.anthropic.messages.acreate( + model="anthropic/claude-haiku-5-5", + messages=[{"role": "user", "content": "hello"}], + max_tokens=128000, + api_base=None if environment_base and api_base == "https://opencode.ai/inference/go/anthropic" else api_base, + api_key="test-key", + stream=stream, + litellm_session_id="conversation-123", + client=upstream, + extra_headers={"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + ) + if stream: + chunks: Final = b"".join([chunk async for chunk in result]) + assert b'"text": "OK"' in chunks + assert b"message_stop" in chunks + else: + assert result["content"] == [{"type": "text", "text": "OK"}] + finally: + await upstream.close() + sent: Final = received.get_nowait() + assert str(sent.url) == f"{api_base}/v1/messages" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["x-api-key"] == "test-key" + assert json.loads(sent.content) == { + "model": "claude-haiku-5-5", + "messages": [{"role": "user", "content": "hello"}], + "max_tokens": 128000, + "stream": stream, + } + assert received.empty() diff --git a/tests/unit/llms/opencode/test_transformation.py b/tests/unit/llms/opencode/test_transformation.py new file mode 100644 index 00000000000..96203c23122 --- /dev/null +++ b/tests/unit/llms/opencode/test_transformation.py @@ -0,0 +1,90 @@ +from typing import Final + +import pytest + +from litellm.constants import SESSION_ID_GENERATED_METADATA_KEY +from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header +from litellm.llms.opencode.transformation import OpenCodeProviderAdapter + + +@pytest.mark.parametrize( + "api_base", + [ + "https://opencode.ai/zen/v1", + "https://opencode.ai/zen/go/v1/", + "https://opencode.ai/inference/openai/v1", + "https://opencode.ai/inference/go/openai/v1", + "https://opencode.ai/inference/go/anthropic/v1", + "https://opencode.ai/inference/anthropic/v1", + ], +) +def test_opencode_defaults_affinity_header_to_the_callers_session(api_base: str) -> None: + headers: Final = {"Authorization": "Bearer test-key"} + params: Final = {"metadata": {"session_id": "conversation-123"}} + assert add_provider_affinity_header( + headers, params, OpenCodeProviderAdapter().get_provider_affinity_header(api_base) + ) == { + **headers, + "x-opencode-session": "conversation-123", + } + assert headers == {"Authorization": "Bearer test-key"} + assert params == {"metadata": {"session_id": "conversation-123"}} + + +@pytest.mark.parametrize( + "api_base", + [ + None, + "https://api.openai.com/v1", + "https://opencode.ai.example/zen/go/v1", + "https://opencode.ai@example.com/zen/go/v1", + "https://example.com/opencode.ai/zen/go/v1", + "https://example.com/v1?upstream=https://opencode.ai/zen/go/v1", + "https://opencode.ai/console/v1", + "http://opencode.ai/zen/go/v1", + ], +) +def test_other_destinations_do_not_receive_an_automatic_session_header(api_base: str | None) -> None: + headers: Final = {"Authorization": "Bearer test-key", "x-existing": "unchanged"} + assert ( + add_provider_affinity_header( + headers, + {"litellm_session_id": "conversation-123"}, + OpenCodeProviderAdapter().get_provider_affinity_header(api_base), + ) + == headers + ) + + +@pytest.mark.parametrize( + "params", + [ + {}, + {"litellm_trace_id": "request-123"}, + {"metadata": {"trace_id": "request-123"}}, + { + "litellm_session_id": "request-123", + "litellm_metadata": {"session_id": "request-123", SESSION_ID_GENERATED_METADATA_KEY: True}, + }, + ], +) +def test_opencode_does_not_invent_a_conversation_id(params: dict[str, object]) -> None: + assert ( + add_provider_affinity_header( + {}, params, OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1") + ) + == {} + ) + + +def test_opencode_preserves_explicit_header_and_affinity_configuration() -> None: + assert add_provider_affinity_header( + {"X-OpenCode-Session": "explicit"}, + {"litellm_session_id": "conversation-123"}, + OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + ) == {"X-OpenCode-Session": "explicit"} + assert add_provider_affinity_header( + {}, + {"litellm_session_id": "conversation-123", "provider_affinity_header": "x-custom-session"}, + OpenCodeProviderAdapter().get_provider_affinity_header("https://opencode.ai/zen/go/v1"), + ) == {"x-custom-session": "conversation-123"} diff --git a/tests/unit/proxy/auth/test_password_policy.py b/tests/unit/proxy/auth/test_password_policy.py index f9f5025b57f..0ad1083ebe0 100644 --- a/tests/unit/proxy/auth/test_password_policy.py +++ b/tests/unit/proxy/auth/test_password_policy.py @@ -180,6 +180,18 @@ async def test_breach_check_skipped_when_disabled(): assert result is None +@pytest.mark.asyncio +async def test_breach_check_skipped_in_offline_mode(monkeypatch): + monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true") + password = "correct horse battery staple" + result = await validate_password_not_breached( + password=password, + general_settings={}, + client=_client_returning(f"{_sha1_upper(password)[5:]}:42"), + ) + assert result is None + + @pytest.mark.asyncio async def test_rejects_breached_password(): password = "correct horse battery staple" diff --git a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py index 4cdd7fefd28..108f7ce2983 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth_request_flow.py @@ -11688,6 +11688,124 @@ def _alias_request(route: str, data: dict, content_type: str = "application/json return request +def _viewer_inference_key(monkeypatch: pytest.MonkeyPatch, team_id: str | None, *, spend: float = 0.25) -> str: + from litellm.proxy import proxy_server + from litellm.proxy.utils import ProxyLogging, hash_token + + key: Final = "sk-viewer-inference" + hashed_key: Final = hash_token(key) + cache: Final = UserApiKeyCache() + cache.set_cache( + key=hashed_key, + value=UserAPIKeyAuth( + token=hashed_key, + user_id="viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + team_id=team_id, + models=["gpt-6.1-sol"], + max_budget=1.0, + spend=spend, + ), + ) + cache.set_cache( + key="viewer", + value=LiteLLM_UserTable( + user_id="viewer", + user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + user_email="viewer@example.com", + models=["gpt-6.1-sol"], + ), + ) + if team_id is not None: + cache.set_cache( + key=team_membership_reservation_cache_key(user_id="viewer", team_id=team_id), + value=NO_TEAM_MEMBERSHIP_SENTINEL, + ) + for name, value in { + **_proxy_attrs_for_centralized_checks(), + "prisma_client": MagicMock(), + "user_api_key_cache": cache, + "spend_counter_cache": DualCache(), + "proxy_logging_obj": ProxyLogging(user_api_key_cache=cache), + "llm_model_list": [], + }.items(): + monkeypatch.setattr(proxy_server, name, value) + return key + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +@pytest.mark.parametrize("route", ["/chat/completions", "/v1/chat/completions"]) +async def test_viewer_inference_uses_the_normal_key_authorization_path( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, route: str +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request(route, {"model": "gpt-6.1-sol", "messages": [{"role": "user", "content": "usage"}]}) + + result: Final = await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert (result.user_id, result.user_role, result.team_id, result.models, result.max_budget) == ( + "viewer", + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + team_id, + ["gpt-6.1-sol"], + 1.0, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +async def test_viewer_inference_rejects_a_model_outside_the_key_allowlist( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request("/chat/completions", {"model": "blocked-model"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert (failure.value.code, failure.value.type, failure.value.param) == ( + "403", + ProxyErrorTypes.key_model_access_denied, + "model", + ) + assert failure.value.message == ( + "The requested model 'blocked-model' is not available for this API key, or the model name is invalid. " + "Check the models available to you and try again." + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +@pytest.mark.parametrize("spend", [1.0, 1.25], ids=["at-limit", "over-limit"]) +async def test_viewer_inference_rejects_an_exhausted_key_budget( + monkeypatch: pytest.MonkeyPatch, team_id: str | None, spend: float +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id, spend=spend) + request: Final = _alias_request("/chat/completions", {"model": "gpt-6.1-sol"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert failure.value.type == ProxyErrorTypes.budget_exceeded + assert f"Current cost: {spend}, Max budget: 1.0" in failure.value.message + + +@pytest.mark.asyncio +@pytest.mark.parametrize("team_id", ["litellm-dashboard", None], ids=["dashboard-session", "virtual-key"]) +async def test_viewer_inference_access_does_not_allow_management_writes( + monkeypatch: pytest.MonkeyPatch, team_id: str | None +) -> None: + key: Final = _viewer_inference_key(monkeypatch, team_id) + request: Final = _alias_request("/team/new", {"team_alias": "forbidden"}) + + with pytest.raises(ProxyException) as failure: + await user_api_key_auth(request=request, api_key=f"Bearer {key}") + + assert failure.value.code == "403" + assert "proxy_admin_viewer" in failure.value.message + + async def _enforce_alias_access(token: UserAPIKeyAuth, data: dict, route: str, request, router: litellm.Router): from litellm.proxy.auth.user_api_key_auth import enforce_key_and_fallback_model_access @@ -12480,11 +12598,14 @@ async def test_lens_oauth2_validates_identity_without_inference_user_field( patch("litellm.proxy.proxy_server.master_key", "sk-fixture-master"), patch("litellm.proxy.proxy_server.prisma_client", None), patch("litellm.proxy.proxy_server.user_api_key_cache", DualCache()), - patch.dict(os.environ, { - "OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect", - "OAUTH_CLIENT_ID": "fixture-client", - "OAUTH_CLIENT_SECRET": "fixture-client-secret", - }), + patch.dict( + os.environ, + { + "OAUTH_TOKEN_INFO_ENDPOINT": "https://oauth.fixture.test/introspect", + "OAUTH_CLIENT_ID": "fixture-client", + "OAUTH_CLIENT_SECRET": "fixture-client-secret", + }, + ), patch("httpx.AsyncClient.send", side_effect=token_info) as network, ): if active: @@ -12611,6 +12732,4 @@ def test_should_skip_budget_checks_spares_only_zero_spend_mcp_discovery( ) -> None: from litellm.proxy.auth.user_api_key_auth import _should_skip_budget_checks - assert ( - _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected - ) + assert _should_skip_budget_checks(request_data=request_data, route=route, request=None, llm_router=None) is expected diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index 7437042e4bd..9b2a96bbc58 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -36,6 +36,7 @@ from litellm.proxy.db.db_transaction_queue.window_spend_update_queue import ( build_window_spend_transaction, ) from litellm.proxy.db.rollup_lock_timeout import ROLLUP_LOCK_TIMEOUT_SQL +from litellm.responses.streaming_iterator import drain_logging_tasks from litellm.types.caching import RedisPipelineLpopOperation, RedisPipelineRpushOperation from tests.unit.proxy.db.fake_prisma_engine import engine_call @@ -2158,46 +2159,122 @@ async def test_commit_spend_updates_to_db_increments_key_total_spend_alongside_s @pytest.mark.asyncio -async def test_update_database_creates_single_task(): +async def test_shutdown_drain_finishes_the_rollups_update_database_started(): """ - Test that update_database() fires exactly 1 asyncio.create_task() call - (the batched task) instead of the previous 11. + Regression for LIT-6027. The key, team and daily rollups ran in an unreferenced asyncio task, + so the shutdown sequence (flush, drain_logging_tasks, flush) never waited for them and spend + logs landed without matching daily rows. Once update_database returns, drain_logging_tasks + must leave every rollup for that request in its queue. """ - db_writer = DBSpendUpdateWriter() - - # Mock all helpers so nothing real runs + db_writer: Final = DBSpendUpdateWriter() db_writer._insert_spend_log_to_db = AsyncMock() - db_writer._batch_database_updates = AsyncMock() - + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + fake_payload: Final = { + "request_id": "req-lit-6027", + "api_key": "hashed-key", + "user": "test-user", + "team_id": "test-team", + "agent_id": None, + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 10, + "completion_tokens": 5, + "spend": 0.0, + "metadata": '{"usage_object": {}}', + } with ( patch("litellm.proxy.proxy_server.disable_spend_logs", False), - patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.prisma_client", prisma), patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), - patch("litellm.proxy.db.db_spend_update_writer.asyncio.create_task") as mock_create_task, + patch("litellm.proxy.spend_tracking.spend_tracking_utils.get_logging_payload", return_value=fake_payload), ): await db_writer.update_database( token="test-token", user_id="test-user", - end_user_id="test-end-user", + end_user_id=None, + team_id="test-team", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), start_time=datetime.now(), end_time=datetime.now(), - team_id="test-team", - org_id="test-org", - completion_response=MagicMock(), response_cost=0.1, - kwargs={"model": "gpt-4", "custom_llm_provider": "openai"}, ) + await drain_logging_tasks() - # Exactly 1 create_task call (the batch), not 11 - assert mock_create_task.call_count == 1 + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + assert spend["key_list_transactions"] == {"test-token": 0.1} + assert spend["team_list_transactions"] == {"test-team": 0.1} + daily_user: Final = ( + await db_writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + daily_team: Final = ( + await db_writer.daily_team_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + ) + assert daily_user is not None and daily_team is not None + (user_row,) = daily_user.values() + (team_row,) = daily_team.values() + assert (user_row["api_requests"], user_row["spend"]) == (1, 0.1) + assert (team_row["api_requests"], team_row["spend"]) == (1, 0.1) @pytest.mark.asyncio -async def test_batch_database_updates_isolation_on_failure(): +async def test_shutdown_drain_waits_for_an_in_flight_rollup_enqueue(): + """ + Regression for LIT-6027. A rollup enqueue that is still in flight when the shutdown drain + starts (here: blocked on a full spend queue) must be finished by the time drain_logging_tasks + returns, so the second shutdown flush commits it. + """ + enqueue_started: Final = asyncio.Event() + release_enqueue: Final = asyncio.Event() + + class _BlockedSpendQueue(SpendUpdateQueue): + async def add_update(self, update: SpendUpdateQueueItem): + enqueue_started.set() + await release_enqueue.wait() + await super().add_update(update) + + db_writer: Final = DBSpendUpdateWriter() + db_writer.spend_update_queue = _BlockedSpendQueue() + db_writer._insert_spend_log_to_db = AsyncMock() + with ( + patch("litellm.proxy.proxy_server.disable_spend_logs", True), + patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), + patch("litellm.proxy.proxy_server.user_api_key_cache", MagicMock()), + patch("litellm.proxy.proxy_server.litellm_proxy_budget_name", "test-budget"), + ): + await db_writer.update_database( + token="t1", + user_id=None, + end_user_id=None, + team_id="team-1", + org_id=None, + kwargs={"model": "gpt-4o-mini", "custom_llm_provider": "openai"}, + completion_response=MagicMock(), + start_time=datetime.now(), + end_time=datetime.now(), + response_cost=0.1, + ) + await enqueue_started.wait() + drain: Final = asyncio.create_task(drain_logging_tasks()) + release_enqueue.set() + await drain + spend: Final = await db_writer.spend_update_queue.flush_and_get_aggregated_db_spend_update_transactions() + + assert spend["key_list_transactions"] == {"t1": 0.1} + assert spend["team_list_transactions"] == {"team-1": 0.1} + + +@pytest.mark.asyncio +async def test_batch_database_updates_isolation_on_failure(caplog: pytest.LogCaptureFixture): """ Test that if one helper inside _batch_database_updates raises, - all other helpers still execute. + all other helpers still execute and the failure is reported at ERROR with the request id. """ db_writer = DBSpendUpdateWriter() @@ -2217,6 +2294,7 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_org_transaction = AsyncMock() db_writer.add_spend_log_transaction_to_daily_tag_transaction = AsyncMock() + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) await db_writer._batch_database_updates( response_cost=0.1, user_id="u1", @@ -2226,7 +2304,7 @@ async def test_batch_database_updates_isolation_on_failure(): end_user_id="eu1", prisma_client=MagicMock(), litellm_proxy_budget_name="budget", - payload={"key": "value"}, + payload={"request_id": "req-iso", "key": "value"}, ) # _update_key_db raised, but all others should still have been called @@ -2242,6 +2320,48 @@ async def test_batch_database_updates_isolation_on_failure(): db_writer.add_spend_log_transaction_to_daily_team_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_org_transaction.assert_awaited_once() db_writer.add_spend_log_transaction_to_daily_tag_transaction.assert_awaited_once() + assert any( + "_update_key_db" in record.getMessage() and "req-iso" in record.getMessage() + for record in caplog.records + if record.levelno >= logging.ERROR + ) + + +@pytest.mark.asyncio +async def test_malformed_metadata_keeps_the_daily_rollup_row_and_is_reported(caplog: pytest.LogCaptureFixture): + """ + Regression for LIT-6027. A spend log whose metadata is not JSON used to vanish from every + daily rollup while the spend log itself persisted, with nothing above debug in the logs. + """ + writer: Final = DBSpendUpdateWriter() + prisma: Final = MagicMock() + prisma.get_request_status = MagicMock(return_value="success") + payload: Final = { + "request_id": "req-bad-metadata", + "user": "test-user", + "startTime": "2024-01-01T00:00:00", + "endTime": "2024-01-01T00:00:01", + "api_key": "test-key", + "model": "gpt-4o-mini", + "custom_llm_provider": "openai", + "model_group": "gpt-4o-mini", + "prompt_tokens": 100, + "completion_tokens": 50, + "spend": 0.05, + "metadata": "{not-json", + } + caplog.set_level(logging.ERROR, logger=verbose_proxy_logger.name) + + await writer.add_spend_log_transaction_to_daily_user_transaction( # pyright: ignore[reportUnknownMemberType] # prisma_client param is untyped on the writer + payload=payload, prisma_client=prisma + ) + + daily: Final = await writer.daily_spend_update_queue.flush_and_get_aggregated_daily_spend_update_transactions() + (row,) = daily.values() + assert (row["api_requests"], row["spend"], row["prompt_tokens"]) == (1, 0.05, 100) + assert any( + "req-bad-metadata" in record.getMessage() for record in caplog.records if record.levelno >= logging.ERROR + ) @pytest.mark.asyncio @@ -2775,12 +2895,12 @@ async def test_update_database_does_not_deepcopy_on_request_path(): """ Regression for LIT-4088: copy.deepcopy must not run while the caller awaits update_database(). The deepcopy used to isolate the daily-spend helpers is - relocated into the _batch_database_updates background task, and the spend-log + relocated into the _batch_database_updates logging task, and the spend-log insert receives the payload directly (all consumers are read-only). Asserts: - zero copy.deepcopy calls happen on the awaited request path - - the batch background task still hands the daily helpers an isolated copy + - the tracked rollup task still hands the daily helpers an isolated copy (mutating the original after the task ran does not bleed into it) - the spend-log insert receives the payload on the request path with the correct content @@ -2856,16 +2976,13 @@ async def test_update_database_does_not_deepcopy_on_request_path(): response_cost=0.1, ) - # Request path is clean: nothing was deepcopied while the caller awaited. assert len(deepcopy_calls) == 0 # The spend-log insert ran inline on the request path with the real payload. assert captured_spend_log["ref"] is fake_payload assert captured_spend_log["model_at_call"] == "gpt-4" assert fake_payload["spend"] == 0.1 - - # Now let the batch background task run; the deepcopy happens here. - await asyncio.sleep(0) + await drain_logging_tasks() assert len(deepcopy_calls) >= 1 assert len(captured_batch_payloads) == 1 diff --git a/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py b/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py index 2b438a9d370..4b42e6404a3 100644 --- a/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py +++ b/tests/unit/proxy/management_endpoints/management_v1/test_budgets.py @@ -52,7 +52,7 @@ app.include_router(router) client = TestClient(app) BUDGETS_PATH = f"{MANAGEMENT_V1_PREFIX}/budgets" -SORTABLE = ["budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"] +SORTABLE = ["budget_duration", "budget_id", "created_at", "max_budget", "rpm_limit", "tpd_limit", "tpm_limit"] def _row(budget_id: str, **overrides: Any) -> dict[str, Any]: @@ -172,22 +172,12 @@ def test_appends_the_tiebreaker_to_an_explicit_sort(query_raw, as_proxy_admin): _get("sort=-max_budget") - assert 'ORDER BY "max_budget" DESC NULLS LAST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0] + assert 'ORDER BY "max_budget" DESC NULLS FIRST, "budget_id" ASC NULLS LAST' in _select_call(query_raw)[0] -def test_refuses_to_sort_on_budget_duration(query_raw, as_proxy_admin): - """The column holds "7d"/"30d", so a lexicographic ORDER BY would put "30d" - before "7d" and silently mis-order the page.""" - _serve(query_raw, []) - +def test_accepts_duration_sorting(query_raw, as_proxy_admin): response = _get("sort=budget_duration") - - assert response.status_code == 400 - assert response.headers["content-type"].startswith("application/problem+json") - body = response.json() - assert "budget_duration" in body["detail"] - assert body["allowed"] == SORTABLE - query_raw.assert_not_called() + assert response.status_code == 200 def test_the_advertised_sort_fields_are_the_ones_that_work(query_raw, as_proxy_admin): @@ -346,7 +336,7 @@ def test_q_does_not_search_any_other_column(query_raw, as_proxy_admin): def test_is_null_selects_the_unlimited_budgets(query_raw, as_proxy_admin): - """"Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap.""" + """ "Unlimited" is max_budget IS NULL; `max_budget = 0` would be a hard zero cap.""" _serve(query_raw, [_row("b-unlimited", max_budget=None)]) body = _get("filter[max_budget][is_null]=true").json() diff --git a/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py b/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py index 62863163b8e..4b62cf79094 100644 --- a/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py +++ b/tests/unit/proxy/management_endpoints/policy_endpoints/test_endpoints.py @@ -5,14 +5,20 @@ Tests _test_guardrail_definitions and _compute_overall_action directly without needing a running proxy. """ +import json +from pathlib import Path +from unittest.mock import patch + import pytest from fastapi import Request +import litellm from litellm.proxy._types import UserAPIKeyAuth from litellm.proxy.management_endpoints.policy_endpoints.endpoints import ( GuardrailTestResultEntry, _compute_overall_action, _test_guardrail_definitions, + get_policy_templates, list_policies, ) from litellm.proxy.policy_engine.policy_registry import get_policy_registry @@ -368,3 +374,19 @@ async def test_list_policies_reports_each_loaded_policy_with_its_resolved_guardr assert response == PolicyListResponse(policies=expected_policies, total_count=0) assert list(response.policies) == list(expected_policies) + + +@pytest.mark.parametrize("flag", ["LITELLM_OFFLINE_MODE", "LITELLM_LOCAL_POLICY_TEMPLATES"]) +@pytest.mark.asyncio +async def test_policy_templates_serve_the_bundled_copy_without_fetching(monkeypatch: pytest.MonkeyPatch, flag: str): + monkeypatch.setenv(flag, "true") + bundled = json.loads((Path(litellm.__file__).parent / "policy_templates_backup.json").read_text(encoding="utf-8")) + + with patch( + "litellm.llms.custom_httpx.http_handler.get_async_httpx_client", + side_effect=AssertionError("policy templates were fetched"), + ) as get_client: + response = await get_policy_templates(request=_policy_list_request(), user_api_key_dict=UserAPIKeyAuth()) + + get_client.assert_not_called() + assert response == bundled diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index 479391e54fe..e1a939273ae 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -9198,58 +9198,6 @@ async def test_saml_callback_blocked_when_admin_ui_disabled(): assert "Admin UI is Disabled" in response.body.decode() -@pytest.mark.asyncio -async def test_saml_callback_enforces_free_sso_user_limit_after_validation(): - """An IdP-initiated assertion must not bypass the >5 free-SSO-user Enterprise gate - that /sso/key/generate enforces; the ACS re-checks it after validating the assertion, - so the entitlement DB query never runs on unvalidated input.""" - from litellm.proxy._types import ProxyException - from litellm.proxy.management_endpoints.types import CustomOpenID - from litellm.proxy.management_endpoints.ui_sso import saml_callback - - call_order: list[str] = [] - - async def _fake_handle_acs(**kwargs): - call_order.append("validate") - return (CustomOpenID( - id="dana@litellm.ai", - email="dana@litellm.ai", - first_name=None, - last_name=None, - display_name="dana", - picture=None, - provider="saml", - team_ids=[], - user_role=None, - ), None) - - async def _fake_count_billable_users(): - call_order.append("count") - return 6 - - async def _stream(): - yield b"SAMLResponse=signed-response" - - request_double = SimpleNamespace(cookies={}, headers={}, stream=_stream) - - with patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}), patch( - "litellm.proxy.proxy_server.premium_user", False - ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy.proxy_server.master_key", MASTER_KEY - ), patch( - "litellm.proxy.management_endpoints.sso.saml_sso.SAMLAuthHandler.handle_acs", - new=_fake_handle_acs, - ), patch( - "litellm.repositories.user_repository.UserRepository.count_billable_users", - new=AsyncMock(side_effect=_fake_count_billable_users), - ): - with pytest.raises(ProxyException) as exc: - await saml_callback(request_double) - - assert str(exc.value.code) == "403" - assert call_order == ["validate", "count"] - - @pytest.mark.asyncio async def test_cli_poll_key_tolerates_missing_user_row(): """The CLI poll must still mint the JWT when the user lookup raises, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py deleted file mode 100644 index 4daddb89350..00000000000 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ /dev/null @@ -1,657 +0,0 @@ -""" -Tests for AI Usage Chat module. -""" - -import json -from collections.abc import AsyncIterator -from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from fastapi import HTTPException - -from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - TOOL_HANDLERS, - TOOLS_ADMIN, - TOOLS_BASE, - _build_system_prompt, - _summarise_entity_data, - _summarise_usage_data, - stream_usage_ai_chat, -) - - -SAMPLE_AGGREGATED_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": { - "spend": 50.25, - "prompt_tokens": 20000, - "completion_tokens": 10000, - "total_tokens": 30000, - "api_requests": 500, - "successful_requests": 480, - "failed_requests": 20, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - "breakdown": { - "models": { - "gpt-4": { - "metrics": { - "spend": 40.0, - "api_requests": 300, - "total_tokens": 25000, - }, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "providers": { - "openai": { - "metrics": {"spend": 50.25, "api_requests": 500}, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "api_keys": { - "sk-test123": { - "metrics": {"spend": 50.25}, - "metadata": {"key_alias": "Production Key"}, - }, - }, - "model_groups": {}, - "mcp_servers": {}, - "entities": {}, - }, - }, - ], - "metadata": { - "total_spend": 50.25, - "total_api_requests": 500, - "total_successful_requests": 480, - "total_failed_requests": 20, - "total_tokens": 30000, - }, -} - -SAMPLE_TEAM_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": {"spend": 100.0, "api_requests": 1000, "total_tokens": 50000}, - "breakdown": { - "entities": { - "team-1": { - "metrics": { - "spend": 60.0, - "api_requests": 600, - "total_tokens": 30000, - }, - "metadata": {"alias": "Engineering"}, - "api_key_breakdown": {}, - }, - "team-2": { - "metrics": { - "spend": 40.0, - "api_requests": 400, - "total_tokens": 20000, - }, - "metadata": {"alias": "Marketing"}, - "api_key_breakdown": {}, - }, - }, - "models": {}, - "providers": {}, - "api_keys": {}, - "model_groups": {}, - "mcp_servers": {}, - }, - }, - ], - "metadata": {"total_spend": 100.0, "total_api_requests": 1000}, -} - - -class TestToolSchemas: - def test_admin_tools_include_all(self): - assert len(TOOLS_ADMIN) == 3 - names = {t["function"]["name"] for t in TOOLS_ADMIN} - assert "get_usage_data" in names - assert "get_team_usage_data" in names - assert "get_tag_usage_data" in names - - def test_base_tools_restricted_to_usage_only(self): - assert len(TOOLS_BASE) == 1 - assert TOOLS_BASE[0]["function"]["name"] == "get_usage_data" - - def test_admin_prompt_mentions_all_tools(self): - prompt = _build_system_prompt(is_admin=True) - assert "get_usage_data" in prompt - assert "get_team_usage_data" in prompt - assert "get_tag_usage_data" in prompt - - def test_non_admin_prompt_only_mentions_usage_tool(self): - prompt = _build_system_prompt(is_admin=False) - assert "get_usage_data" in prompt - assert "get_team_usage_data" not in prompt - assert "get_tag_usage_data" not in prompt - - def test_system_prompt_includes_todays_date(self): - from datetime import date - - prompt = _build_system_prompt(is_admin=True) - assert date.today().isoformat() in prompt - - -class TestSummariseUsageData: - def test_summarise_includes_totals(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "$50.25" in summary - assert "500" in summary - - def test_summarise_includes_models(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "gpt-4" in summary - - def test_summarise_includes_providers(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "openai" in summary - - def test_summarise_handles_empty_data(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_usage_data(empty) - assert "no data" in summary.lower() - - -class TestSummariseEntityData: - def test_team_summary_includes_teams(self): - summary = _summarise_entity_data(SAMPLE_TEAM_RESPONSE, "Team") - assert "Engineering" in summary - assert "Marketing" in summary - assert "$60.0" in summary - assert "$40.0" in summary - - def test_team_summary_empty(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_entity_data(empty, "Team") - assert "No Team usage data" in summary - - -class TestStreamUsageAiChat: - @pytest.mark.asyncio - async def test_stream_emits_status_events(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_123" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Total spend is $50.25" - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_AGGREGATED_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="gpt-4o-mini", - user_id="user-123", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - status_events = [e for e in events if e["type"] == "status"] - tool_call_events = [e for e in events if e["type"] == "tool_call"] - chunk_events = [e for e in events if e["type"] == "chunk"] - done_events = [e for e in events if e["type"] == "done"] - - assert len(status_events) >= 1 - assert "Thinking" in status_events[0]["message"] - assert len(tool_call_events) >= 1 - assert tool_call_events[0]["tool_name"] == "get_usage_data" - assert tool_call_events[0]["status"] in ("running", "complete") - assert len(chunk_events) >= 1 - assert len(done_events) == 1 - - @pytest.mark.asyncio - async def test_stream_handles_team_tool(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_team" - mock_tool_call.function.name = "get_team_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_team", - "type": "function", - "function": { - "name": "get_team_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Engineering is the top team." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_team_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_TEAM_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Which team spends the most?"}], - model="gpt-4o-mini", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - chunk_events = [e for e in events if e["type"] == "chunk"] - assert len(chunk_events) >= 1 - assert "Engineering" in chunk_events[0]["content"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "reason", - [ - "end_date must be on or after start_date", - "start_date and end_date must be valid YYYY-MM-DD dates", - ], - ids=["reversed", "invalid-format"], - ) - @pytest.mark.parametrize( - "tool_name", ["get_usage_data", "get_team_usage_data", "get_tag_usage_data"] - ) - async def test_invalid_tool_dates_are_reported_to_the_model( - self, tool_name: str, reason: str - ) -> None: - fetch: Final = AsyncMock( - side_effect=HTTPException(status_code=400, detail={"error": reason}) - ) - - tool_message, statuses = await _run_one_tool( - tool_name, "2025-01-31", "2025-01-01", fetch - ) - - fetch.assert_awaited_once() - assert tool_message == f"Invalid date range: {reason}" - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - async def test_tool_dates_the_query_layer_accepts_are_queried_as_sent( - self, - ) -> None: - fetch: Final = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - tool_message, statuses = await _run_one_tool( - "get_usage_data", "2025/01/01", "2025-01-31T00:00:00", fetch - ) - - fetch.assert_awaited_once_with( - start_date="2025/01/01", end_date="2025-01-31T00:00:00" - ) - assert tool_message.startswith("Total Spend: $") - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "error", - [ - HTTPException(status_code=500, detail={"error": "db down"}), - HTTPException(status_code=400, detail="page and page_size must be >= 1"), - ], - ids=["server-error", "other-400"], - ) - async def test_other_tool_errors_stay_a_generic_fetch_error( - self, error: HTTPException - ) -> None: - tool_message, statuses = await _run_one_tool( - "get_team_usage_data", - "2025-01-01", - "2025-01-31", - AsyncMock(side_effect=error), - ) - - assert tool_message == "Error fetching team usage data. Please try again." - assert statuses == ["running", "error"] - - @pytest.mark.asyncio - async def test_stream_handles_error(self): - with patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm: - mock_litellm.acompletion = AsyncMock(side_effect=Exception("LLM error")) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "test"}], - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - error_events = [e for e in events if e["type"] == "error"] - assert len(error_events) == 1 - assert "internal error" in error_events[0]["message"].lower() - - @pytest.mark.asyncio - async def test_non_admin_enforces_user_id(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_456" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - "user_id": "other-user", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_456", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31","user_id":"other-user"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Data." - yield chunk - - mock_fetch = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS", - { - "get_usage_data": { - "fetch": mock_fetch, - "summarise": _summarise_usage_data, - "label": "global usage data", - } - }, - ), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Show data"}], - model="gpt-4o-mini", - user_id="my-user-id", - is_admin=False, - ): - events.append(event) - - mock_fetch.assert_called_once_with( - start_date="2025-01-01", - end_date="2025-01-31", - user_id="my-user-id", - ) - - -class TestUsageAiChatServiceAccountGuard: - """ - Security regression: a non-admin caller with user_id=None (service-account - key) must be rejected at the endpoint boundary, before any tool dispatch. - """ - - @pytest.mark.asyncio - async def test_non_admin_with_user_id_none_is_rejected(self): - from fastapi import HTTPException - - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - service_account_key = UserAPIKeyAuth( - user_id=None, - user_role=LitellmUserRoles.INTERNAL_USER, - ) - request = MagicMock() - body = UsageAIChatRequest( - messages=[ChatMessage(role="user", content="hi")], - model="gpt-4o-mini", - ) - - with pytest.raises(HTTPException) as exc_info: - await usage_ai_chat( - data=body, - request=request, - user_api_key_dict=service_account_key, - ) - - assert exc_info.value.status_code == 403 - assert "Service-account keys" in str(exc_info.value.detail) - - def test_resolve_fetch_kwargs_tripwire_fires_on_none_user_id(self): - """ - Defense-in-depth: if a future endpoint forgets the entry guard and - a non-admin caller with user_id=None reaches _resolve_fetch_kwargs, - the tripwire must fire rather than issuing an unscoped query. - """ - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - _resolve_fetch_kwargs, - ) - - with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info: - _resolve_fetch_kwargs( - fn_name="get_usage_data", - fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"}, - user_id=None, - is_admin=False, - ) - assert "Endpoint-level guard missing" in str(exc_info.value) - - -class TestUsageAiChatKeepalive: - async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: - import asyncio - - import litellm - from fastapi.responses import StreamingResponse - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) - - async def slow_acompletion(**kwargs): - await asyncio.sleep(delay) - response = MagicMock() - response.choices = [MagicMock()] - response.choices[0].message.tool_calls = None - response.choices[0].message.content = "Total spend is $50.25" - return response - - with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", - new=AsyncMock(side_effect=slow_acompletion), - ): - response = await usage_ai_chat( - data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"), - request=MagicMock(), - user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), - ) - assert isinstance(response, StreamingResponse) - chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] - return chunks, dict(response.headers) - - @pytest.mark.asyncio - async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch): - chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05) - - assert headers["content-type"].startswith("text/event-stream") - assert headers["cache-control"] == "no-cache" - assert headers["x-accel-buffering"] == "no" - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[1] == b": ping\n\n" - assert chunks.count(b": ping\n\n") >= 3 - assert b'"content": "Total spend is $50.25"' in b"".join(chunks) - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - @pytest.mark.asyncio - async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch): - chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15) - - assert b": ping\n\n" not in chunks - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - -async def _run_one_tool( - tool_name: str, start_date: str, end_date: str, fetch: AsyncMock -) -> tuple[str, list[object]]: - """Runs one admin chat turn whose model calls `tool_name` with these dates; returns the - tool message the follow-up model call carried and the tool statuses streamed.""" - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - mock_tool_call: Final = MagicMock() - mock_tool_call.id = "call_dates" - mock_tool_call.function.name = tool_name - mock_tool_call.function.arguments = arguments - - mock_first_response: Final = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_dates", - "type": "function", - "function": {"name": tool_name, "arguments": arguments}, - } - ], - } - - async def mock_stream() -> AsyncIterator[MagicMock]: - chunk: Final = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Done." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict(TOOL_HANDLERS[tool_name], {"fetch": fetch}), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[mock_first_response, mock_stream()] - ) - events: Final[tuple[dict[str, object], ...]] = tuple( - [ - json.loads(event.replace("data: ", "").strip()) - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "How much did we spend?"}], - model="gpt-4o-mini", - is_admin=True, - ) - ] - ) - final_messages: Final = mock_litellm.acompletion.call_args_list[1].kwargs[ - "messages" - ] - - tool_messages: Final = [m for m in final_messages if m["role"] == "tool"] - assert len(tool_messages) == 1, tool_messages - assert tool_messages[0]["tool_call_id"] == "call_dates" - return str(tool_messages[0]["content"]), [ - e["status"] for e in events if e["type"] == "tool_call" - ] diff --git a/tests/unit/proxy/management_helpers/test_team_list_sorting.py b/tests/unit/proxy/management_helpers/test_team_list_sorting.py new file mode 100644 index 00000000000..bc4c77cc6a4 --- /dev/null +++ b/tests/unit/proxy/management_helpers/test_team_list_sorting.py @@ -0,0 +1,85 @@ +import json +import re +import sqlite3 +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import pytest +from pydantic import JsonValue + +from litellm.proxy.management_helpers.team_list_sorting import UnsupportedTeamFilter, _where_sql, sorted_team_ids + + +@pytest.mark.parametrize( + ("where", "expected"), + ( + ({}, ("a", "b", "c")), + ({"OR": []}, ()), + ({"AND": []}, ("a", "b", "c")), + ({"team_id": "a"}, ("a",)), + ({"team_alias": {"contains": "ALPHA", "mode": "insensitive"}}, ("a", "c")), + ({"team_id": {"startsWith": "b"}}, ("b",)), + ({"organization_id": {"in": ["org-1"]}}, ("a", "b")), + ( + { + "OR": [{"team_id": "b"}, {"team_alias": {"contains": "alpha", "mode": "insensitive"}}], + "AND": [{"OR": [{"organization_id": "org-1"}, {"team_id": "c"}]}], + "team_id": {"in": ["a", "c"]}, + }, + ("a", "c"), + ), + ({"team_id": "a' OR 1=1 --"}, ()), + ), +) +def test_team_filters_preserve_search_and_authorization_intersection( + where: dict[str, JsonValue], expected: tuple[str, ...] +) -> None: + result: Final = _where_sql(where) + assert not isinstance(result, UnsupportedTeamFilter) + predicate, params = result + sqlite_predicate: Final = re.sub( + r"= ANY\((\$\d+)::text\[\]\)", r"IN (SELECT value FROM json_each(\1))", predicate + ).replace(" ILIKE ", " LIKE ") + with sqlite3.connect(":memory:") as db: + db.execute("CREATE TABLE teams (team_id TEXT, team_alias TEXT, organization_id TEXT)") + db.executemany( + "INSERT INTO teams VALUES (?, ?, ?)", + (("a", "Alpha", "org-1"), ("b", "Beta", "org-1"), ("c", "Alpha", "org-2")), + ) + rows: Final = db.execute( + f"SELECT t.team_id FROM teams t WHERE {sqlite_predicate} ORDER BY t.team_id", + { + str(index): json.dumps(value) if isinstance(value, list) else value + for index, value in enumerate(params, 1) + }, + ).fetchall() + assert tuple(row[0] for row in rows) == expected + + +@pytest.mark.parametrize("where", ({"unknown": "x"}, {"team_id": {"not": "x"}}, {"AND": [False]})) +@pytest.mark.asyncio +async def test_unsupported_filter_fails_without_reading_unscoped_teams(where: dict[str, JsonValue]) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock() + result: Final = await sorted_team_ids(prisma, where=where, sort_by="keys_count", descending=False, skip=0, take=25) + assert isinstance(result, UnsupportedTeamFilter) + prisma.db.query_raw.assert_not_awaited() + + +@pytest.mark.parametrize("sort_by", ("keys_count", "members", "models", "organization_alias")) +@pytest.mark.parametrize("descending", (False, True)) +@pytest.mark.asyncio +async def test_sorted_ids_bind_search_and_pagination_without_interpolating_input( + sort_by: str, descending: bool +) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + search: Final = "x' OR TRUE --" + result: Final = await sorted_team_ids( + prisma, where={"team_id": search}, sort_by=sort_by, descending=descending, skip=25, take=25 + ) + assert result == () + sql, *bindings = prisma.db.query_raw.await_args.args + assert search not in sql + assert bindings == [search, 25, 25] + assert ("DESC" if descending else "ASC") + " NULLS LAST, t.team_id ASC" in sql diff --git a/tests/unit/proxy/management_helpers/test_user_list_sorting.py b/tests/unit/proxy/management_helpers/test_user_list_sorting.py new file mode 100644 index 00000000000..263c1691b23 --- /dev/null +++ b/tests/unit/proxy/management_helpers/test_user_list_sorting.py @@ -0,0 +1,71 @@ +from typing import Final, Literal +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from litellm.constants import UI_SESSION_TOKEN_TEAM_ID +from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids + + +@pytest.mark.parametrize("sort_by", ("status", "key_count")) +@pytest.mark.parametrize("descending", (False, True)) +@pytest.mark.asyncio +async def test_user_sort_binds_scope_search_and_page_and_excludes_ui_sessions( + sort_by: Literal["status", "key_count"], descending: bool +) -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + search: Final = "x' OR TRUE --" + result: Final = await sorted_user_ids( + prisma, + sort_by=sort_by, + descending=descending, + skip=25, + take=25, + role="internal_user", + user_ids=" a, b ", + user_email=search, + team="team-1", + sso_user_ids="sso-1, sso-2", + organization_ids="org-1, org-2", + search=search, + ) + assert result == () + sql, *bindings = prisma.db.query_raw.await_args.args + assert search not in sql + assert bindings == [ + "internal_user", + ("a", "b"), + search, + "team-1", + False, + ("sso-1", "sso-2"), + ("org-1", "org-2"), + search, + 25, + 25, + ] + ([UI_SESSION_TOKEN_TEAM_ID] if sort_by == "key_count" else []) + assert ("DESC" if descending else "ASC") + ", u.user_id ASC" in sql + assert ("k.team_id IS DISTINCT FROM $11::text" in sql) == (sort_by == "key_count") + + +@pytest.mark.asyncio +async def test_missing_user_filters_preserve_unfiltered_query() -> None: + prisma: Final = MagicMock() + prisma.db.query_raw = AsyncMock(return_value=[]) + result: Final = await sorted_user_ids( + prisma, + sort_by="status", + descending=False, + skip=0, + take=25, + role=None, + user_ids=None, + user_email=None, + team=None, + sso_user_ids=None, + organization_ids=None, + search=None, + ) + assert result == () + assert prisma.db.query_raw.await_args.args[1:] == (None, None, None, None, True, (), (), None, 25, 0) diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 8f27365fcc1..cc61b6f2fcf 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -1430,6 +1430,30 @@ async def test_get_autorouter_presets_fetches_once_per_process(monkeypatch, rese assert calls == ["https://example.test/presets.json"] +@pytest.mark.asyncio +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_AUTOROUTER_PRESETS", "LITELLM_OFFLINE_MODE"]) +async def test_get_autorouter_presets_local_flags_serve_bundled_without_fetching( + monkeypatch, reset_autorouter_presets_cache, flag +): + from litellm.proxy.public_endpoints.public_endpoints import ( + _load_bundled_autorouter_presets, + get_autorouter_presets, + ) + + monkeypatch.delenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", raising=False) + monkeypatch.setenv(flag, "True") + calls = [] + + async def fake_fetch(url): + calls.append(url) + raise AssertionError("no fetch when the bundled catalog is forced") + + presets = await get_autorouter_presets(url="https://example.test/presets.json", fetch=fake_fetch) + + assert presets == _load_bundled_autorouter_presets() + assert calls == [] + + @pytest.mark.asyncio async def test_get_autorouter_presets_single_flight_on_concurrent_cold_start( monkeypatch, reset_autorouter_presets_cache diff --git a/tests/unit/proxy/telemetry/test_endpoints.py b/tests/unit/proxy/telemetry/test_endpoints.py index 3b5f66da0ed..c962c8df6a5 100644 --- a/tests/unit/proxy/telemetry/test_endpoints.py +++ b/tests/unit/proxy/telemetry/test_endpoints.py @@ -1,13 +1,12 @@ -from dataclasses import dataclass, field from collections.abc import Mapping +from dataclasses import dataclass, field from typing import Final -from typing_extensions import LiteralString - import pytest from fastapi import FastAPI from fastapi.testclient import TestClient from httpx import ASGITransport, AsyncClient, Response +from typing_extensions import LiteralString from litellm.proxy._types import CommonProxyErrors, LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth diff --git a/tests/unit/proxy/telemetry/test_settings.py b/tests/unit/proxy/telemetry/test_settings.py index 0e9c180da4f..b88e6811113 100644 --- a/tests/unit/proxy/telemetry/test_settings.py +++ b/tests/unit/proxy/telemetry/test_settings.py @@ -1,7 +1,6 @@ from typing import Final import pytest - from pydantic import ValidationError from litellm.proxy.telemetry.settings import EnvPolicy, TelemetrySettings, describe_errors, env_policy, load_settings diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index adc4de8c8fa..0749fc12ea9 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -93,6 +93,20 @@ def test_retired_roi_calculator_cannot_be_loaded_or_called( assert response.status_code == 404, f"{method} {path}: {response.text}" +@pytest.mark.parametrize("disable_lazy_routes", ("false", "true")) +@pytest.mark.parametrize("path", ("/lazy/warm/usage_ai", "/usage/ai/chat")) +def test_retired_usage_chat_cannot_be_loaded_or_called( + monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str, path: str +) -> None: + monkeypatch.setenv(FLAG, disable_lazy_routes) + app: Final = FastAPI() + attach_lazy_features(app) + + with TestClient(app) as client: + response: Final = client.post(path) + assert response.status_code == 404, f"{path}: {response.text}" + + @pytest.mark.parametrize("value", ("1", "true", "TRUE", "yes", "on")) def test_flag_registers_every_feature_at_startup(monkeypatch: pytest.MonkeyPatch, value: str) -> None: monkeypatch.setenv(FLAG, value) diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index d9def23bde8..64ff8392c02 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -106,6 +106,13 @@ class TestGetMetadataVariableName: request = self._make_request("/v1/files") assert get_metadata_variable_name(request) == "litellm_metadata" + @pytest.mark.parametrize( + "path", ["/v1beta/interactions", "/interactions", "/v1beta/interactions/v1_abc123", "/interactions/v1_abc123/cancel"] + ) + def test_returns_litellm_metadata_for_interactions_routes(self, path): + request = self._make_request(path) + assert get_metadata_variable_name(request) == "litellm_metadata" + def test_returns_metadata_for_chat_completions(self): request = self._make_request("/chat/completions") assert get_metadata_variable_name(request) == "metadata" @@ -708,6 +715,46 @@ async def test_add_litellm_data_to_request_parses_json_object_string_litellm_met assert updated["litellm_metadata"]["cost_centre"] == "research" +@pytest.mark.asyncio +async def test_interactions_spend_logs_metadata_survives_the_router_metadata_bucket(): + """GH#47841: on /v1beta/interactions the router plants its deployment fields in + `litellm_metadata`. The proxy must put the request's metadata there too: otherwise the + spend log reads only the router's bucket and `spend_logs_metadata` from the + `x-litellm-spend-logs-metadata` header never reaches LiteLLM_SpendLogs.""" + request_mock = MagicMock(spec=Request) + request_mock.url = MagicMock() + request_mock.url.__str__.return_value = "http://localhost/v1beta/interactions" + request_mock.url.path = "/v1beta/interactions" + request_mock.method = "POST" + request_mock.query_params = {} + request_mock.headers = { + "Content-Type": "application/json", + "x-litellm-spend-logs-metadata": '{"cost_centre": "research"}', + } + request_mock.client = MagicMock() + request_mock.client.host = "127.0.0.1" + request_mock.state.parent_otel_span = None + + data = await add_litellm_data_to_request( + data={"model": "gemini-3.1-flash-lite", "input": "Reply OK.", "store": True}, + request=request_mock, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + assert data["litellm_metadata"]["spend_logs_metadata"] == {"cost_centre": "research"} + + # The router then adds its deployment fields to the same bucket, and the spend log + # resolves that bucket. + data["litellm_metadata"].update({"model_group": "gemini-3.1-flash-lite", "deployment": "gemini/gemini-3.1-flash-lite"}) + resolved = get_litellm_metadata_from_kwargs( + {"litellm_params": {"metadata": data.get("metadata") or {}, "litellm_metadata": data["litellm_metadata"]}} + ) + assert resolved["spend_logs_metadata"] == {"cost_centre": "research"} + assert resolved["model_group"] == "gemini-3.1-flash-lite" + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_proxy_server_request_body_is_post_strip(): """Regression: proxy_server_request['body'] used to be snapshotted before @@ -8064,6 +8111,67 @@ def test_client_anthropic_api_headers_stay_off_openai_compatible_providers(): assert forwarded == {} +def test_client_anthropic_api_headers_stay_off_azure_ai_outside_native_messages(): + forwarded = _headers_forwarded_to({"anthropic-beta": "thinking-binding-controls-2026-08-01"}, "azure_ai") + + assert forwarded == {} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("local_beta_headers_config") +async def test_client_thinking_binding_beta_reaches_azure_ai_foundry_claude(): + import httpx + + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + data: Final[dict] = {"model": "claude-opus-4-8"} # mutable-ok: the function under test writes into it + add_provider_specific_headers_to_request( + data=data, + headers={ + "content-type": "application/json", + "anthropic-beta": "thinking-binding-controls-2026-08-01", + "anthropic-version": "2026-08-01", + }, + ) + received: Final[list[httpx.Request]] = [] # mutable-ok: the fake Foundry endpoint records what the proxy sent + + def foundry_answers(request: httpx.Request) -> httpx.Response: + received.append(request) + return httpx.Response( + status_code=200, + json={ + "id": "msg_01", + "type": "message", + "role": "assistant", + "model": "claude-opus-4-8", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 12, "output_tokens": 6}, + }, + request=request, + ) + + upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers)) + try: + await litellm.anthropic.messages.acreate( + max_tokens=16, + messages=[{"role": "user", "content": "hi"}], + thinking={"type": "adaptive", "block_binding": {"prefix_mismatch_behavior": "drop_block"}}, + model="azure_ai/claude-opus-4-8", + api_key="foundry-key", + api_base="https://my-resource.services.ai.azure.com/anthropic", + client=upstream, + provider_specific_header=data["provider_specific_header"], + ) + finally: + await upstream.close() + + assert received[0].headers["anthropic-beta"] == "thinking-binding-controls-2026-08-01" + assert received[0].headers["anthropic-version"] == "2026-08-01" + assert json.loads(received[0].content)["thinking"]["block_binding"] == {"prefix_mismatch_behavior": "drop_block"} + + TYPESAFE_DANGER_ZONE_HEADERS = { "TypeSafe-Danger-Zone-Images": "true", "typesafe-danger-zone-long-context": "true", diff --git a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py index 47f81fb55d1..b5a41c31810 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py @@ -401,6 +401,26 @@ class TestLatestReleaseInfoEndpoint: assert response.status_code == 200 assert response.json() is None + @pytest.mark.parametrize("name", ["LITELLM_DISABLE_LATEST_RELEASE_CHECK", "LITELLM_OFFLINE_MODE"]) + @pytest.mark.parametrize( + ("flag", "expected_body", "expected_urls"), + [ + ("true", None, []), + (" TRUE ", None, []), + ("false", EXPECTED_INFO, [CHANGELOG_FEED_URL, _tag_url("v1.102.0")]), + ], + ) + def test_disable_flag_skips_every_outbound_request( + self, http_client, monkeypatch, name, flag, expected_body, expected_urls + ): + monkeypatch.setenv(name, flag) + client = _minor_release() + _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + response = http_client.get("/get/latest_release_info") + assert response.status_code == 200 + assert response.json() == expected_body + assert client.urls == expected_urls + def test_repeated_requests_reuse_cache(self, http_client): client = _minor_release() _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py index 747e8da8043..e335b08f040 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py @@ -12,15 +12,118 @@ import asyncio import hashlib import json import logging +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from litellm._service_logger import ServiceTypes +from litellm.models.user import LiteLLM_UserTable from litellm.proxy.db.log_db_metrics import record_db_io from litellm.proxy.utils import PrismaClient +from litellm.repositories.user_repository import free_sso_user_creation + + +class _SSOAdmissionUsers: + def __init__(self, count: int) -> None: + self.rows: Mapping[str, LiteLLM_UserTable] = { + f"user-{index}": LiteLLM_UserTable(user_id=f"user-{index}") + for index in range(count) + } + + async def find_unique( + self, where: Mapping[str, object] + ) -> LiteLLM_UserTable | None: + return self.rows.get(str(where["user_id"])) + + async def count(self, where: Mapping[str, object] | None = None) -> int: + if where is None: + return len(self.rows) + return sum((row.metadata or {}).get("scim_active") is False for row in self.rows.values()) + + async def upsert( + self, where: Mapping[str, object], data: Mapping[str, object] + ) -> LiteLLM_UserTable: + existing: Final = await self.find_unique(where) + if existing is not None: + return existing + created: Final = LiteLLM_UserTable.model_validate(data["create"]) + self.rows = {**self.rows, created.user_id: created} + return created + + +class _SSOAdmissionTransaction(AbstractAsyncContextManager["_SSOAdmissionTransaction"]): + def __init__(self, users: _SSOAdmissionUsers) -> None: + self.litellm_usertable: Final = users + + async def __aenter__(self) -> _SSOAdmissionTransaction: + return self + + async def __aexit__(self, *exc_info: object) -> None: + return None + + async def execute_raw(self, query: str) -> int: + return 0 + + async def query_raw(self, query: str) -> object: + return ({"locked": 1},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("existing_count", "scope_id", "user_id", "enabled", "child_task", "refused"), + [ + (4, "new-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, False, True), + (6, "user-0", "user-0", True, False, False), + (5, "sso-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, True, False), + (5, None, "new-user", True, False, True), + (5, None, "new-user", False, False, False), + ], +) +async def test_user_insert_enforces_free_sso_admission_only_for_its_owner( + prisma_client: PrismaClient, + monkeypatch: pytest.MonkeyPatch, + existing_count: int, + scope_id: str | None, + user_id: str, + enabled: bool, + child_task: bool, + refused: bool, +) -> None: + users: Final = _SSOAdmissionUsers(existing_count) + prisma_client.db.litellm_usertable = users + prisma_client.db.tx.return_value = _SSOAdmissionTransaction(users) + monkeypatch.setattr("asyncio.sleep", AsyncMock(return_value=None)) + original: Final = users.rows.get(user_id) + + async def attempt() -> None: + if refused: + with pytest.raises(HTTPException) as error: + await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert error.value.status_code == 403 + assert user_id not in users.rows + elif child_task: + result: Final = await asyncio.create_task( + prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + ) + assert result is users.rows[user_id] + else: + inserted: Final = await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert inserted is users.rows[user_id] + + with free_sso_user_creation(scope_id, enabled=enabled): + for _ in range(2): + await attempt() + + assert len(users.rows) == existing_count + int(not refused and original is None) + if original is not None: + assert users.rows[user_id] is original @pytest.mark.asyncio diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 436367bd82d..36b5d8f135b 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -4,6 +4,7 @@ over the wire and surface provider errors correctly. Expected JSON bodies are st in expected_responses_api_request/. """ +import asyncio import copy import json import logging @@ -1231,3 +1232,110 @@ def test_faked_stream_accepts_a_text_timeout(monkeypatch: pytest.MonkeyPatch) -> assert [event.response.output[0].content[0].text for event in response if event.type == "response.completed"] == [ "Done." ] + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("api_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/inference/go/openai/v1", None, "conversation-123"), + ("https://opencode.ai/inference/go/openai/v1", "caller-override", "caller-override"), + ("https://other.example/v1", None, None), + ], +) +@pytest.mark.asyncio +async def test_responses_opencode_session_header( + asynchronous: bool, stream: bool, api_base: str, explicit_header: str | None, expected_session: str | None +) -> None: + received: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + payload: Final = _minimal_responses_api_payload("resp_test", "gpt-6-luna") + + def respond(request: httpx.Request) -> httpx.Response: + received.put_nowait(request) + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text="".join( + f"event: {kind}\ndata: {json.dumps({'type': kind, 'response': payload})}\n\n" + for kind in ("response.created", "response.completed") + ), + ) + if stream + else httpx.Response(200, json=payload) + ) + + args: Final = { + "model": "openai/gpt-6-luna", + "input": "hello", + "api_base": api_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + "stream": stream, + "extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + } + if asynchronous: + async_client: Final = AsyncHTTPHandler(transport=httpx.MockTransport(respond)) + try: + response: Final = await litellm.aresponses(**args, client=async_client) + if stream: + events: Final = tuple([event.type async for event in response]) + assert "response.completed" in events + else: + assert response.output[0].content[0].text == "Done." + finally: + await async_client.close() + else: + sync_client: Final = HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(respond))) + try: + result: Final = litellm.responses(**args, client=sync_client) + if stream: + assert "response.completed" in tuple(event.type for event in result) + else: + assert result.output[0].content[0].text == "Done." + finally: + sync_client.close() + sent: Final = received.get_nowait() + assert str(sent.url) == f"{api_base}/responses" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello", "stream": stream} + assert received.empty() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"]) +@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"]) +@pytest.mark.asyncio +async def test_responses_opencode_session_uses_effective_destination( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + asynchronous: bool, + source: str, + explicit_base: str | None, +) -> None: + default_base: Final = "https://opencode.ai/inference/go/openai/v1" + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + if source != "litellm.api_base": + monkeypatch.setenv(source, default_base) + endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/responses").respond( + json=_minimal_responses_api_payload("resp_test", "gpt-6-luna") + ) + args: Final = { + "model": "openai/gpt-6-luna", + "input": "hello", + "api_base": explicit_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + } + response: Final = await litellm.aresponses(**args) if asynchronous else litellm.responses(**args) + assert response.output[0].content[0].text == "Done." + assert endpoint.call_count == 1 + sent: Final = endpoint.calls[0].request + assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "gpt-6-luna", "input": "hello"} diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 726128386d5..78722458cf3 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -116,6 +116,9 @@ def _wrap_nested_level(value: object, depth: int) -> object: (responses_detectors.provider_affinity, {"provider_affinity_header": ""}, True), (responses_detectors.provider_affinity, {"provider_affinity_header": None, "session_id": "session-1"}, False), (responses_detectors.provider_affinity, {}, False), + (responses_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/openai/v1"}, True), + (messages_detectors.provider_affinity, {"api_base": "https://opencode.ai/inference/go/anthropic/v1"}, True), + (messages_detectors.provider_affinity, {"api_base": "https://other.example/v1"}, False), (detectors.streaming, {"stream": True}, True), (detectors.streaming, {"stream": False}, False), (detectors.timeout_variants, {"request_timeout": 5}, True), diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index 2a85122d91c..deba7fe9587 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -17,7 +17,9 @@ import pytest import litellm from litellm.anthropic_beta_headers_manager import ( + GetAnthropicBetaHeadersConfig, filter_and_transform_beta_headers, + get_beta_headers_config, update_headers_with_filtered_beta, update_request_with_filtered_beta, ) @@ -429,13 +431,13 @@ class TestAnthropicBetaHeadersFiltering: assert filtered == ["fine-grained-tool-streaming-2025-05-14"] @pytest.mark.parametrize( - "provider", ["anthropic", "bedrock", "bedrock_converse", "vertex_ai", "databricks"] + "provider", ["anthropic", "azure_ai", "bedrock", "bedrock_converse", "vertex_ai", "databricks"] ) def test_thinking_binding_controls_forwarded(self, provider): """`thinking.block_binding` (preserved thinking, Claude Fable 5.1) is only accepted alongside thinking-binding-controls-2026-08-01. The body field is forwarded untouched, so stripping the header (previously unknown, hence - dropped) makes Bedrock and Vertex reject the request with + dropped) makes Bedrock, Vertex and Azure AI Foundry reject the request with "thinking.adaptive.block_binding: Extra inputs are not permitted".""" filtered = filter_and_transform_beta_headers( beta_headers=["thinking-binding-controls-2026-08-01"], @@ -558,3 +560,13 @@ class TestAnthropicBetaHeadersFiltering: headers = {"anthropic-version": "2023-06-01"} assert update_headers_with_filtered_beta(headers, "bedrock_mantle") == {"anthropic-version": "2023-06-01"} + + +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "LITELLM_OFFLINE_MODE"]) +def test_local_flags_load_bundled_beta_headers_without_fetching(monkeypatch, flag): + monkeypatch.delenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", raising=False) + monkeypatch.setenv(flag, "True") + with patch("litellm.anthropic_beta_headers_manager.httpx.get") as remote_get: + config = get_beta_headers_config(url="https://example.test/anthropic_beta_headers_config.json") + remote_get.assert_not_called() + assert config == GetAnthropicBetaHeadersConfig.load_local_beta_headers_config() diff --git a/tests/unit/test_elevenlabs_tts_pricing.py b/tests/unit/test_elevenlabs_tts_pricing.py new file mode 100644 index 00000000000..16f726b9d6c --- /dev/null +++ b/tests/unit/test_elevenlabs_tts_pricing.py @@ -0,0 +1,49 @@ +"""ElevenLabs text-to-speech prices match the published API rates. + +https://elevenlabs.io/pricing/api lists "v3" and "v2 Multilingual" at $0.08 and +"Flash / Turbo" at $0.04, each "Price per 1K characters", and says API usage is +billed in US dollars, not credits. The map had v3 and Multilingual v2 at a +Scale-plan credit estimate ($0.18 per 1K characters) and no Flash rows. +""" + +import json +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +PRICES = { + "elevenlabs/eleven_v3": 8e-05, + "elevenlabs/eleven_multilingual_v2": 8e-05, + "elevenlabs/eleven_flash_v2_5": 4e-05, + "elevenlabs/eleven_flash_v2": 4e-05, +} + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """Never depend on the published cost map having these rows yet.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_metadata_and_backup(model, per_character): + main = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + backup = json.loads((REPO_ROOT / "litellm/model_prices_and_context_window_backup.json").read_text()) + assert main[model] == backup[model] + info = litellm.get_model_info(model) + assert info["litellm_provider"] == "elevenlabs" + assert info["mode"] == "audio_speech" + assert info["input_cost_per_character"] == per_character + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_speech_cost_is_charged_per_character(model, per_character): + cost = litellm.completion_cost(model=model, call_type="speech", prompt="x" * 1000) + assert cost == pytest.approx(per_character * 1000) diff --git a/tests/unit/test_get_blog_posts.py b/tests/unit/test_get_blog_posts.py index 499dd9bcf42..d1de75a4500 100644 --- a/tests/unit/test_get_blog_posts.py +++ b/tests/unit/test_get_blog_posts.py @@ -190,8 +190,9 @@ def test_get_blog_posts_ttl_expired_refetches(): assert len(posts) == 2 -def test_get_blog_posts_local_env_var_skips_remote(monkeypatch): - monkeypatch.setenv("LITELLM_LOCAL_BLOG_POSTS", "true") +@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_BLOG_POSTS", "LITELLM_OFFLINE_MODE"]) +def test_get_blog_posts_local_env_var_skips_remote(monkeypatch, flag): + monkeypatch.setenv(flag, "true") with patch("litellm.litellm_core_utils.get_blog_posts.httpx.get") as mock_get: posts = get_blog_posts(url=litellm.blog_posts_url) mock_get.assert_not_called() diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index c3d3575b8b3..70b268bf2a2 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -20,7 +20,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest import respx -from openai import APITimeoutError, AsyncOpenAI +from openai import APITimeoutError, AsyncOpenAI, OpenAI from openai._legacy_response import LegacyAPIResponse from openai.types.chat.chat_completion import ChatCompletion from pydantic import TypeAdapter @@ -95,6 +95,142 @@ def preserve_litellm_completion_state(monkeypatch: pytest.MonkeyPatch) -> None: WHITE_PNG: Final = (Path(__file__).parents[1] / "white_100x100.png").read_bytes() +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.parametrize( + ("client_base", "explicit_header", "expected_session"), + [ + ("https://opencode.ai/zen/go/v1", None, "conversation-123"), + ("https://opencode.ai/zen/go/v1", "caller-override", "caller-override"), + ("https://other.example/v1", None, None), + ], +) +@pytest.mark.asyncio +async def test_completion_sends_session_only_to_the_actual_opencode_destination( + asynchronous: bool, + stream: bool, + client_base: str, + explicit_header: str | None, + expected_session: str | None, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_BASE_URL", "https://opencode.ai/zen/go/v1") + monkeypatch.setenv("OPENAI_API_BASE", "https://opencode.ai/zen/go/v1") + monkeypatch.setattr(litellm, "api_base", "https://opencode.ai/zen/go/v1") + requests: Final[asyncio.Queue[httpx.Request]] = asyncio.Queue() + + def respond(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + payload: Final = { + "id": "chatcmpl-test", + "object": "chat.completion.chunk" if stream else "chat.completion", + "created": 1, + "model": "session-test", + "choices": [ + { + "index": 0, + "delta" if stream else "message": {"role": "assistant", "content": "OK"}, + "finish_reason": "stop", + } + ], + } + return ( + httpx.Response( + 200, + headers={"content-type": "text/event-stream"}, + text=f"data: {json.dumps(payload)}\n\ndata: [DONE]\n\n", + ) + if stream + else httpx.Response(200, json=payload) + ) + + args: Final = { + "model": "openai/session-test", + "api_base": "https://opencode.ai/zen/go/v1", + "messages": [{"role": "user", "content": "hello"}], + "litellm_session_id": "conversation-123", + "extra_headers": {"X-OpenCode-Session": explicit_header} if explicit_header else {"x-existing": "unchanged"}, + "stream": stream, + } + if asynchronous: + async with AsyncOpenAI( + api_key="test-key", + base_url=client_base, + http_client=httpx.AsyncClient(transport=httpx.MockTransport(respond)), + ) as async_client: + response: Final = await acompletion(**args, client=async_client) + content: Final = ( + "".join([chunk.choices[0].delta.content or "" async for chunk in response]) + if stream + else response.choices[0].message.content + ) + assert content == "OK" + else: + with OpenAI( + api_key="test-key", base_url=client_base, http_client=httpx.Client(transport=httpx.MockTransport(respond)) + ) as sync_client: + result: Final = completion(**args, client=sync_client) + text: Final = ( + "".join(chunk.choices[0].delta.content or "" for chunk in result) + if stream + else result.choices[0].message.content + ) + assert text == "OK" + sent: Final = requests.get_nowait() + assert str(sent.url) == f"{client_base}/chat/completions" + assert sent.headers.get("x-opencode-session") == expected_session + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == { + "model": "session-test", + "messages": args["messages"], + **({"stream": True} if stream else {}), + } + assert requests.empty() + + +@pytest.mark.parametrize("asynchronous", [False, True]) +@pytest.mark.parametrize("source", ["OPENAI_BASE_URL", "OPENAI_API_BASE", "litellm.api_base"]) +@pytest.mark.parametrize("explicit_base", [None, "https://other.example/v1"]) +@pytest.mark.asyncio +async def test_completion_opencode_session_uses_effective_destination( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, + asynchronous: bool, + source: str, + explicit_base: str | None, +) -> None: + default_base: Final = "https://opencode.ai/inference/go/openai/v1" + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "api_base", default_base if source == "litellm.api_base" else None) + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + if source != "litellm.api_base": + monkeypatch.setenv(source, default_base) + endpoint: Final = respx_mock.post(f"{explicit_base or default_base}/chat/completions").respond( + json={ + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1, + "model": "session-test", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "OK"}, "finish_reason": "stop"}], + } + ) + args: Final = { + "model": "openai/session-test", + "messages": [{"role": "user", "content": "hello"}], + "api_base": explicit_base, + "api_key": "test-key", + "litellm_session_id": "conversation-123", + } + response: Final = await litellm.acompletion(**args) if asynchronous else litellm.completion(**args) + assert response.choices[0].message.content == "OK" + assert endpoint.call_count == 1 + sent: Final = endpoint.calls[0].request + assert sent.headers.get("x-opencode-session") == (None if explicit_base else "conversation-123") + assert sent.headers["authorization"] == "Bearer test-key" + assert json.loads(sent.content) == {"model": "session-test", "messages": args["messages"]} + + @pytest.fixture def openai_api_response(): mock_response_data = { diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cdff03af3b4..ad069aa987b 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -90,7 +90,6 @@ from litellm.utils import ( get_llm_provider, get_optional_params_embeddings, ProviderConfigManager, - TextCompletionStreamWrapper, _check_provider_match, get_potential_model_names, _is_litellm_router_call, @@ -2528,41 +2527,6 @@ def test_priced_router_deployment_id_stays_out_of_wildcard_model_listing(): assert litellm.model_cost[deployment_id]["input_cost_per_token"] == 0.000001 -def test_reasoning_content_preserved_in_text_completion_wrapper(): - """Ensure reasoning_content is copied from delta to text_choices.""" - chunk = ModelResponseStream( - id="test-id", - created=1234567890, - model="test-model", - object="chat.completion.chunk", - choices=[ - StreamingChoices( - finish_reason=None, - index=0, - delta=Delta( - content="Some answer text", - role="assistant", - reasoning_content="Here's my chain of thought...", - ), - ) - ], - ) - - wrapper = TextCompletionStreamWrapper( - completion_stream=None, # Not used in convert_to_text_completion_object - model="test-model", - stream_options=None, - ) - - transformed = wrapper.convert_to_text_completion_object(chunk) - - assert "choices" in transformed - assert len(transformed["choices"]) == 1 - choice = transformed["choices"][0] - assert choice["text"] == "Some answer text" - assert choice["reasoning_content"] == "Here's my chain of thought..." - - def test_anthropic_claude_4_invoke_chat_provider_config(): """Test that the Anthropic Claude 4 Invoke chat provider config is correct.""" from litellm.llms.bedrock.chat.invoke_transformations.anthropic_claude3_transformation import ( diff --git a/tests/unit/text_completions/__init__.py b/tests/unit/text_completions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/text_completions/test_streaming_iterator.py b/tests/unit/text_completions/test_streaming_iterator.py new file mode 100644 index 00000000000..b7edffc0753 --- /dev/null +++ b/tests/unit/text_completions/test_streaming_iterator.py @@ -0,0 +1,144 @@ +from collections.abc import AsyncIterator, Iterator +from typing import Final + +import pytest + +from litellm.exceptions import APIConnectionError +from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper +from litellm.types.utils import Choices, Delta, ModelResponseStream, StreamingChoices, Usage + + +def test_reasoning_content_preserved_in_text_completion_wrapper(): + """Ensure reasoning_content is copied from delta to text_choices.""" + chunk = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="Some answer text", + role="assistant", + reasoning_content="Here's my chain of thought...", + ), + ) + ], + ) + + wrapper = TextCompletionStreamWrapper( + completion_stream=None, # Not used in convert_to_text_completion_object + model="test-model", + stream_options=None, + ) + + transformed = wrapper.convert_to_text_completion_object(chunk) + + assert "choices" in transformed + assert len(transformed["choices"]) == 1 + choice = transformed["choices"][0] + assert choice["text"] == "Some answer text" + assert choice["reasoning_content"] == "Here's my chain of thought..." + + +@pytest.mark.asyncio +@pytest.mark.parametrize("async_stream", (False, True)) +@pytest.mark.parametrize("empty_choices", (False, True)) +@pytest.mark.parametrize("stream_options", (None, {}, {"include_usage": False}, {"include_usage": True})) +@pytest.mark.parametrize("usage", (None, Usage(prompt_tokens=7, completion_tokens=2, total_tokens=9), Usage())) +async def test_text_completion_stream_usage_is_opt_in( + async_stream: bool, empty_choices: bool, stream_options: dict[str, bool] | None, usage: Usage | None +) -> None: + chunk: Final = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + choices=[] if empty_choices else [StreamingChoices(index=0, delta=Delta(content="Hello"))], + usage=usage, + ) + + async def chunks() -> AsyncIterator[ModelResponseStream]: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks() if async_stream else iter((chunk,)), + model="test-model", + stream_options=stream_options, + ) + responses: Final = [response async for response in wrapper] if async_stream else list(wrapper) + expected_usage: Final = ( + {"usage": usage.model_dump(exclude_none=True)} + if stream_options and stream_options.get("include_usage") is True and usage is not None + else {} + ) + assert [response.model_dump(exclude_none=True) for response in responses] == [ + { + "id": "test-id", + "object": "text_completion", + "created": 1234567890, + "model": "test-model", + "choices": [] if empty_choices else [{"index": 0, "text": "Hello"}], + **expected_usage, + } + ] + + +def test_text_completion_conversion_rejects_non_streaming_choices() -> None: + chunk: Final = Choices() + wrapper: Final = TextCompletionStreamWrapper(completion_stream=None, model="test-model") + + with pytest.raises(Exception, match="Error occurred converting to text completion object") as error: + wrapper.convert_to_text_completion_object(chunk) + + assert str(chunk) in str(error.value) + + +@pytest.mark.parametrize("chunk", (None, "None")) +def test_sync_text_completion_stream_rejects_invalid_chunks(chunk: str | None) -> None: + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=iter((chunk,)), model="test-model", custom_llm_provider="openai" + ) + + with pytest.raises(APIConnectionError, match="OpenAIException") as error: + next(wrapper) + + assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("chunk", (None, "None")) +async def test_async_text_completion_stream_rejects_invalid_chunks(chunk: str | None) -> None: + async def chunks() -> AsyncIterator[str | None]: + yield chunk + + wrapper: Final = TextCompletionStreamWrapper(completion_stream=chunks(), model="test-model") + + with pytest.raises(Exception, match=r"^$") as error: + await anext(wrapper) + + assert not isinstance(error.value, StopAsyncIteration) + + +def test_sync_text_completion_stream_maps_upstream_failure_after_content() -> None: + chunk: Final = ModelResponseStream( + id="test-id", + created=1234567890, + model="test-model", + choices=[StreamingChoices(index=0, delta=Delta(content="Hello"))], + ) + + def chunks() -> Iterator[ModelResponseStream]: + yield chunk + raise RuntimeError("upstream disconnected") + + wrapper: Final = TextCompletionStreamWrapper( + completion_stream=chunks(), model="test-model", custom_llm_provider="openai" + ) + assert next(wrapper).choices[0].text == "Hello" + + with pytest.raises(APIConnectionError, match="upstream disconnected") as error: + next(wrapper) + + assert (error.value.model, error.value.llm_provider) == ("test-model", "openai") diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 1451256631f..38d4c435e9a 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1098,14 +1098,6 @@ "count": 1 } }, - "src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - } - }, "src/app/(dashboard)/usage/_components/components/UsagePageView.tsx": { "react-hooks/purity": { "count": 1 @@ -1767,13 +1759,13 @@ "count": 1 }, "max-params": { - "count": 15 + "count": 14 }, "no-nested-ternary": { "count": 5 }, "no-restricted-syntax": { - "count": 140 + "count": 139 }, "prefer-const": { "count": 28 diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index add749c88cb..8704dc55c38 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -54,7 +54,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -12876,9 +12876,9 @@ } }, "node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 8160836515f..b1b8fde44ee 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -71,7 +71,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -107,6 +107,10 @@ "vitest": "4.1.12" }, "overrides": { + "@litellm/lens-ui": { + "next": "$next" + }, + "zod": "$zod", "prismjs": "1.30.0", "js-yaml": "4.3.2", "brace-expansion": "5.0.12", @@ -117,10 +121,7 @@ "axios": "1.13.6", "postcss": "8.5.23", "esbuild": "0.28.1", - "sharp": "^0.35.5", - "@litellm/lens-ui": { - "next": "$next" - } + "sharp": "^0.35.5" }, "engines": { "node": ">=24.14.1", diff --git a/ui/litellm-dashboard/public/assets/aria-agent.png b/ui/litellm-dashboard/public/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/aria-agent.png differ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx index ae898645de4..b4fb7586e6d 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTable.test.tsx @@ -134,13 +134,16 @@ describe("BudgetTable", () => { } }); - it("should not make the reset column sortable", async () => { + it("should request duration sorting from the server", async () => { const user = userEvent.setup(); - renderWithProviders(); + const onSortingChange = vi.fn(); + renderWithProviders(); await showColumn(user, "budget_duration"); const headers = screen.getAllByRole("columnheader").map((header) => header.textContent); expect(headers).toContain("Reset"); - expect(screen.queryByTestId("sort-header-budget_duration")).not.toBeInTheDocument(); + await user.click(screen.getByTestId("sort-header-budget_duration")); + expect(onSortingChange).toHaveBeenCalled(); + expect(onSortingChange.mock.calls[0][0]([])).toEqual([{ id: "budget_duration", desc: false }]); }); it("should ask the list for a new sort when a sortable header is clicked", async () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx index fb894322208..3ec291c1158 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/budgets/_components/BudgetTableColumns.tsx @@ -139,8 +139,7 @@ export const getBudgetTableColumns = ({ accessorKey: "budget_duration", filterFn: serverFilter, meta: { title: "Reset" }, - // "7d"/"30d" sort lexicographically, not chronologically, so the route does not offer it. - enableSorting: false, + sortDescFirst: false, header: ({ column }) => , size: 110, cell: ({ row }) => , diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx index b957393be64..1891a580088 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx @@ -49,6 +49,9 @@ const requestUrl = (input: RequestInfo | URL): string => { }; const fakeFetch = (input: RequestInfo | URL) => { + if (requestUrl(input).endsWith("/models")) { + return Promise.resolve(Response.json({ data: [{ id: "claude-sonnet-5-5" }] })); + } if (requestUrl(input).includes("/model_group/info")) { return Promise.resolve(Response.json({ data: [{ model_group: "claude-sonnet-5-5", mode: "chat" }] })); } @@ -141,7 +144,7 @@ describe("HomePage", () => { expect(screen.queryByRole("link", { name: /create new virtual key/i })).not.toBeInTheDocument(); }); - it("offers LiteAdmin on Home inside the dashboard frame", async () => { + it("should offer Aria Agent on Home inside the dashboard frame", async () => { render( @@ -149,7 +152,7 @@ describe("HomePage", () => { , ); - const entry = screen.getByRole("region", { name: "Ask LiteAdmin" }); + const entry = screen.getByRole("region", { name: "Ask Aria Agent" }); expect(await within(entry).findByPlaceholderText("What would you like to do?")).toBeInTheDocument(); expect(within(entry).getByRole("button", { name: "Send message" })).toBeDisabled(); expect(await screen.findByText("$501.74")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx index 7ae7e976651..ad08ff48a5b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/PassThroughPanel.tsx @@ -4,8 +4,6 @@ import PassThroughSettings from "@/components/PassThroughSettings/PassThroughSet import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; export default function PassThroughPanel() { - const { accessToken, userRole, userId: userID, premiumUser } = useAuthorized(); - return ( - - ); + const { accessToken, userRole, userId: userID } = useAuthorized(); + return ; } diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx index 839eb406200..724a72a77f3 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.test.tsx @@ -66,6 +66,85 @@ const baseProps = { }; describe("OrganizationsTable", () => { + it.each(["max_budget", "tpm_limit", "rpm_limit"])( + "sorts %s numerically with Unlimited as infinity", + async (field) => { + const user = userEvent.setup(); + const organizations = [20, 3, null, 100, 0].map((value, index) => + makeOrganization({ + organization_id: `org-${index}`, + organization_alias: `Value ${value}`, + litellm_budget_table: { [field]: value }, + }), + ); + renderWithProviders(, { + searchParams: { sort_by: field, sort_order: "asc", page_size: "25" }, + }); + const names = () => + screen + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getByText(/^Value /).textContent); + expect(names()).toEqual(["Value 0", "Value 3", "Value 20", "Value 100", "Value null"]); + if (field === "max_budget") { + await user.click(screen.getByTestId("sort-header-max_budget")); + } else { + await user.click(screen.getByRole("button", { name: "Sort options for TPM or RPM" })); + await user.click( + await screen.findByRole("menuitem", { name: `${field === "tpm_limit" ? "TPM" : "RPM"} descending` }), + ); + } + await waitFor(() => expect(names()).toEqual(["Value null", "Value 100", "Value 20", "Value 3", "Value 0"])); + }, + ); + + it.each(["asc", "desc"])("groups both All Proxy Models representations when sorting %s", (direction) => { + const organizations = [ + makeOrganization({ organization_id: "org-empty", organization_alias: "Name Empty", models: [] }), + makeOrganization({ organization_id: "org-z", organization_alias: "Name Z", models: ["z"] }), + makeOrganization({ + organization_id: "org-wildcard", + organization_alias: "Name Wildcard", + models: ["all-proxy-models"], + }), + makeOrganization({ organization_id: "org-a", organization_alias: "Name A", models: ["a"] }), + ]; + renderWithProviders(, { + searchParams: { sort_by: "models", sort_order: direction }, + }); + const names = screen + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getByText(/^Name /).textContent); + const models = direction === "asc" ? ["Name A", "Name Z"] : ["Name Z", "Name A"]; + expect(names).toEqual([...models, "Name Empty", "Name Wildcard"]); + expect(screen.getAllByText("All Proxy Models")).toHaveLength(2); + }); + + it.each(["models", "members"])("sorts %s by its displayed data", async (field) => { + const user = userEvent.setup(); + const organizations = [ + { organization_id: "org-zed", organization_alias: "Zed", models: ["z"], members: [] }, + { + organization_id: "org-ace", + organization_alias: "Ace", + models: ["a"], + members: [{ user_id: "1" }, { user_id: "2" }], + }, + { + organization_id: "org-mid", + organization_alias: "Mid", + models: ["m"], + members: [{ user_id: "3" }], + }, + ].map(makeOrganization); + renderWithProviders(); + await user.click(screen.getByTestId(`sort-header-${field}`)); + await waitFor(() => expect(bodyRowAliases()).toEqual(["Ace", "Mid", "Zed"])); + await user.click(screen.getByTestId(`sort-header-${field}`)); + await waitFor(() => expect(bodyRowAliases()).toEqual(["Zed", "Mid", "Ace"])); + }); + it("renders every column header", () => { renderWithProviders(); for (const header of [ @@ -75,7 +154,6 @@ describe("OrganizationsTable", () => { "Spend (USD)", "Budget (USD)", "Models", - "TPM / RPM Limits", "Members", ]) { expect(screen.getByText(header)).toBeInTheDocument(); @@ -87,7 +165,7 @@ describe("OrganizationsTable", () => { for (const header of ["Spend (USD)", "Budget (USD)", "Members"]) { expect(screen.getByRole("columnheader", { name: header })).toHaveClass("text-right"); } - for (const header of ["Organization Name", "TPM / RPM Limits"]) { + for (const header of ["Organization Name"]) { expect(screen.getByRole("columnheader", { name: header })).not.toHaveClass("text-right"); } }); @@ -270,7 +348,7 @@ describe("OrganizationsTable URL state", () => { it("falls back to sorting by creation date for a ?sort_by= column that cannot be sorted", () => { renderWithProviders(, { - searchParams: "?sort_by=members&sort_order=asc", + searchParams: "?sort_by=actions&sort_order=asc", }); expect(bodyRowAliases()).toEqual(["Zed", "Mid", "Ace"]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx index a9ac0b7e699..df1c475ebb1 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTable.tsx @@ -62,6 +62,7 @@ const OrganizationsTable: React.FC = ({ onPaginationChange={onPaginationChange} columns={columns} getRowId={(organization, index) => organization.organization_id || String(index)} + defaultColumnVisibility={{ tpm_limit: false, rpm_limit: false }} sortingMode="client" sorting={sorting} onSortingChange={onSortingChange} diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx index 5f170a32941..dcbf141314b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/OrganizationsTableColumns.tsx @@ -3,7 +3,7 @@ import { ColumnDef } from "@tanstack/react-table"; import { MoreHorizontal, Pencil, Trash2 } from "lucide-react"; -import { DataTableSortHeader } from "@/components/shared/DataTable"; +import { DataTableMultiSortHeader, DataTableSortHeader } from "@/components/shared/DataTable"; import { DateCell, IdentityCell, ModelsCell, MoneyCell } from "@/components/shared/table_cells"; import { Organization } from "@/components/networking"; import { buttonVariants } from "@/components/ui/button"; @@ -137,38 +137,67 @@ export const getOrganizationsTableColumns = ({ }, { id: "max_budget", + accessorFn: (organization) => getOrganizationBudget(organization).max_budget ?? Number.POSITIVE_INFINITY, + sortUndefined: "last", + sortDescFirst: true, meta: { title: "Budget (USD)", numeric: true }, - header: "Budget (USD)", + header: ({ column }) => , size: 120, - enableSorting: false, + enableSorting: true, cell: ({ row }) => ( ), }, { id: "models", + accessorFn: (organization) => + organization.models?.length && !organization.models.includes("all-proxy-models") + ? [...organization.models].sort().join(", ") + : undefined, + sortUndefined: "last", + sortDescFirst: false, meta: { title: "Models", skeleton: "chips" }, - header: "Models", + header: ({ column }) => , size: 260, - enableSorting: false, + enableSorting: true, cell: ({ row }) => , }, { id: "limits", meta: { title: "TPM / RPM Limits" }, - header: "TPM / RPM Limits", + header: ({ table }) => ( + + ), size: 150, enableSorting: false, cell: ({ row }) => , }, { id: "members", + accessorFn: (organization) => organization.members?.length ?? 0, + sortUndefined: "last", + sortDescFirst: true, meta: { title: "Members", numeric: true }, - header: "Members", + header: ({ column }) => , size: 100, - enableSorting: false, + enableSorting: true, cell: ({ row }) => {row.original.members?.length ?? 0} Members, }, + ...(["tpm_limit", "rpm_limit"] as const).map( + (field): ColumnDef => ({ + id: field, + accessorFn: (organization) => getOrganizationBudget(organization)[field] ?? Number.POSITIVE_INFINITY, + sortUndefined: "last", + sortDescFirst: true, + enableHiding: false, + }), + ), { id: "actions", meta: { className: "text-right", headerClassName: "text-right" }, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts index 20a54a25ba1..ffe67c0aa14 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/organizations/_components/useOrganizationsTableState.ts @@ -4,7 +4,17 @@ const FILTER_COLUMNS = ["org_id"] as const; type FilterColumn = (typeof FILTER_COLUMNS)[number]; const TABLE_STATE_OPTIONS: UrlTableStateOptions = { - sortFields: ["organization_id", "organization_alias", "created_at", "spend"], + sortFields: [ + "organization_id", + "organization_alias", + "created_at", + "spend", + "max_budget", + "models", + "members", + "tpm_limit", + "rpm_limit", + ], defaultSort: { id: "created_at", desc: true }, defaultPageSize: 25, filterColumns: FILTER_COLUMNS, diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx index da2e4debb3c..4903a431ff7 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.test.tsx @@ -1,4 +1,5 @@ import { fireEvent, render, screen } from "@testing-library/react"; +import userEvent from "@testing-library/user-event"; import { describe, expect, it, vi } from "vitest"; import { ChatComposer } from "./ChatComposer"; @@ -111,4 +112,23 @@ describe("ChatComposer", () => { expect(screen.getByTestId("chat-composer-input")).toHaveFocus(); }); + + it.each([ + { label: "Find a model", control: }, + { + label: "Choose a model", + control: ( + + ), + }, + ])("should preserve focus when clicking the $label toolbar control", async ({ label, control }) => { + renderComposer({ tools: control }); + + await userEvent.setup().click(screen.getByLabelText(label)); + + expect(screen.getByLabelText(label)).toHaveFocus(); + expect(screen.getByPlaceholderText("Send a message")).not.toHaveFocus(); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx index 5d2edbdfc94..c33b5e992fc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/chat_ui/ChatComposer.tsx @@ -51,6 +51,14 @@ export function ChatComposer({ } }; + const handleToolbarClick = (event: React.MouseEvent) => { + const control = (event.target as HTMLElement).closest( + "button, input, select, textarea, a[href], [contenteditable='true'], [tabindex]", + ); + if (event.defaultPrevented || (control && event.currentTarget.contains(control))) return; + event.currentTarget.parentElement?.querySelector("[data-slot=input-group-control]")?.focus(); + }; + return (
{showSuggestions && suggestions.length > 0 && ( @@ -99,12 +107,7 @@ export function ChatComposer({ { - if ((event.target as HTMLElement).closest("button")) { - return; - } - event.currentTarget.parentElement?.querySelector("[data-slot=input-group-control]")?.focus(); - }} + onClick={handleToolbarClick} >
{tools}
diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx index 0fff919de4d..39627329b85 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx @@ -147,6 +147,10 @@ describe("ClientUsagePanel", () => { ); vi.mocked(userListCall).mockResolvedValue(userPage); vi.mocked(uiSpendLogsCall).mockResolvedValue(requestPage); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ results: [] })), + ); }); afterEach(() => vi.unstubAllGlobals()); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx deleted file mode 100644 index d971f5d0506..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { screen } from "@testing-library/react"; -import { beforeAll, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; -import UsageAIChatPanel from "./UsageAIChatPanel"; - -beforeAll(() => { - if (typeof window !== "undefined" && !window.ResizeObserver) { - window.ResizeObserver = class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} - } as any; - } -}); - -vi.mock("@/components/networking", () => ({ - modelHubCall: vi.fn().mockResolvedValue({ - data: [{ model_group: "gpt-4" }, { model_group: "claude-3-opus" }], - }), - usageAiChatStream: vi.fn(), -})); - -const defaultProps = { - open: true, - onClose: vi.fn(), - accessToken: "test-token", -}; - -describe("UsageAIChatPanel", () => { - it("should render the panel when open", () => { - renderWithProviders(); - - expect(screen.getByText("Ask AI")).toBeInTheDocument(); - expect(screen.getByText("Ask about your spend, models, keys, and trends")).toBeInTheDocument(); - }); - - it("should render model selector", () => { - renderWithProviders(); - - // One library paints the prompt as its own text node and the other leaves it on the input's - // placeholder attribute, so either one means the user is being told what to pick. - const prompt = "Select a model (optional, defaults to gpt-4o-mini)"; - expect(screen.queryAllByText(prompt).length + screen.queryAllByPlaceholderText(prompt).length).toBeGreaterThan(0); - }); - - it("should render empty state message when no conversation", () => { - renderWithProviders(); - - expect(screen.getByText("Ask a question about your usage")).toBeInTheDocument(); - }); - - it("should render the send button", () => { - renderWithProviders(); - - expect(screen.getByText("Send")).toBeInTheDocument(); - }); - - it("should render input placeholder", () => { - renderWithProviders(); - - expect(screen.getByPlaceholderText("Ask about your usage...")).toBeInTheDocument(); - }); - - it("should render clear chat button", () => { - renderWithProviders(); - - expect(screen.getByText("Clear chat")).toBeInTheDocument(); - }); - - it("should have the panel element even when closed (just off-screen)", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).toBeInTheDocument(); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-full"); - }); - - it("should not have translate-x-full class when open", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).not.toHaveClass("translate-x-full"); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-0"); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx deleted file mode 100644 index dc4291d25e1..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx +++ /dev/null @@ -1,385 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import ReactMarkdown from "react-markdown"; -import { MessageSquare, Sparkles, X } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { - Combobox, - ComboboxContent, - ComboboxEmpty, - ComboboxInput, - ComboboxItem, - ComboboxList, -} from "@/components/ui/combobox"; -import { Textarea } from "@/components/ui/textarea"; -import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -import { modelHubCall, usageAiChatStream, UsageAiToolCallEvent } from "@/components/networking"; - -interface ToolCallStep { - tool_name: string; - tool_label: string; - arguments: Record; - status: "running" | "complete" | "error"; - error?: string; -} - -interface ChatMessage { - role: "user" | "assistant"; - content: string; - toolCalls?: ToolCallStep[]; -} - -interface UsageAIChatPanelProps { - open: boolean; - onClose: () => void; - accessToken: string | null; -} - -const TOOL_ICONS: Record = { - get_usage_data: "📊", - get_team_usage_data: "👥", - get_tag_usage_data: "🏷️", -}; - -const ToolCallDisplay: React.FC<{ step: ToolCallStep }> = ({ step }) => { - const icon = TOOL_ICONS[step.tool_name] || "🔧"; - const args = step.arguments; - const dateRange = args.start_date && args.end_date ? `${args.start_date} → ${args.end_date}` : ""; - const filter = args.team_ids || args.tags || args.user_id || ""; - - return ( -
- - {step.status === "running" ? ( - - ) : step.status === "error" ? ( - ✗ - ) : ( - ✓ - )} - -
-
- {icon} {step.tool_label} -
- {dateRange &&
{dateRange}
} - {filter &&
Filter: {filter}
} - {step.status === "error" && step.error &&
{step.error}
} -
-
- ); -}; - -const MarkdownContent: React.FC<{ content: string }> = ({ content }) => ( -

{children}

, - strong: ({ children }) => {children}, - ul: ({ children }) =>
    {children}
, - ol: ({ children }) =>
    {children}
, - li: ({ children }) =>
  • {children}
  • , - h1: ({ children }) =>

    {children}

    , - h2: ({ children }) =>

    {children}

    , - h3: ({ children }) =>

    {children}

    , - code: ({ children, className }) => { - const isBlock = className?.includes("language-"); - return isBlock ? ( -
    -            {children}
    -          
    - ) : ( - {children} - ); - }, - table: ({ children }) => ( -
    - {children}
    -
    - ), - th: ({ children }) => ( - {children} - ), - td: ({ children }) => {children}, - }} - > - {content} -
    -); - -const UsageAIChatPanel: React.FC = ({ open, onClose, accessToken }) => { - const [messages, setMessages] = useState([]); - const [inputText, setInputText] = useState(""); - const [isLoading, setIsLoading] = useState(false); - const [selectedModel, setSelectedModel] = useState(undefined); - const [availableModels, setAvailableModels] = useState([]); - const [isLoadingModels, setIsLoadingModels] = useState(false); - const [streamingContent, setStreamingContent] = useState(""); - const [statusMessage, setStatusMessage] = useState(null); - const [activeToolCalls, setActiveToolCalls] = useState([]); - const messagesEndRef = useRef(null); - const abortControllerRef = useRef(null); - - useEffect(() => { - if (open && availableModels.length === 0) { - loadModels(); - } - }, [open]); - - useEffect(() => { - if (typeof messagesEndRef.current?.scrollIntoView === "function") { - messagesEndRef.current.scrollIntoView({ behavior: "smooth" }); - } - }, [messages, streamingContent, activeToolCalls, statusMessage]); - - const loadModels = async () => { - if (!accessToken) return; - setIsLoadingModels(true); - try { - const fetchedModels = await modelHubCall(accessToken); - if (fetchedModels?.data?.length > 0) { - const models = fetchedModels.data.map((item: any) => item.model_group as string).sort(); - setAvailableModels(models); - } - } catch (error) { - console.error("Failed to load models:", error); - } finally { - setIsLoadingModels(false); - } - }; - - const handleSend = async () => { - if (!accessToken || !inputText.trim() || isLoading) return; - - const userMessage: ChatMessage = { role: "user", content: inputText.trim() }; - const updatedMessages = [...messages, userMessage]; - setMessages(updatedMessages); - setInputText(""); - setIsLoading(true); - setStreamingContent(""); - setStatusMessage(null); - setActiveToolCalls([]); - - const abortController = new AbortController(); - abortControllerRef.current = abortController; - - let accumulated = ""; - const toolCalls: ToolCallStep[] = []; - - try { - await usageAiChatStream( - accessToken, - updatedMessages.slice(-20).map((m) => ({ role: m.role, content: m.content })), - selectedModel || "", - (content: string) => { - setStatusMessage(null); - accumulated += content; - setStreamingContent(accumulated); - }, - () => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [ - ...prev, - { role: "assistant", content: accumulated, toolCalls: toolCalls.length > 0 ? [...toolCalls] : undefined }, - ]); - setStreamingContent(""); - }, - (errorMsg: string) => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - }, - (status: string) => { - setStatusMessage(status); - }, - (event: UsageAiToolCallEvent) => { - const idx = toolCalls.findIndex((tc) => tc.tool_name === event.tool_name); - if (idx >= 0) { - toolCalls[idx] = { ...event }; - } else { - toolCalls.push({ ...event }); - } - setActiveToolCalls([...toolCalls]); - }, - abortController.signal, - ); - } catch (error: any) { - if (error?.name === "AbortError" || abortController.signal.aborted) { - return; - } - const errorMsg = error?.message || "Failed to get response. Please try again."; - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - } finally { - setIsLoading(false); - abortControllerRef.current = null; - } - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleSend(); - } - }; - - const handleClose = () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - } - onClose(); - }; - - const handleClear = () => { - setMessages([]); - setStreamingContent(""); - setActiveToolCalls([]); - setStatusMessage(null); - }; - - return ( -
    - {/* Header */} -
    -
    -

    -

    -

    Ask about your spend, models, keys, and trends

    -
    - -
    - - {/* Model selector */} -
    - setSelectedModel(value ?? undefined)} - > - - - {isLoadingModels ? "Loading models…" : "No models found"} - - {(model: string) => ( - - {model} - - )} - - - -
    - - {/* Chat messages */} -
    - {messages.length === 0 && !streamingContent && !isLoading && ( -
    -
    - )} - - {messages.map((msg, idx) => ( -
    - {msg.role === "user" ? ( -
    -
    - {msg.content} -
    -
    - ) : ( -
    - {/* Tool calls for this message */} - {msg.toolCalls && msg.toolCalls.length > 0 && ( -
    - {msg.toolCalls.map((tc, tcIdx) => ( - - ))} -
    - )} - {/* Response */} -
    - -
    -
    - )} -
    - ))} - - {/* Active tool calls (in-progress) */} - {isLoading && activeToolCalls.length > 0 && ( -
    - {activeToolCalls.map((tc, idx) => ( - - ))} -
    - )} - - {/* Status / spinner */} - {isLoading && !streamingContent && ( -
    - - {statusMessage || "Thinking..."} -
    - )} - - {/* Streaming response */} - {streamingContent && ( -
    - -
    - )} - -
    -
    - - {/* Input area */} -
    -
    -