diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a707eb8044c..f6c06f7d83f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,4 +1,4 @@ -/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri -/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri +/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri +/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri /helm/ @yassin-berriai /terraform/ @yassin-berriai diff --git a/README.md b/README.md index 5fd84b529dc..4feb6e00ffa 100644 --- a/README.md +++ b/README.md @@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws ### Verify Docker Image Signatures -All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0). +All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065). **Verify using the pinned commit hash (recommended):** -A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key: +A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key: ```bash cosign verify \ - --key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \ + --key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \ ghcr.io/berriai/litellm: ``` diff --git a/cosign.pub b/cosign.pub index 2c2a555ab09..7603d69641f 100644 --- a/cosign.pub +++ b/cosign.pub @@ -1,4 +1,4 @@ -----BEGIN PUBLIC KEY----- -MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb -POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g== +MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg +JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ== -----END PUBLIC KEY----- diff --git a/litellm/integrations/otel/mount.py b/litellm/integrations/otel/mount.py index cb52ad24c8e..c0d42eeb5c5 100644 --- a/litellm/integrations/otel/mount.py +++ b/litellm/integrations/otel/mount.py @@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi`` is unavailable. This MUST be called at app-creation time — once the lifespan - runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add - middleware after an application has started". + runs, the middleware stack is frozen and middleware added later never runs, so + a late call is skipped without registering anything. No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global ``ProxyTracerProvider``; the proxy publishes the real provider as the global @@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No try: if not is_otel_v2_enabled(): return + if app.middleware_stack is not None: + verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started") + return # Lazy: only the V2-enabled path needs the optional # ``opentelemetry-instrumentation-fastapi`` package, which is not part of the diff --git a/litellm/integrations/otel/plumbing/otlp_json.py b/litellm/integrations/otel/plumbing/otlp_json.py index bc6d7d435b0..fc0351d5049 100644 --- a/litellm/integrations/otel/plumbing/otlp_json.py +++ b/litellm/integrations/otel/plumbing/otlp_json.py @@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex). import base64 import json +import logging from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final, TypeAlias @@ -13,9 +14,13 @@ from typing import Final, TypeAlias import requests from google.protobuf.json_format import MessageToDict from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans +from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.sdk.trace.export import SpanExportResult +from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE +_logger: Final = logging.getLogger(__name__) JSON_CONTENT_TYPE: Final = "application/json" _HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"}) @@ -66,12 +71,49 @@ class OTLPJsonSpanExporter(OTLPSpanExporter): def __init__( self, endpoint: str | None, - headers: dict[str, str], # mutable-ok: SDK __init__ takes Dict + headers: Mapping[str, str], certificate_file: str | None = None, session: "requests.Session | None" = None, + *, + max_request_size: int | None = None, ) -> None: - super().__init__(endpoint=endpoint, headers=headers, certificate_file=certificate_file, session=session) - self._session.headers["Content-Type"] = JSON_CONTENT_TYPE + super().__init__( + endpoint=endpoint, + headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE}, + certificate_file=certificate_file, + session=session, + max_request_size=max_request_size, + ) - def _serialize_spans(self, spans: Sequence[ReadableSpan]) -> bytes: - return encode_spans_json(spans) + def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult: + if self._shutdown: + _logger.warning("Exporter already shutdown, ignoring batch") + return SpanExportResult.FAILURE + + with self._metrics.export_operation(len(spans)) as result: + try: + serialized_data: Final = encode_spans_json(spans) + except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results + _logger.error("Failed to encode span batch: %s", error) + result.error = error + return SpanExportResult.FAILURE + + if 0 < self._max_request_size < len(serialized_data): + result.error = RequestPayloadTooLargeError( + f"Serialized span request size {len(serialized_data)} bytes " + f"exceeds max_request_size {self._max_request_size} bytes." + ) + _logger.warning("Dropping span batch: %s", result.error) + return SpanExportResult.FAILURE + + export_result: Final = self._client.export(serialized_data) + if not export_result.success: + result.error = export_result.error + result.error_attrs = ( + {HTTP_RESPONSE_STATUS_CODE: export_result.status_code} + if export_result.status_code is not None + else None + ) + return SpanExportResult.FAILURE + + return SpanExportResult.SUCCESS diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 3befb73b0cc..141162bdfa5 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -76,6 +76,7 @@ def agentops_preset( def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """Factory for the ``agentops`` exporter kind: a lazy-auth OTLP/HTTP exporter.""" + import requests from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( OTLPSpanExporter, ) @@ -89,7 +90,8 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: """ def __init__(self, *, endpoint: str | None, api_key: str | None) -> None: - super().__init__(endpoint=endpoint) + self._agentops_session = requests.Session() + super().__init__(endpoint=endpoint, session=self._agentops_session) self._agentops_api_key = api_key self._auth_resolved = False @@ -102,10 +104,7 @@ def _build_agentops_exporter(spec: ExporterSpec) -> SpanExporter: try: token: Final = _fetch_agentops_jwt(self._agentops_api_key).get("token") if token: - # ``_session`` is the requests.Session the base exporter - # POSTs through; updating its Authorization header is how the - # minted JWT reaches every subsequent export. - self._session.headers["Authorization"] = f"Bearer {token}" + self._agentops_session.headers["Authorization"] = f"Bearer {token}" except Exception as e: verbose_logger.debug("AgentOps JWT fetch failed: %s", e) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 90a0763f786..3edb30b4218 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -48499,6 +48723,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -52906,12 +53141,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53163,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -73330,6 +73569,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/litellm/proxy/_experimental/out/assets/aria-agent.png b/litellm/proxy/_experimental/out/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/litellm/proxy/_experimental/out/assets/aria-agent.png differ diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 0e2eed73364..28c701e0fc2 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( module_path="litellm.proxy.spend_tracking.vantage_endpoints", path_prefixes=("/vantage",), ), - LazyFeature( - name="usage_ai", - module_path="litellm.proxy.management_endpoints.usage_endpoints", - path_prefixes=("/usage/ai",), - ), LazyFeature( name="prompts", module_path="litellm.proxy.prompts.prompt_endpoints", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index e5e0027a808..0dd978ee855 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -57898,163 +57898,6 @@ } } }, - "usage_ai": { - "components": { - "schemas": { - "ChatMessage": { - "properties": { - "content": { - "title": "Content", - "type": "string" - }, - "role": { - "enum": [ - "user", - "assistant" - ], - "title": "Role", - "type": "string" - } - }, - "required": [ - "role", - "content" - ], - "title": "ChatMessage", - "type": "object" - }, - "HTTPValidationError": { - "properties": { - "detail": { - "items": { - "$ref": "#/components/schemas/ValidationError" - }, - "title": "Detail", - "type": "array" - } - }, - "title": "HTTPValidationError", - "type": "object" - }, - "UsageAIChatRequest": { - "properties": { - "messages": { - "description": "Chat messages (user/assistant history)", - "items": { - "$ref": "#/components/schemas/ChatMessage" - }, - "title": "Messages", - "type": "array" - }, - "model": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "null" - } - ], - "description": "Model to use for AI chat", - "title": "Model" - } - }, - "required": [ - "messages" - ], - "title": "UsageAIChatRequest", - "type": "object" - }, - "ValidationError": { - "properties": { - "ctx": { - "title": "Context", - "type": "object" - }, - "input": { - "title": "Input" - }, - "loc": { - "items": { - "anyOf": [ - { - "type": "string" - }, - { - "type": "integer" - } - ] - }, - "title": "Location", - "type": "array" - }, - "msg": { - "title": "Message", - "type": "string" - }, - "type": { - "title": "Error Type", - "type": "string" - } - }, - "required": [ - "loc", - "msg", - "type" - ], - "title": "ValidationError", - "type": "object" - } - } - }, - "paths": { - "/usage/ai/chat": { - "post": { - "description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.", - "operationId": "usage_ai_chat_usage_ai_chat_post", - "requestBody": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/UsageAIChatRequest" - } - } - }, - "required": true - }, - "responses": { - "200": { - "content": { - "application/json": { - "schema": {} - } - }, - "description": "Successful Response" - }, - "422": { - "content": { - "application/json": { - "schema": { - "$ref": "#/components/schemas/HTTPValidationError" - } - } - }, - "description": "Validation Error" - } - }, - "security": [ - { - "APIKeyHeader": [] - } - ], - "summary": "Usage Ai Chat", - "tags": [ - "usage_ai" - ] - } - } - } - }, "vantage": { "components": { "schemas": { diff --git a/litellm/proxy/management_endpoints/ui_sso.py b/litellm/proxy/management_endpoints/ui_sso.py index 19bca3d2bf8..fd5eb672811 100644 --- a/litellm/proxy/management_endpoints/ui_sso.py +++ b/litellm/proxy/management_endpoints/ui_sso.py @@ -152,7 +152,7 @@ from litellm.proxy.utils import ( from litellm.repositories.prisma_protocols import TableActions from litellm.repositories.table_repositories import SSOConfigRepository from litellm.repositories.team_repository import TeamRepository -from litellm.repositories.user_repository import UserRepository +from litellm.repositories.user_repository import UserRepository, free_sso_user_creation from litellm.secret_managers.main import get_secret_bool, get_secret_str from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403 @@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token( ) -async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None: - """Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license.""" - if premium_user is True: - return - if prisma_client is None: - raise ProxyException( - message=CommonProxyErrors.db_not_connected_error.value, - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) - billable_users: Final = await UserRepository(prisma_client).count_billable_users() - if billable_users and billable_users > 5: - raise ProxyException( - message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it", - type=ProxyErrorTypes.auth_error, - param="premium_user", - code=status.HTTP_403_FORBIDDEN, - ) +class _SSOUserAdmissionError(ProxyException): + pass @router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False) @@ -1112,8 +1095,6 @@ async def google_login( from litellm.proxy.proxy_server import ( cli_sso_session_cache, general_settings, - premium_user, - prisma_client, saml_sso_cache, user_custom_ui_sso_sign_in_handler, ) @@ -1128,15 +1109,6 @@ async def google_login( if admin_ui_is_disabled and not is_cli_sso_login: return admin_ui_disabled() - ####### Check if user is a Enterprise / Premium User ####### - if ( - microsoft_client_id is not None - or google_client_id is not None - or generic_client_id is not None - or SAMLAuthHandler.is_saml_configured() - ): - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ####### Detect DB + MASTER KEY in .env ####### missing_env_vars: Final = show_missing_vars_in_env() if missing_env_vars is not None: @@ -1963,6 +1935,8 @@ async def get_user_info_from_db( ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e): verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e) @@ -2328,7 +2302,6 @@ async def saml_callback(request: Request): general_settings, jwt_handler, master_key, - premium_user, prisma_client, saml_sso_cache, ) @@ -2352,8 +2325,6 @@ async def saml_callback(request: Request): user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data) - await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client) - ui_access_mode: Final = general_settings.get("ui_access_mode", None) relay_state: Final = post_data.get("RelayState") cp_return_to: Final[str | None] = ( @@ -2821,12 +2792,20 @@ async def insert_sso_user( if result_openid and hasattr(result_openid, "provider"): new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")} - response: Final = await new_user( - data=new_user_request, - user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), - ) + from litellm.proxy.proxy_server import premium_user - return response + with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True): + try: + return await new_user( + data=new_user_request, + user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN), + ) + except ProxyException as error: + if premium_user is True: + raise + raise _SSOUserAdmissionError( + message=error.message, type=error.type, param=error.param, code=error.code + ) from error DefaultTeamJoin: TypeAlias = Callable[ @@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler: prisma_client=prisma_client, ) return user_info + except _SSOUserAdmissionError: + raise except Exception as e: if ( user_info is None diff --git a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py b/litellm/proxy/management_endpoints/usage_endpoints/__init__.py deleted file mode 100644 index 6e68dcd2a2e..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -""" -Usage endpoints package. - -Re-exports the router from endpoints module. -""" - -from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401 - router, -) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py b/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py deleted file mode 100644 index 7a45d0438de..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py +++ /dev/null @@ -1,627 +0,0 @@ -""" -AI Usage Chat - uses LLM tool calling to answer questions about -usage/spend data by querying the aggregated daily activity endpoints. -""" - -import json -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence -from datetime import date -from typing import Final, Literal, NamedTuple, Protocol, cast, overload - -from fastapi import HTTPException -from typing_extensions import ReadOnly, TypedDict - -import litellm -from litellm._logging import verbose_proxy_logger -from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL -from litellm.proxy._types import CommonProxyErrors -from litellm.types.proxy.management_endpoints.common_daily_activity import ( - SpendAnalyticsPaginatedResponse, -) -from litellm.types.utils import ChatCompletionMessageToolCall - -# --------------------------------------------------------------------------- -# Constants -# --------------------------------------------------------------------------- - -USAGE_AI_TEMPERATURE: Final = 0.2 - -TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend" -TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend" -TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend" - -ENTITY_FIELD_USER: Final = "user_id" -ENTITY_FIELD_TEAM: Final = "team_id" -ENTITY_FIELD_TAG: Final = "tag" - -PAGINATED_PAGE_SIZE: Final = 200 -MAX_CHAT_MESSAGES: Final = 20 -TOP_N_MODELS: Final = 15 -TOP_N_PROVIDERS: Final = 10 -TOP_N_KEYS: Final = 10 - -# --------------------------------------------------------------------------- -# Types -# --------------------------------------------------------------------------- - - -class SSEStatusEvent(TypedDict): - type: Literal["status"] - message: str - - -class SSEToolCallEvent(TypedDict, total=False): - type: Literal["tool_call"] - tool_name: str - tool_label: str - arguments: dict[str, str] - status: Literal["running", "complete", "error"] - error: str - - -class SSEChunkEvent(TypedDict): - type: Literal["chunk"] - content: str - - -class SSEDoneEvent(TypedDict): - type: Literal["done"] - - -class SSEErrorEvent(TypedDict): - type: Literal["error"] - message: str - - -SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent - - -class _EntityEntry(TypedDict, total=False): - metrics: ReadOnly[Mapping[str, float]] - metadata: ReadOnly[Mapping[str, str]] - - -class _DayDump(TypedDict, total=False): - breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]] - - -class _EntityTotal(NamedTuple): - """Running per-entity totals accumulated while summarising a usage dump.""" - - alias: str - spend: float - requests: float - tokens: float - - -class _UsageDump(Protocol): - @overload - def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ... - @overload - def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ... - - -class _ToolFunctionDef(TypedDict): - name: ReadOnly[str] - description: ReadOnly[str] - parameters: ReadOnly[Mapping[str, object]] - - -class _ToolDef(TypedDict): - type: ReadOnly[str] - function: ReadOnly[_ToolFunctionDef] - - -class ToolHandler(TypedDict): - fetch: Callable[..., Awaitable[_UsageDump]] - summarise: Callable[[_UsageDump], str] - label: str - - -# --------------------------------------------------------------------------- -# Tool definitions (OpenAI function-calling schema) -# --------------------------------------------------------------------------- - -_DATE_PARAMS: Final = { - "start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"}, - "end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"}, -} - -_TOOL_USAGE: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_usage_data", - "description": ( - "Fetch aggregated global usage/spend data. Returns daily spend, " - "token counts, request counts, and breakdowns by model, provider, " - "and API key. Use for overall spend, top models, top providers." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "user_id": { - "type": "string", - "description": "Optional user ID filter. Omit for global view.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TEAM: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_team_usage_data", - "description": ( - "Fetch usage/spend data broken down by team. Use for questions " - "like 'which team spends the most' or 'show me team X usage'." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "team_ids": { - "type": "string", - "description": "Optional comma-separated team IDs. Omit for all teams.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -_TOOL_TAG: Final[_ToolDef] = { - "type": "function", - "function": { - "name": "get_tag_usage_data", - "description": ( - "Fetch usage/spend data broken down by tag. Tags are labels " - "attached to requests (features, environments, credentials)." - ), - "parameters": { - "type": "object", - "properties": { - **_DATE_PARAMS, - "tags": { - "type": "string", - "description": "Optional comma-separated tag names. Omit for all tags.", - }, - }, - "required": ["start_date", "end_date"], - }, - }, -} - -TOOLS_BASE: Final = [_TOOL_USAGE] -TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG] - - -def get_tools_for_role(is_admin: bool) -> list[_ToolDef]: - """Return the tool list appropriate for the user's role.""" - return TOOLS_ADMIN if is_admin else TOOLS_BASE - - -_SYSTEM_PROMPT_BASE: Final = ( - "You are an AI assistant embedded in the LiteLLM Usage dashboard. " - "You help users understand their LLM API spend and usage data.\n\n" - "ALWAYS call the appropriate tool(s) first to fetch data before answering. " - "You may call multiple tools if the question spans different dimensions.\n\n" - "Guidelines:\n" - "- Be concise and specific. Use exact numbers from the data.\n" - "- Format costs as dollar amounts (e.g. $12.34).\n" - "- When comparing entities, show a ranked list.\n" - "- If data is empty or no results found, say so clearly.\n" - "- Do not hallucinate data — only use what the tools return.\n" - "- Today's date will be provided below. Use it to interpret relative dates " - "like 'this week', 'this month', 'last 7 days', etc." -) - -_TOOL_DESCRIPTIONS_ADMIN: Final = ( - "You have access to these tools:\n" - "- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n" - "- `get_team_usage_data`: Team-level usage breakdown\n" - "- `get_tag_usage_data`: Tag-level usage breakdown\n\n" -) - -_TOOL_DESCRIPTIONS_BASE: Final = ( - "You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n" -) - - -def _build_system_prompt(is_admin: bool) -> str: - """Build role-appropriate system prompt with today's date.""" - tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE - return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}" - - -# keep a public reference for test assertions -SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE - -# --------------------------------------------------------------------------- -# Data fetchers -# --------------------------------------------------------------------------- - - -def _parse_csv_ids(raw: str | None) -> list[str] | None: - if not raw: - return None - return [t.strip() for t in raw.split(",") if t.strip()] - - -async def _query_activity( - table_name: str, - entity_id_field: str, - entity_id: str | list[str] | None, - start_date: str, - end_date: str, - *, - use_aggregated: bool = False, -) -> SpendAnalyticsPaginatedResponse: - """Shared helper that calls the daily activity query layer.""" - from litellm.proxy.management_endpoints.common_daily_activity import ( - daily_activity_repository, - daily_activity_scope, - get_daily_activity, - get_daily_activity_aggregated, - ) - from litellm.proxy.proxy_server import prisma_client - - if use_aggregated: - if prisma_client is None: - raise HTTPException( - status_code=500, - detail={"error": CommonProxyErrors.db_not_connected_error.value}, - ) - repository: Final = daily_activity_repository(prisma_client) - scope: Final = daily_activity_scope( - table_name, - entity_id_field, - entity_id, - None, - None, - start_date, - end_date, - None, - None, - ) - return await get_daily_activity_aggregated( - repository, - scope, - ) - return await get_daily_activity( - prisma_client=prisma_client, - table_name=table_name, - entity_id_field=entity_id_field, - entity_id=entity_id, - entity_metadata_field=None, - start_date=start_date, - end_date=end_date, - model=None, - api_key=None, - page=1, - page_size=PAGINATED_PAGE_SIZE, - ) - - -async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_USER_SPEND, - ENTITY_FIELD_USER, - user_id, - start_date, - end_date, - use_aggregated=True, - ) - return resp.model_dump(mode="json") - - -async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TEAM_SPEND, - ENTITY_FIELD_TEAM, - _parse_csv_ids(team_ids), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump: - resp: Final = await _query_activity( - TABLE_DAILY_TAG_SPEND, - ENTITY_FIELD_TAG, - _parse_csv_ids(tags), - start_date, - end_date, - ) - return resp.model_dump(mode="json") - - -# --------------------------------------------------------------------------- -# Summarisers — convert raw JSON to concise text the LLM can reason over -# --------------------------------------------------------------------------- - - -def _accumulate_breakdown( - results: Sequence[_DayDump], dimension: str, fields: Sequence[str] -) -> dict[str, dict[str, float]]: - """Aggregate a single breakdown dimension across days.""" - totals: Final[dict[str, dict[str, float]]] = {} - for day in results: - for key, entry in day.get("breakdown", {}).get(dimension, {}).items(): - if key not in totals: - totals[key] = {f: 0.0 for f in fields} - m = entry.get("metrics", {}) - for f in fields: - totals[key][f] += m.get(f, 0) - return totals - - -def _ranked_lines( - totals: dict[str, dict[str, float]], - fmt: Callable[[str, dict[str, float]], str], - limit: int, -) -> list[str]: - """Sort by spend descending, format each entry, and truncate.""" - return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]] - - -def _summarise_usage_data(data: _UsageDump) -> str: - meta: Final = data.get("metadata", {}) - results: Final = data.get("results", []) - - header: Final = ( - f"Total Spend: ${meta.get('total_spend', 0):.4f}\n" - f"Total Requests: {meta.get('total_api_requests', 0)}\n" - f"Successful: {meta.get('total_successful_requests', 0)} | " - f"Failed: {meta.get('total_failed_requests', 0)}\n" - f"Total Tokens: {meta.get('total_tokens', 0)}" - ) - - models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"]) - providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"]) - - model_lines: Final = _ranked_lines( - models, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)", - TOP_N_MODELS, - ) - provider_lines: Final = _ranked_lines( - providers, - lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)", - TOP_N_PROVIDERS, - ) - - sections = [header, ""] - sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""] - sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"]) - return "\n".join(sections) - - -def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str: - """Summarise team/tag entity usage data.""" - results: Final = data.get("results", []) - if not results: - return f"No {entity_label} usage data found for the given date range." - - totals: Final[dict[str, _EntityTotal]] = {} - for day in results: - for eid, entry in day.get("breakdown", {}).get("entities", {}).items(): - previous = totals.get(eid) - m = entry.get("metrics", {}) - totals[eid] = _EntityTotal( - alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid), - spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0), - requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0), - tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0), - ) - - lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""] - for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend): - label = d.alias if d.alias != eid else eid - lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Tool dispatch registry -# --------------------------------------------------------------------------- - -TOOL_HANDLERS: Final[dict[str, ToolHandler]] = { - "get_usage_data": ToolHandler( - fetch=_fetch_usage_data, - summarise=_summarise_usage_data, - label="global usage data", - ), - "get_team_usage_data": ToolHandler( - fetch=_fetch_team_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Team"), - label="team usage data", - ), - "get_tag_usage_data": ToolHandler( - fetch=_fetch_tag_usage_data, - summarise=lambda data: _summarise_entity_data(data, "Tag"), - label="tag usage data", - ), -} - - -# --------------------------------------------------------------------------- -# SSE streaming -# --------------------------------------------------------------------------- - - -def _sse(event: SSEEvent) -> str: - return f"data: {json.dumps(event)}\n\n" - - -def _resolve_fetch_kwargs( - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> dict[str, str]: - """Build keyword arguments for a tool's fetch function.""" - start_date: Final = fn_args.get("start_date", "") - end_date: Final = fn_args.get("end_date", "") - if not start_date or not end_date: - raise ValueError("Missing required start_date or end_date from tool arguments") - kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date} - if fn_name == "get_usage_data": - if not is_admin: - if user_id is None: - # Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py - # should have already rejected this. If we ever reach here it means - # a future caller invoked the helper without scoping — fail loudly - # rather than issuing an unfiltered global query. - raise ValueError( - "Non-admin caller has user_id=None; refusing to issue an " - "unscoped query. Endpoint-level guard missing." - ) - kwargs["user_id"] = user_id - elif fn_args.get("user_id"): - kwargs["user_id"] = fn_args["user_id"] - elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"): - kwargs["team_ids"] = fn_args["team_ids"] - elif fn_name == "get_tag_usage_data" and fn_args.get("tags"): - kwargs["tags"] = fn_args["tags"] - return kwargs - - -async def _execute_tool_call( - handler: ToolHandler, - fn_name: str, - fn_args: Mapping[str, str], - user_id: str | None, - is_admin: bool, -) -> str: - """Run a single tool and return the summarised result text, or the reason the query - layer rejected its dates so the model can correct them.""" - kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin) - try: - raw_data: Final = await handler["fetch"](**kwargs) - except HTTPException as e: - reason: Final = _invalid_date_reason(e) - if reason is None: - raise - return f"Invalid date range: {reason}" - return handler["summarise"](raw_data) - - -def _invalid_date_reason(error: HTTPException) -> str | None: - """The reason in the daily activity query layer's 400 for invalid or reversed dates.""" - detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict - match detail: - case {"error": str() as reason} if error.status_code == 400: - return reason - case _: - return None - - -async def _process_tool_call( - tc: ChatCompletionMessageToolCall, - chat_messages: list[Mapping[str, object]], - user_id: str | None, - is_admin: bool, -) -> AsyncIterator[str]: - """Execute a single tool call, yielding SSE events for status.""" - fn_name: Final = tc.function.name - fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments) - - allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)} - handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None - - if fn_name is None or fn_name not in allowed_names or not handler: - chat_messages.append( - { - "role": "tool", - "tool_call_id": tc.id, - "content": f"Tool not available: {fn_name}", - } - ) - return - - tool_event_base: Final = { - "type": "tool_call", - "tool_name": fn_name, - "tool_label": handler["label"], - "arguments": fn_args, - } - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"})) - - try: - tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin) - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"})) - except Exception as e: - verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e) - tool_result = f"Error fetching {handler['label']}. Please try again." - yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"})) - - chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result}) - - -async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]: - """Stream the final LLM response after tool results are appended.""" - yield _sse({"type": "status", "message": "Analyzing results..."}) - - response: Final = await litellm.acompletion( - model=model, - messages=chat_messages, - stream=True, - temperature=USAGE_AI_TEMPERATURE, - ) - async for chunk in response: - delta = chunk.choices[0].delta.content - if delta: - yield _sse({"type": "chunk", "content": delta}) - - -async def stream_usage_ai_chat( - messages: list[dict[str, str]], - model: str | None = None, - user_id: str | None = None, - is_admin: bool = False, -) -> AsyncGenerator[str, None]: - """Stream SSE events: status → tool_call → chunk → done.""" - resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL - truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages - chat_messages: Final[list[Mapping[str, object]]] = [ - {"role": "system", "content": _build_system_prompt(is_admin)}, - *truncated, - ] - - try: - yield _sse({"type": "status", "message": "Thinking..."}) - tools: Final = get_tools_for_role(is_admin) - response: Final = await litellm.acompletion( - model=resolved_model, - messages=chat_messages, - tools=tools, - temperature=USAGE_AI_TEMPERATURE, - ) - choice: Final = response.choices[0] - - if not choice.message.tool_calls: - if choice.message.content: - yield _sse({"type": "chunk", "content": choice.message.content}) - yield _sse({"type": "done"}) - return - - chat_messages.append(choice.message.model_dump()) - for tc in choice.message.tool_calls: - async for event in _process_tool_call(tc, chat_messages, user_id, is_admin): - yield event - async for event in _stream_final_response(resolved_model, chat_messages): - yield event - yield _sse({"type": "done"}) - - except Exception as e: - verbose_proxy_logger.error("AI usage chat failed: %s", e) - yield _sse( - { - "type": "error", - "message": "An internal error occurred. Please try again.", - } - ) diff --git a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py b/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py deleted file mode 100644 index cff5bef05a0..00000000000 --- a/litellm/proxy/management_endpoints/usage_endpoints/endpoints.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -USAGE AI CHAT ENDPOINTS - -/usage/ai/chat - Stream AI chat responses about usage data -""" - -from typing import Final, Literal - -from fastapi import APIRouter, Depends, Request -from fastapi.responses import StreamingResponse -from pydantic import Field - -import litellm -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.sse_keepalive import ( - SSE_COMMENT_PING, - wrap_sse_stream_with_keepalive_pings, -) -from litellm.types.llms.base import LiteLLMBaseModel - -router: Final = APIRouter() - - -class ChatMessage(LiteLLMBaseModel): - role: Literal["user", "assistant"] - content: str - - -class UsageAIChatRequest(LiteLLMBaseModel): - messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)") - model: str | None = Field(default=None, description="Model to use for AI chat") - - -@router.post( - "/usage/ai/chat", - tags=["Budget & Spend Tracking"], - dependencies=[Depends(user_api_key_auth)], -) -async def usage_ai_chat( - data: UsageAIChatRequest, - request: Request, - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), -): - """ - AI chat about usage data. Streams SSE events with the AI response. - The AI agent has access to tools that query aggregated daily activity data. - """ - from litellm.proxy.management_endpoints.common_utils import ( - require_caller_user_id_for_non_admin, - user_api_key_has_admin_view, - ) - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - stream_usage_ai_chat, - ) - - is_admin: Final = user_api_key_has_admin_view(user_api_key_dict) - if is_admin: - user_id = user_api_key_dict.user_id - else: - user_id = require_caller_user_id_for_non_admin(user_api_key_dict) - messages: Final = [{"role": m.role, "content": m.content} for m in data.messages] - - return StreamingResponse( - wrap_sse_stream_with_keepalive_pings( - stream_usage_ai_chat( - messages=messages, - model=data.model, - user_id=user_id, - is_admin=is_admin, - ), - ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds, - ping_chunk=SSE_COMMENT_PING, - ), - media_type="text/event-stream", - headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"}, - ) diff --git a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py index ef2a5f88e4e..14d75633100 100644 --- a/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py +++ b/litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py @@ -1,4 +1,5 @@ import asyncio +import os import re import xml.etree.ElementTree as ET from collections import Counter @@ -88,6 +89,10 @@ def _default_fetch_lock() -> asyncio.Lock: return _latest_release_fetch_lock +def _default_release_check_enabled() -> bool: + return os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true" + + def _bucket_for(line: str) -> _Bucket | None: if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None: return None @@ -224,12 +229,16 @@ async def latest_release_info( client: Annotated[_AsyncGetClient, Depends(_default_client)], cache: Annotated[InMemoryCache, Depends(_default_cache)], fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)], + release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)], ) -> LatestReleaseInfo | None: """ Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates. Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render. + Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request. """ + if not release_check_enabled: + return None result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock) if isinstance(result, LatestReleaseUnavailable): verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason) diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 28be11509ae..efc83132568 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -6032,13 +6032,12 @@ class PrismaClient: elif table_name == "user": db_data = self.jsonify_object(data=data) try: - new_user_row: Final = await UserRepository(self).table.upsert( - where={"user_id": data["user_id"]}, - data={ - "create": {**db_data}, - "update": {}, # don't do anything if it already exists - }, - ) + new_user_row: Final = await UserRepository(self).upsert_user(db_data) + if new_user_row is None: + raise HTTPException( + status_code=403, + detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.", + ) except Exception as e: if ( "Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`" diff --git a/litellm/repositories/user_repository.py b/litellm/repositories/user_repository.py index 5ed1574c6bc..1fce0717ea3 100644 --- a/litellm/repositories/user_repository.py +++ b/litellm/repositories/user_repository.py @@ -2,8 +2,11 @@ User repository for database operations on LiteLLM_UserTable. """ +import asyncio import json -from collections.abc import Mapping, Sequence +from collections.abc import Generator, Mapping, Sequence +from contextlib import AbstractAsyncContextManager, contextmanager +from contextvars import ContextVar from itertools import chain from typing import TYPE_CHECKING, Final, Protocol @@ -35,6 +38,16 @@ ORDER BY p.user_id """ _PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...]) +_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None) + + +@contextmanager +def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]: + token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None) + try: + yield + finally: + _FREE_SSO_CREATION.reset(token) class _UserDb(Protocol): @@ -42,6 +55,12 @@ class _UserDb(Protocol): def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ... +class _UserTransaction(_UserDb, Protocol): + async def execute_raw(self, query: str) -> int: ... + + async def query_raw(self, query: str) -> object: ... + + class _PrismaClientView(Protocol): @property def db(self) -> _UserDb: ... @@ -49,6 +68,16 @@ class _PrismaClientView(Protocol): @property def writer_db(self) -> _UserDb: ... + def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ... + + +async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int: + from prisma import Json + + total: Final = await table.count() + deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) + return max(0, total - deactivated) + class UserRepository(BaseRepository[LiteLLM_UserTable]): """Repository for user database operations.""" @@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]): null, or true all count, so seats are counted as total users minus the deactivated ones. """ - from prisma import Json # pyright: ignore[reportUnknownVariableType] + return await _count_billable_users(self.table) - total: Final = await self.count() - deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}}) - return max(0, total - deactivated) + async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None": + """Return the user row, or None when a new free SSO user would exceed capacity.""" + scope: Final = _FREE_SSO_CREATION.get() + if ( + scope is None + or (scope[0] is not None and scope[0] != data["user_id"]) + or scope[1] is not asyncio.current_task() + ): + return await self.table.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) + + client: Final[_PrismaClientView] = self.prisma_client + async with client.tx() as transaction: + await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED") + await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)") + existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]}) + if existing is not None: + return existing + if await _count_billable_users(transaction.litellm_usertable) >= 5: + return None + return await transaction.litellm_usertable.upsert( + where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}} + ) async def create_user( self, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 90a0763f786..3edb30b4218 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12827,6 +12827,7 @@ "supports_vision": true }, "azure_ai/Phi-4": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 16384, @@ -12840,6 +12841,7 @@ "supports_vision": false }, "azure_ai/Phi-4-mini-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12851,6 +12853,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-multimodal-instruct": { + "deprecation_date": "2027-01-17", "input_cost_per_audio_token": 4e-06, "input_cost_per_token": 8e-08, "litellm_provider": "azure_ai", @@ -12865,6 +12868,7 @@ "supports_vision": true }, "azure_ai/Phi-4-mini-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 7.5e-08, "litellm_provider": "azure_ai", "max_input_tokens": 131072, @@ -12876,6 +12880,7 @@ "supports_function_calling": true }, "azure_ai/Phi-4-reasoning": { + "deprecation_date": "2027-01-17", "input_cost_per_token": 1.25e-07, "litellm_provider": "azure_ai", "max_input_tokens": 32768, @@ -24892,27 +24897,118 @@ ] }, "elevenlabs/eleven_v3": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] }, "elevenlabs/eleven_multilingual_v2": { - "input_cost_per_character": 0.00018, + "input_cost_per_character": 8e-05, "litellm_provider": "elevenlabs", "metadata": { - "calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)", + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", "notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support" }, "mode": "audio_speech", - "source": "https://elevenlabs.io/pricing", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4": { + "input_cost_per_character": 8e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v4_turbo": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_v3_conversational": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_flash_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2_5": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", + "supported_endpoints": [ + "/v1/audio/speech" + ] + }, + "elevenlabs/eleven_turbo_v2": { + "input_cost_per_character": 4e-05, + "litellm_provider": "elevenlabs", + "metadata": { + "calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)", + "notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model" + }, + "mode": "audio_speech", + "source": "https://elevenlabs.io/pricing/api", "supported_endpoints": [ "/v1/audio/speech" ] @@ -42337,6 +42433,134 @@ ], "supports_audio_output": true }, + "mistral/voxtral-mini-transcribe-3": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-3-0": { + "input_cost_per_second": 5.83333e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-small-transcribe-3-0": { + "input_cost_per_second": 6.66667e-05, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-transcribe-realtime-3-0": { + "input_cost_per_second": 0.000133333, + "litellm_provider": "mistral", + "mode": "audio_transcription", + "source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0", + "supported_endpoints": [ + "/v1/audio/transcriptions" + ], + "supported_modalities": [ + "audio" + ], + "supported_output_modalities": [ + "text" + ], + "supports_audio_input": true + }, + "mistral/voxtral-mini-tts-3": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, + "mistral/voxtral-mini-tts-3-0": { + "input_cost_per_character": 1.3e-05, + "litellm_provider": "mistral", + "mode": "audio_speech", + "source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0", + "supported_endpoints": [ + "/v1/audio/speech" + ], + "supported_modalities": [ + "text" + ], + "supported_output_modalities": [ + "audio" + ], + "supports_audio_output": true + }, "mistral/voxtral-small-2507": { "cache_read_input_token_cost": 1e-08, "input_cost_per_token": 1e-07, @@ -48499,6 +48723,17 @@ "supports_tool_choice": true, "supports_vision": true }, + "together_ai/MiniMaxAI/MiniMax-M3.1-Flash": { + "cache_read_input_token_cost": 1.5e-08, + "input_cost_per_token": 1.5e-07, + "litellm_provider": "together_ai", + "max_input_tokens": 1048576, + "max_tokens": 1048576, + "mode": "chat", + "output_cost_per_token": 6e-07, + "source": "https://www.together.ai/pricing", + "supports_prompt_caching": true + }, "together_ai/Prism-ML/Ternary-Bonsai-27B": { "input_cost_per_token": 0.0, "litellm_provider": "together_ai", @@ -52906,12 +53141,14 @@ }, "vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": { "input_cost_per_token": 3.5e-07, + "input_cost_per_token_batches": 1.75e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 1000000, "max_output_tokens": 1000000, "max_tokens": 1000000, "mode": "chat", "output_cost_per_token": 1.15e-06, + "output_cost_per_token_batches": 5.75e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -52926,12 +53163,14 @@ }, "vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": { "input_cost_per_token": 2.5e-07, + "input_cost_per_token_batches": 1.25e-07, "litellm_provider": "vertex_ai-llama_models", "max_input_tokens": 10000000, "max_output_tokens": 10000000, "max_tokens": 10000000, "mode": "chat", "output_cost_per_token": 7e-07, + "output_cost_per_token_batches": 3.5e-07, "source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models", "supported_modalities": [ "text", @@ -73330,6 +73569,7 @@ ] }, "vertex_ai/veo-2.0-generate-001": { + "deprecation_date": "2026-06-30", "litellm_provider": "vertex_ai-video-models", "max_input_tokens": 1024, "max_tokens": 1024, diff --git a/pyproject.toml b/pyproject.toml index b43f0b254fd..0dabba4e450 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,16 +177,16 @@ proxy-runtime = [ "grpcio==1.84.1", "prometheus-client>=0.20.0,<1.0", "langfuse>=4.7,<5.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "ddtrace>=4.8.2,<5.0", "sentry-sdk>=2.21.0,<3.0", "mangum>=0.17.0,<1.0", "azure-ai-contentsafety>=1.0.0,<2.0", "azure-storage-file-datalake>=12.20.0,<13.0", - "pypdf>=6.16.1,<7.0", + "pypdf>=6.21.1,<7.0", "llm-sandbox>=0.3.39,<1.0", "detect-secrets>=1.5.0,<2.0", ] @@ -223,10 +223,10 @@ dev = [ "types-PyYAML==6.0.12.20250915", "botocore-stubs==1.43.14", "types-boto3[bedrock,bedrock-agent,bedrock-runtime,kms,s3,sagemaker-runtime,sts]==1.43.30", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "langfuse>=4.7,<5.0", "fastapi-offline==1.7.6", "fakeredis==2.34.1", @@ -255,10 +255,10 @@ proxy-dev = [ "hypercorn==0.17.3", "prometheus-client==0.20.0", "sentry-sdk==2.21.0", - "opentelemetry-api==1.33.1", - "opentelemetry-sdk==1.33.1", - "opentelemetry-exporter-otlp==1.33.1", - "opentelemetry-instrumentation-fastapi==0.54b1", + "opentelemetry-api==1.45.1", + "opentelemetry-sdk==1.45.1", + "opentelemetry-exporter-otlp==1.45.1", + "opentelemetry-instrumentation-fastapi==0.66b1", "azure-identity==1.25.2", "a2a-sdk==1.1.0", "backports-zstd==1.8.0; python_version < '3.14'", @@ -280,7 +280,7 @@ ci = [ "langchain==1.3.9", "lunary==1.4.36; python_version == '3.10'", "lunary==1.4.37; python_version >= '3.11'", - "logfire==4.6.0", + "logfire==5.2.0", "traceloop-sdk==0.34.0", "detect-secrets==1.5.0", "PyGithub==2.8.1", @@ -370,10 +370,17 @@ override-dependencies = [ default-groups = ["dev"] required-version = ">=0.10.9" exclude-newer = "3 days" + +[tool.uv.exclude-newer-package] # TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window +certifi = "2026-10-20T03:00:00Z" # TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window +grpcio = "2026-10-22T15:00:00Z" # TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window -exclude-newer-package = { certifi = "2026-10-20T03:00:00Z", grpcio = "2026-10-22T15:00:00Z", boto3 = "2026-10-21T19:31:00Z", botocore = "2026-10-21T19:31:00Z" } +boto3 = "2026-10-21T19:31:00Z" +botocore = "2026-10-21T19:31:00Z" +# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window +brotli = "2026-10-22T14:01:00Z" [tool.uv.sources] litellm-proxy-extras = { workspace = true } diff --git a/tests/integration/compatibility/test_status_code_followups_usage_chat.py b/tests/integration/compatibility/test_status_code_followups_usage_chat.py deleted file mode 100644 index de9ec16cf30..00000000000 --- a/tests/integration/compatibility/test_status_code_followups_usage_chat.py +++ /dev/null @@ -1,125 +0,0 @@ -"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check -the daily activity routes answer 400 with, instead of a generic fetch error.""" - -from __future__ import annotations - -import json -import uuid -from collections.abc import Iterator -from dataclasses import dataclass -from typing import Final - -import pytest -from integration._support.client import JSON_OBJECT, Gateway, object_value -from integration._support.upstream import delete_scenario, register_scenario -from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway -from pydantic import JsonValue - -pytestmark: Final = pytest.mark.timeout(180) - -_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data") - - -@dataclass(frozen=True, slots=True) -class _UsageChat: - gateway: Gateway - identity: str - - -@pytest.fixture(scope="module") -def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]: - """A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream.""" - identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}" - handle: Final = register_scenario(identity, json_response({})) - try: - with owned_gateway( - tmp_path_factory.mktemp("audit-usage-chat"), - {"model_list": [], "router_settings": {"num_retries": 0}}, - environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity}, - ) as owned: - yield _UsageChat(owned, identity) - finally: - delete_scenario(handle) - - -def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]: - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - return { - "id": "chatcmpl-usage", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}} - for tool in _TOOLS - ], - }, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, - } - - -def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]: - """Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up - model call carried, keyed by tool, and the tool statuses the SSE stream reported.""" - register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date))) - Upstream(usage_chat.gateway.upstream_url).reset() - with usage_chat.gateway.client.stream( - "POST", - "/usage/ai/chat", - json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]}, - headers={"Authorization": f"Bearer {usage_chat.gateway.key}"}, - timeout=60, - ) as response: - assert response.status_code == 200 - events: Final = tuple( - object_value(JSON_OBJECT.validate_json(line.removeprefix("data: "))) - for line in response.iter_lines() - if line.startswith("data: ") - ) - calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity) - assert len(calls) == 2, calls - messages: Final = object_value(calls[1]["body"]).get("messages") - assert isinstance(messages, list), calls[1] - tool_messages: Final = { - str(message["tool_call_id"]).removeprefix("call-"): str(message["content"]) - for message in messages - if isinstance(message, dict) and message.get("role") == "tool" - } - statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call") - return tool_messages, statuses - - -_REVERSED: Final = "Invalid date range: end_date must be on or after start_date" -_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates" -_NO_TEAM_DATA: Final = "No Team usage data found for the given date range." -_NO_TAG_DATA: Final = "No Tag usage data found for the given date range." - - -@pytest.mark.parametrize( - ("start_date", "end_date", "team_message", "tag_message"), - ( - pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"), - pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"), - pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"), - pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"), - ), -) -def test_usage_chat_tools_answer_with_the_query_layers_date_check( - usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str -) -> None: - """The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query - takes these dates without that check, still answers with its usage summary.""" - tool_messages, statuses = _tool_results(usage_chat, start_date, end_date) - assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages - assert tool_messages["get_team_usage_data"] == team_message, tool_messages - assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages - assert statuses == ("running", "complete") * len(_TOOLS), statuses diff --git a/tests/integration/management/test_sso_sign_in.py b/tests/integration/management/test_sso_sign_in.py index c0c03571a13..d0b6e2579c8 100644 --- a/tests/integration/management/test_sso_sign_in.py +++ b/tests/integration/management/test_sso_sign_in.py @@ -3,7 +3,9 @@ import contextlib import json import uuid from collections.abc import Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor from contextlib import contextmanager +from functools import partial from pathlib import Path from typing import Final from urllib.parse import parse_qs, urlencode, urlparse @@ -15,7 +17,7 @@ from pydantic import JsonValue from tests.integration._support.client import Gateway, gateway_from_environment from tests.integration._support.database import read_rows -from tests.integration._support.process import owned_proxy +from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy from tests.integration._support.wire import Reply, Request, wire_server pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state] @@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role( "metadata": {"auth_provider": IDP_PROVIDER}, } ] + + +def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value) + monkeypatch.delenv("LITELLM_LICENSE", raising=False) + with _sso_gateway(tmp_path, {}) as gateway: + for index in range(4): + signed_in: Final = _sign_in(gateway, f"free-sso-{index}") + assert signed_in.status_code == 303, signed_in.text + + contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8)) + with ThreadPoolExecutor(max_workers=len(contenders)) as pool: + responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders)) + + assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403] + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}] + for subject, response in zip(contenders, responses): + if response.status_code == 403: + assert "requires a LiteLLM Enterprise license" in response.text + assert not response.cookies + assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == [] + + existing: Final = _sign_in(gateway, "free-sso-0") + assert existing.status_code == 303, existing.text + + admin_created: Final = gateway.request( + "POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False} + ) + assert admin_created.status_code == 200, admin_created.text + above_capacity: Final = _sign_in(gateway, "free-sso-0") + refused: Final = _sign_in(gateway, "free-sso-extra") + assert (above_capacity.status_code, refused.status_code) == (303, 403) + assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}] diff --git a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py index 780ee43fb3f..0c18f2a4b6f 100644 --- a/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py +++ b/tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py @@ -99,16 +99,9 @@ class _Activity(BaseModel): results: tuple[_Day, ...] -class _UpstreamMessage(BaseModel): - model_config = ConfigDict(frozen=True) - role: str | None = None - - class _UpstreamRequest(BaseModel): model_config = ConfigDict(frozen=True) stream: bool | None = None - tools: tuple[JsonValue, ...] | None = None - messages: tuple[_UpstreamMessage, ...] = () @dataclass(frozen=True, slots=True) @@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]: return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n") -def _usage_tool_call() -> Mapping[str, JsonValue]: - arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)}) - return { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}} - ], - } - - def _response_object() -> bytes: return json.dumps( { @@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply: body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}") if body.stream: return Reply(content_type="text/event-stream", chunks=_stream_frames()) - roles: Final = frozenset(message.role for message in body.messages) - if body.tools and "tool" not in roles: - return Reply(body=_completion(_usage_tool_call(), "tool_calls")) return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop")) @@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None: ) -def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]: - return tuple( - json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ") - ) - - @dataclass(frozen=True, slots=True) class Tenant: label: str @@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]: @pytest.mark.timeout(360) -def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( +def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees( gateway: Gateway, tmp_path: Path ) -> None: with _rig(gateway, tmp_path) as (proxy, database_url): - spender: Final = _spender(proxy, database_url, "ai-chat") + spender: Final = _spender(proxy, database_url, "repeated-timeout") with _pinned(proxy) as pinned: with _locked_spend_logs(database_url): - with _recording(database_url) as during_chat: - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - assert len(_lookups(during_chat)) == 1 + with _recording(database_url) as during_timeout: + timed_out: Final = _read(pinned, spender.digest) + assert timed_out.status_code == 200, timed_out.text + assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed + assert _aliases(timed_out, spender.digest) == (None,), timed_out.text + assert len(_lookups(during_timeout)) == 1 cached_miss: Final = _read(pinned, spender.digest) assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text @@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked assert gateway_activity == 200, gateway_activity chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}") assert object_value(chat["usage"]) == dict(USAGE), chat - - -@pytest.mark.timeout(300) -def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery( - gateway: Gateway, tmp_path: Path -) -> None: - with ( - scratch_database() as database_url, - database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url), - wire_server(_respond) as wire, - _proxy(gateway, tmp_path, relayed_url, wire.url) as proxy, - ): - spender: Final = _spender(proxy, database_url, "dropped-ai-chat") - with _pinned(proxy) as pinned: - relay.arm() - chat: Final = pinned.request( - "POST", - "/usage/ai/chat", - body={ - "messages": [{"role": "user", "content": "What did we spend?"}], - "model": "openai/gpt-4o-mini", - }, - ) - assert chat.status_code == 200, chat.text - assert relay.tripped.is_set(), chat.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": _day(-1), "end_date": _day(1)}, - } - assert _events(chat) == ( - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": REPLY_TEXT}, - {"type": "done"}, - ), chat.text - recovered: Final = _named_once_reconnected(pinned, spender) - assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text diff --git a/tests/integration/spend/test_daily_activity_key_owner_traffic.py b/tests/integration/spend/test_daily_activity_key_owner_traffic.py index b8f113aca49..5284ad631de 100644 --- a/tests/integration/spend/test_daily_activity_key_owner_traffic.py +++ b/tests/integration/spend/test_daily_activity_key_owner_traffic.py @@ -6,7 +6,6 @@ from collections.abc import Iterable from concurrent.futures import ThreadPoolExecutor from datetime import UTC, datetime, timedelta from hashlib import sha256 -from pathlib import Path from queue import SimpleQueue from typing import Final @@ -31,8 +30,7 @@ from integration._support.daily_activity import ( user_row, user_with_an_email, ) -from integration._support.database import read_rows, scratch_database -from integration._support.process import owned_proxy +from integration._support.database import read_rows from integration._support.wire import Reply, Request, wire_server from pydantic import JsonValue @@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons REQUESTS_OF_A_BURST: Final = 21 READS_DURING_A_BURST: Final = 30 TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models") -TOOL_CALL: Final = "call_integration_usage" -ANSWER: Final = "One request cost $0.25" -SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join( - ( - "Total Spend: $0.2500", - "Total Requests: 1", - "Successful: 1 | Failed: 0", - "Total Tokens: 15", - "", - "Top Models by Spend:", - " - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)", - "", - "Top Providers by Spend:", - " - openai: $0.2500 (1 reqs)", - ) -) def _chat_completion() -> dict[str, JsonValue]: @@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply: return Reply(body=json.dumps(body).encode()) -def _usage_tool_call() -> dict[str, JsonValue]: - call: Final[dict[str, JsonValue]] = { - "id": TOOL_CALL, - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": json.dumps({"start_date": DAY, "end_date": DAY}), - }, - } - return { - "id": f"chatcmpl-{uuid.uuid4().hex}", - "object": "chat.completion", - "created": 1, - "model": "gpt-4o-mini", - "choices": [ - { - "index": 0, - "message": {"role": "assistant", "content": None, "tool_calls": [call]}, - "finish_reason": "tool_calls", - } - ], - "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, - } - - -def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes: - chunk: Final = { - "id": "chatcmpl-integration-usage", - "object": "chat.completion.chunk", - "created": 1, - "model": "gpt-4o-mini", - "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], - } - return f"data: {json.dumps(chunk)}\n\n".encode() - - -def _usage_analyst(request: Request) -> Reply: - if json.loads(request.body).get("stream"): - return Reply( - chunks=( - _streamed_chunk({"role": "assistant", "content": ANSWER}, None), - _streamed_chunk({}, "stop"), - b"data: [DONE]\n\n", - ), - content_type="text/event-stream", - ) - return Reply(body=json.dumps(_usage_tool_call()).encode()) - - def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]: return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY) @@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session( ) -@pytest.mark.timeout(300) -def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner( - gateway: Gateway, tmp_path: Path -) -> None: - question: Final = f"what did we spend {uuid.uuid4().hex}" - owner: Final = f"integration-{uuid.uuid4().hex}" - ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}" - with ( - scratch_database() as scratch_url, - wire_server(_usage_analyst) as wire, - owned_proxy( - gateway, - tmp_path, - { - "DATABASE_URL": scratch_url, - "OPENAI_API_BASE": f"{wire.url}/v1", - "OPENAI_BASE_URL": f"{wire.url}/v1", - "OPENAI_API_KEY": "integration-provider-key", - }, - remove_environment=("DATABASE_URL_READ_REPLICA",), - ) as candidate, - ): - candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False}) - with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url): - answer: Final = candidate.request( - "POST", - "/usage/ai/chat", - {"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"}, - ) - assert answer.status_code == 200, answer.text - tool_call: Final = { - "type": "tool_call", - "tool_name": "get_usage_data", - "tool_label": "global usage data", - "arguments": {"start_date": DAY, "end_date": DAY}, - } - events: Final = [ - json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ") - ] - assert events == [ - {"type": "status", "message": "Thinking..."}, - {**tool_call, "status": "running"}, - {**tool_call, "status": "complete"}, - {"type": "status", "message": "Analyzing results..."}, - {"type": "chunk", "content": ANSWER}, - {"type": "done"}, - ], answer.text - asked, analysed = wire.drain() - assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"] - assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question} - assert json.loads(analysed.body)["messages"][-1] == { - "role": "tool", - "tool_call_id": TOOL_CALL, - "content": SUMMARY_OF_ONE_SEEDED_ROW, - } - assert owner not in analysed.body.decode() - assert ownerless_key not in analysed.body.decode() - - @pytest.mark.timeout(300) def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None: released: Final = threading.Event() diff --git a/tests/unit/integrations/otel/test_otel_v2_components.py b/tests/unit/integrations/otel/test_otel_v2_components.py index 3f4389765d3..8af13fd6e7e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_components.py +++ b/tests/unit/integrations/otel/test_otel_v2_components.py @@ -14,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer import pytest import requests +from requests_mock import Adapter pytest.importorskip("opentelemetry") @@ -22,16 +23,18 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa: ) from opentelemetry import baggage # noqa: E402 from opentelemetry.context import attach, detach # noqa: E402 -from opentelemetry._logs.severity import SeverityNumber # noqa: E402 -from opentelemetry.sdk._logs import LogData, LogRecord # noqa: E402 -from opentelemetry.sdk._logs.export import LogExportResult # noqa: E402 +from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above +from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above +from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above from opentelemetry.sdk.metrics import MeterProvider # noqa: E402 from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402 -from opentelemetry.sdk.trace import TracerProvider # noqa: E402 +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above from opentelemetry.sdk.trace.export import ( # noqa: E402 BatchSpanProcessor, ConsoleSpanExporter, SimpleSpanProcessor, + SpanExportResult, ) from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E402 InMemorySpanExporter, @@ -46,6 +49,7 @@ import litellm # noqa: E402 from tests.unit.integrations.conftest import TlsSink # noqa: E402 from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 +from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402 from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402 from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402 @@ -733,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict] class Collector(BaseHTTPRequestHandler): def do_POST(self): body = self.rfile.read(int(self.headers["Content-Length"])) - received.append({"path": self.path, "headers": dict(self.headers), "body": body}) + received.append( + {"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body} + ) self.send_response(200) self.end_headers() @@ -793,6 +799,95 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint(): assert child["parentSpanId"] == format(parent_id, "016x") +@pytest.mark.parametrize( + ("statuses", "expected"), + [ + ((200,), SpanExportResult.SUCCESS), + ((400,), SpanExportResult.FAILURE), + ((503, 200), SpanExportResult.SUCCESS), + ], +) +def test_http_json_exporter_preserves_http_failure_and_retry_results( + statuses: tuple[int, ...], expected: SpanExportResult +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri( + "POST", + endpoint, + [{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses], + ) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter( + endpoint=endpoint, + headers={ + "CONTENT-TYPE": "application/x-protobuf", + "x-collector-token": "secret", + }, + session=session, + ) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == len(statuses) + for request in adapter.request_history: + assert request.headers["Content-Type"] == "application/json" + assert request.headers["x-collector-token"] == "secret" + assert request.json() == {"resourceSpans": []} + + +@pytest.mark.parametrize( + ("size_limit", "expected", "request_count"), + [ + (0, SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1), + (len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0), + ], +) +def test_http_json_exporter_checks_json_size_before_sending( + size_limit: int, expected: SpanExportResult, request_count: int +) -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit) + try: + assert exporter.export(()) is expected + finally: + exporter.shutdown() + assert len(adapter.request_history) == request_count + + +def test_http_json_exporter_rejects_export_after_shutdown() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + exporter.shutdown() + assert exporter.export(()) is SpanExportResult.FAILURE + assert adapter.request_history == [] + + +def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None: + adapter: Final = Adapter() + endpoint: Final = "https://collector.test/v1/traces" + adapter.register_uri("POST", endpoint, status_code=200) + with requests.Session() as session: + session.mount("https://", adapter) + exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session) + try: + assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE + finally: + exporter.shutdown() + assert adapter.request_history == [] + + def test_http_protobuf_exporter_still_posts_protobuf(): (_child_request, parent_request), (trace_id, _parent_id, _child_id) = _export_one_trace_to_local_collector( "http/protobuf" @@ -1630,9 +1725,13 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey severity_number=SeverityNumber.INFO, body="v2-tls-test", ) - log_data = LogData(log_record=record, instrumentation_scope=InstrumentationScope("v2-tls-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("v2-tls-test"), + ) result = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() diff --git a/tests/unit/integrations/otel/test_otel_v2_mount.py b/tests/unit/integrations/otel/test_otel_v2_mount.py index 3a27c3ec4e4..61f8b9ae0d9 100644 --- a/tests/unit/integrations/otel/test_otel_v2_mount.py +++ b/tests/unit/integrations/otel/test_otel_v2_mount.py @@ -382,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch): assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n" for path, response in responses.items(): context = server_spans[path].context - assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-01"] + assert response.headers.get_list("traceresponse") == [ + f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" + ] @pytest.mark.parametrize("path", ["/relayed", "/relayed/health"]) @@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo headers = contextvars.Context().run(trace_response_headers, span) - assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-01" + assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}" diff --git a/tests/unit/integrations/otel/test_otel_v2_presets.py b/tests/unit/integrations/otel/test_otel_v2_presets.py index 64366626784..570c27a24e1 100644 --- a/tests/unit/integrations/otel/test_otel_v2_presets.py +++ b/tests/unit/integrations/otel/test_otel_v2_presets.py @@ -2,13 +2,19 @@ event loop: the preset does no network I/O, and a custom exporter mints the JWT lazily on its first export (in the BatchSpanProcessor worker thread).""" +import json + import httpx import pytest +import requests_mock +import respx +from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.otel.plumbing import providers from litellm.integrations.otel.model.config import ExporterSpec from litellm.integrations.otel.presets import agentops as agentops_mod from litellm.integrations.otel.presets.agentops import ( + _AGENTOPS_AUTH_ENDPOINT, _AGENTOPS_ENDPOINT, _AGENTOPS_EXPORTER_KIND, _build_agentops_exporter, @@ -75,50 +81,44 @@ def test_dynamic_cred_presets_tag_exporter_with_matching_owner(monkeypatch): ) -def test_agentops_exporter_mints_jwt_lazily(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - monkeypatch.setattr( - agentops_mod, "_fetch_agentops_jwt", lambda _k: {"token": "jwt-xyz"} - ) - spec = ExporterSpec( - kind=_AGENTOPS_EXPORTER_KIND, - endpoint=_AGENTOPS_ENDPOINT, - options={"api_key": "ak"}, - ) - exporter = _build_agentops_exporter(spec) - - # No auth header until the first export triggers the (off-loop) fetch. - assert "Authorization" not in exporter._session.headers - exporter._ensure_authenticated() - assert exporter._session.headers["Authorization"] == "Bearer jwt-xyz" - - # Cached: a second resolution does not re-fetch. - calls = [] - monkeypatch.setattr( - agentops_mod, - "_fetch_agentops_jwt", - lambda k: calls.append(k) or {"token": "again"}, - ) - exporter._ensure_authenticated() - assert calls == [] - - -def test_agentops_exporter_tolerates_fetch_failure(monkeypatch): - pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") - - def _raise(_k): - raise RuntimeError("auth down") - - monkeypatch.setattr(agentops_mod, "_fetch_agentops_jwt", _raise) - exporter = _build_agentops_exporter( +def _agentops_exporter(): + return _build_agentops_exporter( ExporterSpec( kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, options={"api_key": "ak"}, ) ) - exporter._ensure_authenticated() # must not raise - assert "Authorization" not in exporter._session.headers + + +def test_agentops_exporter_mints_the_jwt_once_on_first_export_and_sends_it_with_every_batch(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + token_route = auth.post(_AGENTOPS_AUTH_ENDPOINT).respond(200, json={"token": "jwt-xyz"}) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + exporter = _agentops_exporter() + assert token_route.call_count == 0, "building the exporter must not fetch the JWT" + + results = [exporter.export(()), exporter.export(())] + + assert results == [SpanExportResult.SUCCESS, SpanExportResult.SUCCESS] + assert [json.loads(call.request.content) for call in token_route.calls] == [{"api_key": "ak"}] + assert [request.headers.get("Authorization") for request in collector.request_history] == [ + "Bearer jwt-xyz", + "Bearer jwt-xyz", + ] + + +def test_agentops_exporter_still_exports_without_auth_when_the_jwt_fetch_fails(): + pytest.importorskip("opentelemetry.exporter.otlp.proto.http.trace_exporter") + with respx.mock() as auth, requests_mock.Mocker() as collector: + auth.post(_AGENTOPS_AUTH_ENDPOINT).mock(side_effect=httpx.ConnectError("auth down")) + collector.post(_AGENTOPS_ENDPOINT, status_code=200) + + result = _agentops_exporter().export(()) + + assert result is SpanExportResult.SUCCESS + assert [request.headers.get("Authorization") for request in collector.request_history] == [None] def test_fetch_jwt_uses_owned_client_not_shared_pool(monkeypatch): diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 9f536ab0dac..c7608159515 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -513,6 +513,22 @@ class TestLangfuseOtelIntegration: # Endpoint assertion removed as side effect is gone +def _exported_auth_headers(exporter): + import requests_mock + from opentelemetry.sdk.trace.export import SpanExportResult + + with requests_mock.Mocker() as collector: + collector.post(requests_mock.ANY, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + return [ + ( + request.url, + {name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")}, + ) + for request in collector.request_history + ] + + class TestLangfuseOtelKeyDynamicConfig: """Key/team-scoped Langfuse credentials must define the full export target (OTLP endpoint + auth), not just auth headers on the init-time exporter.""" @@ -617,15 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig: exporter = span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces" - import base64 expected_auth = base64.b64encode(b"key_public:key_secret").decode() - assert exporter._headers == { - "Authorization": f"Basic {expected_auth}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"}, + ) + ] def test_key_dynamic_params_reuse_cached_provider(self): with self._clean_env(): @@ -678,10 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig: provider = next(iter(logger._tracer_provider_cache.values())).provider exporter = provider._active_span_processor._span_processors[0].span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._headers == { - "Authorization": f"Basic {secret}", - "x-langfuse-ingestion-version": "4", - } + assert _exported_auth_headers(exporter) == [ + ( + "https://langfuse.example.com/api/public/otel/v1/traces", + {"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"}, + ) + ] class TestLangfuseOtelResponsesAPI: diff --git a/tests/unit/integrations/test_opentelemetry.py b/tests/unit/integrations/test_opentelemetry.py index d14703e48a1..f908c685382 100644 --- a/tests/unit/integrations/test_opentelemetry.py +++ b/tests/unit/integrations/test_opentelemetry.py @@ -19,14 +19,16 @@ import pytest # Adds the grandparent directory to sys.path to allow importing project modules from opentelemetry import trace +from opentelemetry._logs import LogRecord from opentelemetry._logs.severity import SeverityNumber -from opentelemetry.sdk._logs import LogData, LogRecord +from opentelemetry.sdk._logs import ReadableLogRecord from opentelemetry.sdk._logs import LoggerProvider as OTLoggerProvider -from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogExportResult, SimpleLogRecordProcessor +from opentelemetry.sdk._logs.export import InMemoryLogExporter, LogRecordExportResult, SimpleLogRecordProcessor from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import InMemoryMetricReader, MetricsData +from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import ReadableSpan, SpanLimits, TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExportResult +from opentelemetry.sdk.trace.export import BatchSpanProcessor, SimpleSpanProcessor, SpanExportResult from opentelemetry.sdk.util.instrumentation import InstrumentationScope from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from parameterized import parameterized @@ -2321,9 +2323,13 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat severity_number=SeverityNumber.INFO, body="tls-export-test", ) - log_data: Final = LogData(log_record=record, instrumentation_scope=InstrumentationScope("tls-export-test")) + log_data: Final = ReadableLogRecord( + log_record=record, + resource=Resource({}), + instrumentation_scope=InstrumentationScope("tls-export-test"), + ) result: Final = exporter.export([log_data]) - assert result is LogExportResult.SUCCESS, f"log export failed: {result}" + assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}" assert tls_sink.received.get(timeout=5) == "/v1/logs" finally: exporter.shutdown() @@ -6639,7 +6645,7 @@ class TestOpenTelemetryProviderlessCallAttributes(unittest.TestCase): time.sleep(self.POLL_INTERVAL) return None - def _emitted_log_records(self, semconv_opt_in: str) -> tuple[LogData, ...]: + def _emitted_log_records(self, semconv_opt_in: str) -> tuple[ReadableLogRecord, ...]: log_exporter = InMemoryLogExporter() logger_provider = OTLoggerProvider() logger_provider.add_log_record_processor(SimpleLogRecordProcessor(log_exporter)) @@ -6713,7 +6719,12 @@ class TestDynamicTracerProviderCache(unittest.TestCase): must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated until pods were OOMKilled).""" - BSP_THREAD_NAME = "OtelBatchSpanProcessor" + def setUp(self) -> None: + processor: Final = BatchSpanProcessor(InMemorySpanExporter()) + try: + self.bsp_thread_name = processor._batch_processor._worker_thread.name + finally: + processor.shutdown() def _logger(self, cap=3, exporter="console"): logger = OpenTelemetry( @@ -6730,7 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._tracer_provider_cache.clear() def _live_exporter_threads(self): - return [t for t in threading.enumerate() if t.name == self.BSP_THREAD_NAME] + return [t for t in threading.enumerate() if t.name == self.bsp_thread_name] def _wait_for_exporter_threads(self, expected, timeout=10.0): """Dropped providers are shut down off-thread, so poll instead of sleeping.""" @@ -6751,8 +6762,6 @@ class TestDynamicTracerProviderCache(unittest.TestCase): logger._get_tracer_with_dynamic_headers({"authorization": f"Basic tenant-{i}"}) self.assertEqual(len(logger._tracer_provider_cache), 3) - # Guards the thread-name constant: a rename upstream would make this read 0 and the - # bound assertion below would pass while measuring nothing. self.assertGreaterEqual(len(self._live_exporter_threads()), 1) self.assertLessEqual(self._wait_for_exporter_threads(before + 3) - before, 3) diff --git a/tests/unit/litellm_core_utils/test_litellm_logging.py b/tests/unit/litellm_core_utils/test_litellm_logging.py index bbfae2229ce..8509b1d6f7d 100644 --- a/tests/unit/litellm_core_utils/test_litellm_logging.py +++ b/tests/unit/litellm_core_utils/test_litellm_logging.py @@ -2047,7 +2047,9 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( ) -> None: """The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key, without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS.""" + import requests_mock from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter + from opentelemetry.sdk.trace.export import SpanExportResult from litellm.integrations.opentelemetry import OpenTelemetry from litellm.litellm_core_utils import litellm_logging as logging_module @@ -2069,8 +2071,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key( assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace" exporter: Final = logger._get_span_processor().span_exporter assert isinstance(exporter, OTLPSpanExporter) - assert exporter._endpoint == expected_endpoint - assert exporter._headers == {"x-api-key": api_key} + with requests_mock.Mocker() as collector: + collector.post(expected_endpoint, status_code=200) + assert exporter.export(()) is SpanExportResult.SUCCESS + assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [ + (expected_endpoint, api_key) + ] assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ finally: logging_module._in_memory_loggers.clear() diff --git a/tests/unit/proxy/management_endpoints/test_ui_sso.py b/tests/unit/proxy/management_endpoints/test_ui_sso.py index 479391e54fe..e1a939273ae 100644 --- a/tests/unit/proxy/management_endpoints/test_ui_sso.py +++ b/tests/unit/proxy/management_endpoints/test_ui_sso.py @@ -9198,58 +9198,6 @@ async def test_saml_callback_blocked_when_admin_ui_disabled(): assert "Admin UI is Disabled" in response.body.decode() -@pytest.mark.asyncio -async def test_saml_callback_enforces_free_sso_user_limit_after_validation(): - """An IdP-initiated assertion must not bypass the >5 free-SSO-user Enterprise gate - that /sso/key/generate enforces; the ACS re-checks it after validating the assertion, - so the entitlement DB query never runs on unvalidated input.""" - from litellm.proxy._types import ProxyException - from litellm.proxy.management_endpoints.types import CustomOpenID - from litellm.proxy.management_endpoints.ui_sso import saml_callback - - call_order: list[str] = [] - - async def _fake_handle_acs(**kwargs): - call_order.append("validate") - return (CustomOpenID( - id="dana@litellm.ai", - email="dana@litellm.ai", - first_name=None, - last_name=None, - display_name="dana", - picture=None, - provider="saml", - team_ids=[], - user_role=None, - ), None) - - async def _fake_count_billable_users(): - call_order.append("count") - return 6 - - async def _stream(): - yield b"SAMLResponse=signed-response" - - request_double = SimpleNamespace(cookies={}, headers={}, stream=_stream) - - with patch.dict(os.environ, {"DISABLE_ADMIN_UI": "false"}), patch( - "litellm.proxy.proxy_server.premium_user", False - ), patch("litellm.proxy.proxy_server.prisma_client", MagicMock()), patch( - "litellm.proxy.proxy_server.master_key", MASTER_KEY - ), patch( - "litellm.proxy.management_endpoints.sso.saml_sso.SAMLAuthHandler.handle_acs", - new=_fake_handle_acs, - ), patch( - "litellm.repositories.user_repository.UserRepository.count_billable_users", - new=AsyncMock(side_effect=_fake_count_billable_users), - ): - with pytest.raises(ProxyException) as exc: - await saml_callback(request_double) - - assert str(exc.value.code) == "403" - assert call_order == ["validate", "count"] - - @pytest.mark.asyncio async def test_cli_poll_key_tolerates_missing_user_row(): """The CLI poll must still mint the JWT when the user lookup raises, diff --git a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py b/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py deleted file mode 100644 index 4daddb89350..00000000000 --- a/tests/unit/proxy/management_endpoints/usage_endpoints/test_ai_usage_chat.py +++ /dev/null @@ -1,657 +0,0 @@ -""" -Tests for AI Usage Chat module. -""" - -import json -from collections.abc import AsyncIterator -from typing import Final -from unittest.mock import AsyncMock, MagicMock, patch - -import pytest -from fastapi import HTTPException - -from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - TOOL_HANDLERS, - TOOLS_ADMIN, - TOOLS_BASE, - _build_system_prompt, - _summarise_entity_data, - _summarise_usage_data, - stream_usage_ai_chat, -) - - -SAMPLE_AGGREGATED_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": { - "spend": 50.25, - "prompt_tokens": 20000, - "completion_tokens": 10000, - "total_tokens": 30000, - "api_requests": 500, - "successful_requests": 480, - "failed_requests": 20, - "cache_read_input_tokens": 0, - "cache_creation_input_tokens": 0, - }, - "breakdown": { - "models": { - "gpt-4": { - "metrics": { - "spend": 40.0, - "api_requests": 300, - "total_tokens": 25000, - }, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "providers": { - "openai": { - "metrics": {"spend": 50.25, "api_requests": 500}, - "metadata": {}, - "api_key_breakdown": {}, - }, - }, - "api_keys": { - "sk-test123": { - "metrics": {"spend": 50.25}, - "metadata": {"key_alias": "Production Key"}, - }, - }, - "model_groups": {}, - "mcp_servers": {}, - "entities": {}, - }, - }, - ], - "metadata": { - "total_spend": 50.25, - "total_api_requests": 500, - "total_successful_requests": 480, - "total_failed_requests": 20, - "total_tokens": 30000, - }, -} - -SAMPLE_TEAM_RESPONSE = { - "results": [ - { - "date": "2025-01-15", - "metrics": {"spend": 100.0, "api_requests": 1000, "total_tokens": 50000}, - "breakdown": { - "entities": { - "team-1": { - "metrics": { - "spend": 60.0, - "api_requests": 600, - "total_tokens": 30000, - }, - "metadata": {"alias": "Engineering"}, - "api_key_breakdown": {}, - }, - "team-2": { - "metrics": { - "spend": 40.0, - "api_requests": 400, - "total_tokens": 20000, - }, - "metadata": {"alias": "Marketing"}, - "api_key_breakdown": {}, - }, - }, - "models": {}, - "providers": {}, - "api_keys": {}, - "model_groups": {}, - "mcp_servers": {}, - }, - }, - ], - "metadata": {"total_spend": 100.0, "total_api_requests": 1000}, -} - - -class TestToolSchemas: - def test_admin_tools_include_all(self): - assert len(TOOLS_ADMIN) == 3 - names = {t["function"]["name"] for t in TOOLS_ADMIN} - assert "get_usage_data" in names - assert "get_team_usage_data" in names - assert "get_tag_usage_data" in names - - def test_base_tools_restricted_to_usage_only(self): - assert len(TOOLS_BASE) == 1 - assert TOOLS_BASE[0]["function"]["name"] == "get_usage_data" - - def test_admin_prompt_mentions_all_tools(self): - prompt = _build_system_prompt(is_admin=True) - assert "get_usage_data" in prompt - assert "get_team_usage_data" in prompt - assert "get_tag_usage_data" in prompt - - def test_non_admin_prompt_only_mentions_usage_tool(self): - prompt = _build_system_prompt(is_admin=False) - assert "get_usage_data" in prompt - assert "get_team_usage_data" not in prompt - assert "get_tag_usage_data" not in prompt - - def test_system_prompt_includes_todays_date(self): - from datetime import date - - prompt = _build_system_prompt(is_admin=True) - assert date.today().isoformat() in prompt - - -class TestSummariseUsageData: - def test_summarise_includes_totals(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "$50.25" in summary - assert "500" in summary - - def test_summarise_includes_models(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "gpt-4" in summary - - def test_summarise_includes_providers(self): - summary = _summarise_usage_data(SAMPLE_AGGREGATED_RESPONSE) - assert "openai" in summary - - def test_summarise_handles_empty_data(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_usage_data(empty) - assert "no data" in summary.lower() - - -class TestSummariseEntityData: - def test_team_summary_includes_teams(self): - summary = _summarise_entity_data(SAMPLE_TEAM_RESPONSE, "Team") - assert "Engineering" in summary - assert "Marketing" in summary - assert "$60.0" in summary - assert "$40.0" in summary - - def test_team_summary_empty(self): - empty = {"results": [], "metadata": {}} - summary = _summarise_entity_data(empty, "Team") - assert "No Team usage data" in summary - - -class TestStreamUsageAiChat: - @pytest.mark.asyncio - async def test_stream_emits_status_events(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_123" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_123", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Total spend is $50.25" - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_AGGREGATED_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "What is my total spend?"}], - model="gpt-4o-mini", - user_id="user-123", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - status_events = [e for e in events if e["type"] == "status"] - tool_call_events = [e for e in events if e["type"] == "tool_call"] - chunk_events = [e for e in events if e["type"] == "chunk"] - done_events = [e for e in events if e["type"] == "done"] - - assert len(status_events) >= 1 - assert "Thinking" in status_events[0]["message"] - assert len(tool_call_events) >= 1 - assert tool_call_events[0]["tool_name"] == "get_usage_data" - assert tool_call_events[0]["status"] in ("running", "complete") - assert len(chunk_events) >= 1 - assert len(done_events) == 1 - - @pytest.mark.asyncio - async def test_stream_handles_team_tool(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_team" - mock_tool_call.function.name = "get_team_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_team", - "type": "function", - "function": { - "name": "get_team_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Engineering is the top team." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat._fetch_team_usage_data", - new_callable=AsyncMock, - ) as mock_fetch, - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - mock_fetch.return_value = SAMPLE_TEAM_RESPONSE - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Which team spends the most?"}], - model="gpt-4o-mini", - is_admin=True, - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - chunk_events = [e for e in events if e["type"] == "chunk"] - assert len(chunk_events) >= 1 - assert "Engineering" in chunk_events[0]["content"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "reason", - [ - "end_date must be on or after start_date", - "start_date and end_date must be valid YYYY-MM-DD dates", - ], - ids=["reversed", "invalid-format"], - ) - @pytest.mark.parametrize( - "tool_name", ["get_usage_data", "get_team_usage_data", "get_tag_usage_data"] - ) - async def test_invalid_tool_dates_are_reported_to_the_model( - self, tool_name: str, reason: str - ) -> None: - fetch: Final = AsyncMock( - side_effect=HTTPException(status_code=400, detail={"error": reason}) - ) - - tool_message, statuses = await _run_one_tool( - tool_name, "2025-01-31", "2025-01-01", fetch - ) - - fetch.assert_awaited_once() - assert tool_message == f"Invalid date range: {reason}" - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - async def test_tool_dates_the_query_layer_accepts_are_queried_as_sent( - self, - ) -> None: - fetch: Final = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - tool_message, statuses = await _run_one_tool( - "get_usage_data", "2025/01/01", "2025-01-31T00:00:00", fetch - ) - - fetch.assert_awaited_once_with( - start_date="2025/01/01", end_date="2025-01-31T00:00:00" - ) - assert tool_message.startswith("Total Spend: $") - assert statuses == ["running", "complete"] - - @pytest.mark.asyncio - @pytest.mark.parametrize( - "error", - [ - HTTPException(status_code=500, detail={"error": "db down"}), - HTTPException(status_code=400, detail="page and page_size must be >= 1"), - ], - ids=["server-error", "other-400"], - ) - async def test_other_tool_errors_stay_a_generic_fetch_error( - self, error: HTTPException - ) -> None: - tool_message, statuses = await _run_one_tool( - "get_team_usage_data", - "2025-01-01", - "2025-01-31", - AsyncMock(side_effect=error), - ) - - assert tool_message == "Error fetching team usage data. Please try again." - assert statuses == ["running", "error"] - - @pytest.mark.asyncio - async def test_stream_handles_error(self): - with patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm: - mock_litellm.acompletion = AsyncMock(side_effect=Exception("LLM error")) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "test"}], - ): - events.append(json.loads(event.replace("data: ", "").strip())) - - error_events = [e for e in events if e["type"] == "error"] - assert len(error_events) == 1 - assert "internal error" in error_events[0]["message"].lower() - - @pytest.mark.asyncio - async def test_non_admin_enforces_user_id(self): - mock_tool_call = MagicMock() - mock_tool_call.id = "call_456" - mock_tool_call.function.name = "get_usage_data" - mock_tool_call.function.arguments = json.dumps( - { - "start_date": "2025-01-01", - "end_date": "2025-01-31", - "user_id": "other-user", - } - ) - - mock_first_response = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_456", - "type": "function", - "function": { - "name": "get_usage_data", - "arguments": '{"start_date":"2025-01-01","end_date":"2025-01-31","user_id":"other-user"}', - }, - } - ], - } - - async def mock_stream(): - chunk = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Data." - yield chunk - - mock_fetch = AsyncMock(return_value=SAMPLE_AGGREGATED_RESPONSE) - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.TOOL_HANDLERS", - { - "get_usage_data": { - "fetch": mock_fetch, - "summarise": _summarise_usage_data, - "label": "global usage data", - } - }, - ), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[ - mock_first_response, - mock_stream(), - ] - ) - - events = [] - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "Show data"}], - model="gpt-4o-mini", - user_id="my-user-id", - is_admin=False, - ): - events.append(event) - - mock_fetch.assert_called_once_with( - start_date="2025-01-01", - end_date="2025-01-31", - user_id="my-user-id", - ) - - -class TestUsageAiChatServiceAccountGuard: - """ - Security regression: a non-admin caller with user_id=None (service-account - key) must be rejected at the endpoint boundary, before any tool dispatch. - """ - - @pytest.mark.asyncio - async def test_non_admin_with_user_id_none_is_rejected(self): - from fastapi import HTTPException - - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - service_account_key = UserAPIKeyAuth( - user_id=None, - user_role=LitellmUserRoles.INTERNAL_USER, - ) - request = MagicMock() - body = UsageAIChatRequest( - messages=[ChatMessage(role="user", content="hi")], - model="gpt-4o-mini", - ) - - with pytest.raises(HTTPException) as exc_info: - await usage_ai_chat( - data=body, - request=request, - user_api_key_dict=service_account_key, - ) - - assert exc_info.value.status_code == 403 - assert "Service-account keys" in str(exc_info.value.detail) - - def test_resolve_fetch_kwargs_tripwire_fires_on_none_user_id(self): - """ - Defense-in-depth: if a future endpoint forgets the entry guard and - a non-admin caller with user_id=None reaches _resolve_fetch_kwargs, - the tripwire must fire rather than issuing an unscoped query. - """ - from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import ( - _resolve_fetch_kwargs, - ) - - with pytest.raises(ValueError, match='Non-admin caller has user_id=None; refusing to issue an') as exc_info: - _resolve_fetch_kwargs( - fn_name="get_usage_data", - fn_args={"start_date": "2025-01-01", "end_date": "2025-01-31"}, - user_id=None, - is_admin=False, - ) - assert "Endpoint-level guard missing" in str(exc_info.value) - - -class TestUsageAiChatKeepalive: - async def _collect_endpoint_body(self, monkeypatch, interval, delay=0.3) -> tuple[list[bytes], dict]: - import asyncio - - import litellm - from fastapi.responses import StreamingResponse - from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth - from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( - ChatMessage, - UsageAIChatRequest, - usage_ai_chat, - ) - - monkeypatch.setattr(litellm, "sse_keepalive_ping_interval_seconds", interval) - - async def slow_acompletion(**kwargs): - await asyncio.sleep(delay) - response = MagicMock() - response.choices = [MagicMock()] - response.choices[0].message.tool_calls = None - response.choices[0].message.content = "Total spend is $50.25" - return response - - with patch( # test-quality-ok: the stream calls the module-level litellm.acompletion directly; no injection seam - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm.acompletion", - new=AsyncMock(side_effect=slow_acompletion), - ): - response = await usage_ai_chat( - data=UsageAIChatRequest(messages=[ChatMessage(role="user", content="hi")], model="gpt-4o-mini"), - request=MagicMock(), - user_api_key_dict=UserAPIKeyAuth(user_id="admin", user_role=LitellmUserRoles.PROXY_ADMIN), - ) - assert isinstance(response, StreamingResponse) - chunks = [chunk if isinstance(chunk, bytes) else chunk.encode() async for chunk in response.body_iterator] - return chunks, dict(response.headers) - - @pytest.mark.asyncio - async def test_endpoint_pings_while_the_planning_completion_is_still_running(self, monkeypatch): - chunks, headers = await self._collect_endpoint_body(monkeypatch, interval=0.05) - - assert headers["content-type"].startswith("text/event-stream") - assert headers["cache-control"] == "no-cache" - assert headers["x-accel-buffering"] == "no" - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[1] == b": ping\n\n" - assert chunks.count(b": ping\n\n") >= 3 - assert b'"content": "Total spend is $50.25"' in b"".join(chunks) - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - @pytest.mark.asyncio - async def test_endpoint_stream_is_untouched_while_keepalives_are_unconfigured(self, monkeypatch): - chunks, _ = await self._collect_endpoint_body(monkeypatch, interval=None, delay=0.15) - - assert b": ping\n\n" not in chunks - assert chunks[0].startswith(b'data: {"type": "status"') - assert chunks[-1] == b'data: {"type": "done"}\n\n' - - -async def _run_one_tool( - tool_name: str, start_date: str, end_date: str, fetch: AsyncMock -) -> tuple[str, list[object]]: - """Runs one admin chat turn whose model calls `tool_name` with these dates; returns the - tool message the follow-up model call carried and the tool statuses streamed.""" - arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date}) - mock_tool_call: Final = MagicMock() - mock_tool_call.id = "call_dates" - mock_tool_call.function.name = tool_name - mock_tool_call.function.arguments = arguments - - mock_first_response: Final = MagicMock() - mock_first_response.choices = [MagicMock()] - mock_first_response.choices[0].message.tool_calls = [mock_tool_call] - mock_first_response.choices[0].message.model_dump.return_value = { - "role": "assistant", - "content": None, - "tool_calls": [ - { - "id": "call_dates", - "type": "function", - "function": {"name": tool_name, "arguments": arguments}, - } - ], - } - - async def mock_stream() -> AsyncIterator[MagicMock]: - chunk: Final = MagicMock() - chunk.choices = [MagicMock()] - chunk.choices[0].delta.content = "Done." - yield chunk - - with ( - patch( - "litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat.litellm" - ) as mock_litellm, - patch.dict(TOOL_HANDLERS[tool_name], {"fetch": fetch}), - ): - mock_litellm.acompletion = AsyncMock( - side_effect=[mock_first_response, mock_stream()] - ) - events: Final[tuple[dict[str, object], ...]] = tuple( - [ - json.loads(event.replace("data: ", "").strip()) - async for event in stream_usage_ai_chat( - messages=[{"role": "user", "content": "How much did we spend?"}], - model="gpt-4o-mini", - is_admin=True, - ) - ] - ) - final_messages: Final = mock_litellm.acompletion.call_args_list[1].kwargs[ - "messages" - ] - - tool_messages: Final = [m for m in final_messages if m["role"] == "tool"] - assert len(tool_messages) == 1, tool_messages - assert tool_messages[0]["tool_call_id"] == "call_dates" - return str(tool_messages[0]["content"]), [ - e["status"] for e in events if e["type"] == "tool_call" - ] diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index adc4de8c8fa..0749fc12ea9 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -93,6 +93,20 @@ def test_retired_roi_calculator_cannot_be_loaded_or_called( assert response.status_code == 404, f"{method} {path}: {response.text}" +@pytest.mark.parametrize("disable_lazy_routes", ("false", "true")) +@pytest.mark.parametrize("path", ("/lazy/warm/usage_ai", "/usage/ai/chat")) +def test_retired_usage_chat_cannot_be_loaded_or_called( + monkeypatch: pytest.MonkeyPatch, disable_lazy_routes: str, path: str +) -> None: + monkeypatch.setenv(FLAG, disable_lazy_routes) + app: Final = FastAPI() + attach_lazy_features(app) + + with TestClient(app) as client: + response: Final = client.post(path) + assert response.status_code == 404, f"{path}: {response.text}" + + @pytest.mark.parametrize("value", ("1", "true", "TRUE", "yes", "on")) def test_flag_registers_every_feature_at_startup(monkeypatch: pytest.MonkeyPatch, value: str) -> None: monkeypatch.setenv(FLAG, value) diff --git a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py index 47f81fb55d1..d5139263dc3 100644 --- a/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py +++ b/tests/unit/proxy/ui_crud_endpoints/test_latest_release_endpoints.py @@ -401,6 +401,25 @@ class TestLatestReleaseInfoEndpoint: assert response.status_code == 200 assert response.json() is None + @pytest.mark.parametrize( + ("flag", "expected_body", "expected_urls"), + [ + ("true", None, []), + (" TRUE ", None, []), + ("false", EXPECTED_INFO, [CHANGELOG_FEED_URL, _tag_url("v1.102.0")]), + ], + ) + def test_disable_flag_skips_every_outbound_request( + self, http_client, monkeypatch, flag, expected_body, expected_urls + ): + monkeypatch.setenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", flag) + client = _minor_release() + _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) + response = http_client.get("/get/latest_release_info") + assert response.status_code == 200 + assert response.json() == expected_body + assert client.urls == expected_urls + def test_repeated_requests_reuse_cache(self, http_client): client = _minor_release() _override_dependencies(client, _fresh_cache(), LitellmUserRoles.PROXY_ADMIN) diff --git a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py index 747e8da8043..e335b08f040 100644 --- a/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py +++ b/tests/unit/proxy/utils/prisma_and_spend/test_prisma_client_writes.py @@ -12,15 +12,118 @@ import asyncio import hashlib import json import logging +from collections.abc import Mapping +from contextlib import AbstractAsyncContextManager from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch import pytest from fastapi import HTTPException from litellm._service_logger import ServiceTypes +from litellm.models.user import LiteLLM_UserTable from litellm.proxy.db.log_db_metrics import record_db_io from litellm.proxy.utils import PrismaClient +from litellm.repositories.user_repository import free_sso_user_creation + + +class _SSOAdmissionUsers: + def __init__(self, count: int) -> None: + self.rows: Mapping[str, LiteLLM_UserTable] = { + f"user-{index}": LiteLLM_UserTable(user_id=f"user-{index}") + for index in range(count) + } + + async def find_unique( + self, where: Mapping[str, object] + ) -> LiteLLM_UserTable | None: + return self.rows.get(str(where["user_id"])) + + async def count(self, where: Mapping[str, object] | None = None) -> int: + if where is None: + return len(self.rows) + return sum((row.metadata or {}).get("scim_active") is False for row in self.rows.values()) + + async def upsert( + self, where: Mapping[str, object], data: Mapping[str, object] + ) -> LiteLLM_UserTable: + existing: Final = await self.find_unique(where) + if existing is not None: + return existing + created: Final = LiteLLM_UserTable.model_validate(data["create"]) + self.rows = {**self.rows, created.user_id: created} + return created + + +class _SSOAdmissionTransaction(AbstractAsyncContextManager["_SSOAdmissionTransaction"]): + def __init__(self, users: _SSOAdmissionUsers) -> None: + self.litellm_usertable: Final = users + + async def __aenter__(self) -> _SSOAdmissionTransaction: + return self + + async def __aexit__(self, *exc_info: object) -> None: + return None + + async def execute_raw(self, query: str) -> int: + return 0 + + async def query_raw(self, query: str) -> object: + return ({"locked": 1},) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("existing_count", "scope_id", "user_id", "enabled", "child_task", "refused"), + [ + (4, "new-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, False, True), + (6, "user-0", "user-0", True, False, False), + (5, "sso-user", "new-user", True, False, False), + (5, "new-user", "new-user", True, True, False), + (5, None, "new-user", True, False, True), + (5, None, "new-user", False, False, False), + ], +) +async def test_user_insert_enforces_free_sso_admission_only_for_its_owner( + prisma_client: PrismaClient, + monkeypatch: pytest.MonkeyPatch, + existing_count: int, + scope_id: str | None, + user_id: str, + enabled: bool, + child_task: bool, + refused: bool, +) -> None: + users: Final = _SSOAdmissionUsers(existing_count) + prisma_client.db.litellm_usertable = users + prisma_client.db.tx.return_value = _SSOAdmissionTransaction(users) + monkeypatch.setattr("asyncio.sleep", AsyncMock(return_value=None)) + original: Final = users.rows.get(user_id) + + async def attempt() -> None: + if refused: + with pytest.raises(HTTPException) as error: + await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert error.value.status_code == 403 + assert user_id not in users.rows + elif child_task: + result: Final = await asyncio.create_task( + prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + ) + assert result is users.rows[user_id] + else: + inserted: Final = await prisma_client.insert_data(data={"user_id": user_id}, table_name="user") + assert inserted is users.rows[user_id] + + with free_sso_user_creation(scope_id, enabled=enabled): + for _ in range(2): + await attempt() + + assert len(users.rows) == existing_count + int(not refused and original is None) + if original is not None: + assert users.rows[user_id] is original @pytest.mark.asyncio diff --git a/tests/unit/test_elevenlabs_tts_pricing.py b/tests/unit/test_elevenlabs_tts_pricing.py new file mode 100644 index 00000000000..16f726b9d6c --- /dev/null +++ b/tests/unit/test_elevenlabs_tts_pricing.py @@ -0,0 +1,49 @@ +"""ElevenLabs text-to-speech prices match the published API rates. + +https://elevenlabs.io/pricing/api lists "v3" and "v2 Multilingual" at $0.08 and +"Flash / Turbo" at $0.04, each "Price per 1K characters", and says API usage is +billed in US dollars, not credits. The map had v3 and Multilingual v2 at a +Scale-plan credit estimate ($0.18 per 1K characters) and no Flash rows. +""" + +import json +from pathlib import Path + +import pytest + +import litellm + +REPO_ROOT = Path(__file__).parents[2] +PRICES = { + "elevenlabs/eleven_v3": 8e-05, + "elevenlabs/eleven_multilingual_v2": 8e-05, + "elevenlabs/eleven_flash_v2_5": 4e-05, + "elevenlabs/eleven_flash_v2": 4e-05, +} + + +@pytest.fixture(autouse=True) +def local_model_cost_map(monkeypatch): + """Never depend on the published cost map having these rows yet.""" + monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True") + monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url="")) + litellm.get_model_info.cache_clear() + yield + litellm.get_model_info.cache_clear() + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_metadata_and_backup(model, per_character): + main = json.loads((REPO_ROOT / "model_prices_and_context_window.json").read_text()) + backup = json.loads((REPO_ROOT / "litellm/model_prices_and_context_window_backup.json").read_text()) + assert main[model] == backup[model] + info = litellm.get_model_info(model) + assert info["litellm_provider"] == "elevenlabs" + assert info["mode"] == "audio_speech" + assert info["input_cost_per_character"] == per_character + + +@pytest.mark.parametrize("model, per_character", PRICES.items()) +def test_speech_cost_is_charged_per_character(model, per_character): + cost = litellm.completion_cost(model=model, call_type="speech", prompt="x" * 1000) + assert cost == pytest.approx(per_character * 1000) diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 1451256631f..38d4c435e9a 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1098,14 +1098,6 @@ "count": 1 } }, - "src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx": { - "no-nested-ternary": { - "count": 1 - }, - "react-hooks/immutability": { - "count": 1 - } - }, "src/app/(dashboard)/usage/_components/components/UsagePageView.tsx": { "react-hooks/purity": { "count": 1 @@ -1767,13 +1759,13 @@ "count": 1 }, "max-params": { - "count": 15 + "count": 14 }, "no-nested-ternary": { "count": 5 }, "no-restricted-syntax": { - "count": 140 + "count": 139 }, "prefer-const": { "count": 28 diff --git a/ui/litellm-dashboard/package-lock.json b/ui/litellm-dashboard/package-lock.json index add749c88cb..8704dc55c38 100644 --- a/ui/litellm-dashboard/package-lock.json +++ b/ui/litellm-dashboard/package-lock.json @@ -54,7 +54,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -12876,9 +12876,9 @@ } }, "node_modules/zod": { - "version": "4.6.5", - "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.5.tgz", - "integrity": "sha512-v5l/aFXZQeai4awLbOpSoHecE9UiMrnfx75tEXLjNonXVARxQ5mOeipTjROUchszUNCqnE+hqAMujRsRHsut2Q==", + "version": "4.6.6", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.6.6.tgz", + "integrity": "sha512-4UmzQUF5fxvfcYHlbjBmejU+w+hbDnWFx4aS8kBx7nPBHcO2SqPbpAG8snbdg+Xse4pRefR98hgjFgIMCI/UhA==", "license": "MIT", "funding": { "url": "https://github.com/sponsors/colinhacks" diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 8160836515f..b1b8fde44ee 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -71,7 +71,7 @@ "tailwind-merge": "3.4.0", "usehooks-ts": "3.1.1", "uuid": "14.0.0", - "zod": "4.6.5" + "zod": "4.6.6" }, "devDependencies": { "@eslint/js": "9.39.2", @@ -107,6 +107,10 @@ "vitest": "4.1.12" }, "overrides": { + "@litellm/lens-ui": { + "next": "$next" + }, + "zod": "$zod", "prismjs": "1.30.0", "js-yaml": "4.3.2", "brace-expansion": "5.0.12", @@ -117,10 +121,7 @@ "axios": "1.13.6", "postcss": "8.5.23", "esbuild": "0.28.1", - "sharp": "^0.35.5", - "@litellm/lens-ui": { - "next": "$next" - } + "sharp": "^0.35.5" }, "engines": { "node": ">=24.14.1", diff --git a/ui/litellm-dashboard/public/assets/aria-agent.png b/ui/litellm-dashboard/public/assets/aria-agent.png new file mode 100644 index 00000000000..870773933e3 Binary files /dev/null and b/ui/litellm-dashboard/public/assets/aria-agent.png differ diff --git a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx index b957393be64..edde511d1cd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/home/_components/HomePage.integration.test.tsx @@ -141,7 +141,7 @@ describe("HomePage", () => { expect(screen.queryByRole("link", { name: /create new virtual key/i })).not.toBeInTheDocument(); }); - it("offers LiteAdmin on Home inside the dashboard frame", async () => { + it("should offer Aria Agent on Home inside the dashboard frame", async () => { render( @@ -149,7 +149,7 @@ describe("HomePage", () => { , ); - const entry = screen.getByRole("region", { name: "Ask LiteAdmin" }); + const entry = screen.getByRole("region", { name: "Ask Aria Agent" }); expect(await within(entry).findByPlaceholderText("What would you like to do?")).toBeInTheDocument(); expect(within(entry).getByRole("button", { name: "Send message" })).toBeDisabled(); expect(await screen.findByText("$501.74")).toBeInTheDocument(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx index 0fff919de4d..39627329b85 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/ClientUsagePanel.integration.test.tsx @@ -147,6 +147,10 @@ describe("ClientUsagePanel", () => { ); vi.mocked(userListCall).mockResolvedValue(userPage); vi.mocked(uiSpendLogsCall).mockResolvedValue(requestPage); + vi.stubGlobal( + "fetch", + vi.fn(async () => Response.json({ results: [] })), + ); }); afterEach(() => vi.unstubAllGlobals()); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx deleted file mode 100644 index d971f5d0506..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.test.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { screen } from "@testing-library/react"; -import { beforeAll, describe, expect, it, vi } from "vitest"; -import { renderWithProviders } from "@/../tests/test-utils"; -import UsageAIChatPanel from "./UsageAIChatPanel"; - -beforeAll(() => { - if (typeof window !== "undefined" && !window.ResizeObserver) { - window.ResizeObserver = class ResizeObserver { - observe() {} - unobserve() {} - disconnect() {} - } as any; - } -}); - -vi.mock("@/components/networking", () => ({ - modelHubCall: vi.fn().mockResolvedValue({ - data: [{ model_group: "gpt-4" }, { model_group: "claude-3-opus" }], - }), - usageAiChatStream: vi.fn(), -})); - -const defaultProps = { - open: true, - onClose: vi.fn(), - accessToken: "test-token", -}; - -describe("UsageAIChatPanel", () => { - it("should render the panel when open", () => { - renderWithProviders(); - - expect(screen.getByText("Ask AI")).toBeInTheDocument(); - expect(screen.getByText("Ask about your spend, models, keys, and trends")).toBeInTheDocument(); - }); - - it("should render model selector", () => { - renderWithProviders(); - - // One library paints the prompt as its own text node and the other leaves it on the input's - // placeholder attribute, so either one means the user is being told what to pick. - const prompt = "Select a model (optional, defaults to gpt-4o-mini)"; - expect(screen.queryAllByText(prompt).length + screen.queryAllByPlaceholderText(prompt).length).toBeGreaterThan(0); - }); - - it("should render empty state message when no conversation", () => { - renderWithProviders(); - - expect(screen.getByText("Ask a question about your usage")).toBeInTheDocument(); - }); - - it("should render the send button", () => { - renderWithProviders(); - - expect(screen.getByText("Send")).toBeInTheDocument(); - }); - - it("should render input placeholder", () => { - renderWithProviders(); - - expect(screen.getByPlaceholderText("Ask about your usage...")).toBeInTheDocument(); - }); - - it("should render clear chat button", () => { - renderWithProviders(); - - expect(screen.getByText("Clear chat")).toBeInTheDocument(); - }); - - it("should have the panel element even when closed (just off-screen)", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).toBeInTheDocument(); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-full"); - }); - - it("should not have translate-x-full class when open", () => { - renderWithProviders(); - - expect(screen.getByTestId("usage-ai-chat-panel")).not.toHaveClass("translate-x-full"); - expect(screen.getByTestId("usage-ai-chat-panel")).toHaveClass("translate-x-0"); - }); -}); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx deleted file mode 100644 index dc4291d25e1..00000000000 --- a/ui/litellm-dashboard/src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx +++ /dev/null @@ -1,385 +0,0 @@ -import React, { useEffect, useRef, useState } from "react"; -import ReactMarkdown from "react-markdown"; -import { MessageSquare, Sparkles, X } from "lucide-react"; -import { Button } from "@/components/ui/button"; -import { - Combobox, - ComboboxContent, - ComboboxEmpty, - ComboboxInput, - ComboboxItem, - ComboboxList, -} from "@/components/ui/combobox"; -import { Textarea } from "@/components/ui/textarea"; -import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; -import { modelHubCall, usageAiChatStream, UsageAiToolCallEvent } from "@/components/networking"; - -interface ToolCallStep { - tool_name: string; - tool_label: string; - arguments: Record; - status: "running" | "complete" | "error"; - error?: string; -} - -interface ChatMessage { - role: "user" | "assistant"; - content: string; - toolCalls?: ToolCallStep[]; -} - -interface UsageAIChatPanelProps { - open: boolean; - onClose: () => void; - accessToken: string | null; -} - -const TOOL_ICONS: Record = { - get_usage_data: "📊", - get_team_usage_data: "👥", - get_tag_usage_data: "🏷️", -}; - -const ToolCallDisplay: React.FC<{ step: ToolCallStep }> = ({ step }) => { - const icon = TOOL_ICONS[step.tool_name] || "🔧"; - const args = step.arguments; - const dateRange = args.start_date && args.end_date ? `${args.start_date} → ${args.end_date}` : ""; - const filter = args.team_ids || args.tags || args.user_id || ""; - - return ( -
- - {step.status === "running" ? ( - - ) : step.status === "error" ? ( - ✗ - ) : ( - ✓ - )} - -
-
- {icon} {step.tool_label} -
- {dateRange &&
{dateRange}
} - {filter &&
Filter: {filter}
} - {step.status === "error" && step.error &&
{step.error}
} -
-
- ); -}; - -const MarkdownContent: React.FC<{ content: string }> = ({ content }) => ( -

{children}

, - strong: ({ children }) => {children}, - ul: ({ children }) =>
    {children}
, - ol: ({ children }) =>
    {children}
, - li: ({ children }) =>
  • {children}
  • , - h1: ({ children }) =>

    {children}

    , - h2: ({ children }) =>

    {children}

    , - h3: ({ children }) =>

    {children}

    , - code: ({ children, className }) => { - const isBlock = className?.includes("language-"); - return isBlock ? ( -
    -            {children}
    -          
    - ) : ( - {children} - ); - }, - table: ({ children }) => ( -
    - {children}
    -
    - ), - th: ({ children }) => ( - {children} - ), - td: ({ children }) => {children}, - }} - > - {content} -
    -); - -const UsageAIChatPanel: React.FC = ({ open, onClose, accessToken }) => { - const [messages, setMessages] = useState([]); - const [inputText, setInputText] = useState(""); - const [isLoading, setIsLoading] = useState(false); - const [selectedModel, setSelectedModel] = useState(undefined); - const [availableModels, setAvailableModels] = useState([]); - const [isLoadingModels, setIsLoadingModels] = useState(false); - const [streamingContent, setStreamingContent] = useState(""); - const [statusMessage, setStatusMessage] = useState(null); - const [activeToolCalls, setActiveToolCalls] = useState([]); - const messagesEndRef = useRef(null); - const abortControllerRef = useRef(null); - - useEffect(() => { - if (open && availableModels.length === 0) { - loadModels(); - } - }, [open]); - - useEffect(() => { - if (typeof messagesEndRef.current?.scrollIntoView === "function") { - messagesEndRef.current.scrollIntoView({ behavior: "smooth" }); - } - }, [messages, streamingContent, activeToolCalls, statusMessage]); - - const loadModels = async () => { - if (!accessToken) return; - setIsLoadingModels(true); - try { - const fetchedModels = await modelHubCall(accessToken); - if (fetchedModels?.data?.length > 0) { - const models = fetchedModels.data.map((item: any) => item.model_group as string).sort(); - setAvailableModels(models); - } - } catch (error) { - console.error("Failed to load models:", error); - } finally { - setIsLoadingModels(false); - } - }; - - const handleSend = async () => { - if (!accessToken || !inputText.trim() || isLoading) return; - - const userMessage: ChatMessage = { role: "user", content: inputText.trim() }; - const updatedMessages = [...messages, userMessage]; - setMessages(updatedMessages); - setInputText(""); - setIsLoading(true); - setStreamingContent(""); - setStatusMessage(null); - setActiveToolCalls([]); - - const abortController = new AbortController(); - abortControllerRef.current = abortController; - - let accumulated = ""; - const toolCalls: ToolCallStep[] = []; - - try { - await usageAiChatStream( - accessToken, - updatedMessages.slice(-20).map((m) => ({ role: m.role, content: m.content })), - selectedModel || "", - (content: string) => { - setStatusMessage(null); - accumulated += content; - setStreamingContent(accumulated); - }, - () => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [ - ...prev, - { role: "assistant", content: accumulated, toolCalls: toolCalls.length > 0 ? [...toolCalls] : undefined }, - ]); - setStreamingContent(""); - }, - (errorMsg: string) => { - setStatusMessage(null); - setActiveToolCalls([]); - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - }, - (status: string) => { - setStatusMessage(status); - }, - (event: UsageAiToolCallEvent) => { - const idx = toolCalls.findIndex((tc) => tc.tool_name === event.tool_name); - if (idx >= 0) { - toolCalls[idx] = { ...event }; - } else { - toolCalls.push({ ...event }); - } - setActiveToolCalls([...toolCalls]); - }, - abortController.signal, - ); - } catch (error: any) { - if (error?.name === "AbortError" || abortController.signal.aborted) { - return; - } - const errorMsg = error?.message || "Failed to get response. Please try again."; - setMessages((prev) => [...prev, { role: "assistant", content: `Error: ${errorMsg}` }]); - setStreamingContent(""); - } finally { - setIsLoading(false); - abortControllerRef.current = null; - } - }; - - const handleKeyDown = (e: React.KeyboardEvent) => { - if (e.key === "Enter" && !e.shiftKey) { - e.preventDefault(); - handleSend(); - } - }; - - const handleClose = () => { - if (abortControllerRef.current) { - abortControllerRef.current.abort(); - } - onClose(); - }; - - const handleClear = () => { - setMessages([]); - setStreamingContent(""); - setActiveToolCalls([]); - setStatusMessage(null); - }; - - return ( -
    - {/* Header */} -
    -
    -

    -

    -

    Ask about your spend, models, keys, and trends

    -
    - -
    - - {/* Model selector */} -
    - setSelectedModel(value ?? undefined)} - > - - - {isLoadingModels ? "Loading models…" : "No models found"} - - {(model: string) => ( - - {model} - - )} - - - -
    - - {/* Chat messages */} -
    - {messages.length === 0 && !streamingContent && !isLoading && ( -
    -
    - )} - - {messages.map((msg, idx) => ( -
    - {msg.role === "user" ? ( -
    -
    - {msg.content} -
    -
    - ) : ( -
    - {/* Tool calls for this message */} - {msg.toolCalls && msg.toolCalls.length > 0 && ( -
    - {msg.toolCalls.map((tc, tcIdx) => ( - - ))} -
    - )} - {/* Response */} -
    - -
    -
    - )} -
    - ))} - - {/* Active tool calls (in-progress) */} - {isLoading && activeToolCalls.length > 0 && ( -
    - {activeToolCalls.map((tc, idx) => ( - - ))} -
    - )} - - {/* Status / spinner */} - {isLoading && !streamingContent && ( -
    - - {statusMessage || "Thinking..."} -
    - )} - - {/* Streaming response */} - {streamingContent && ( -
    - -
    - )} - -
    -
    - - {/* Input area */} -
    -
    -