scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-11-10 08:28:25 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 108

chore: merge main and resolve logging conflicts

Browse source
This commit is contained in:
adc77 2026-10-24 20:30:48 +05:30
commit c873bb9aab
315 changed files with 30341 additions and 6325 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
.github/CODEOWNERS vendored
View file

@ -1,4 +1,4 @@
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri
/model_prices_and_context_window.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/litellm/model_prices_and_context_window_backup.json @mateo-berri @ryan-crabbe-berri @kerry-berri @joshua-berri
/helm/ @yassin-berriai
/terraform/ @yassin-berriai

91
.github/workflows/interactions-spec-canary.yml vendored Normal file
View file

@ -0,0 +1,91 @@
name: Interactions spec canary
on:
schedule:
- cron: "17 7 * * *"
pull_request:
paths:
- .github/workflows/interactions-spec-canary.yml
- tests/unit/interactions/**
- litellm/types/interactions/**
workflow_dispatch:
permissions: {}
concurrency: interactions-spec-canary
jobs:
interactions-spec-canary:
name: interactions-spec-canary
if: github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 15
permissions:
contents: read
outputs:
compliance: ${{ steps.compliance.outcome }}
steps:
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.12"
- name: Set up uv
uses: ./.github/actions/setup-uv-with-retries
with:
version: "0.10.9"
- name: Install dependencies
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
- name: Run the compliance tests against Google's live spec
id: compliance
run: |
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
tracking-issue:
name: tracking-issue
needs: interactions-spec-canary
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
issues: write
steps:
- name: File the tracking issue unless the compliance step passed, close it when it did
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
env:
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
with:
script: |
const title = "Interactions spec canary is red";
const outcome = process.env.COMPLIANCE_OUTCOME;
const runUrl = process.env.RUN_URL;
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
const open = found.items.find((issue) => issue.title === title);
if (outcome === "success") {
if (open === undefined) return;
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
return;
}
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
if (open !== undefined) {
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
return;
}
await github.rest.issues.create({
...context.repo,
title,
assignees: ["mateo-berri"],
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
});

1
.github/workflows/test-unit.yml vendored
View file

@ -425,6 +425,7 @@ jobs:
tests/unit/realtime_api
tests/unit/rerank_api
tests/unit/search
tests/unit/text_completions
tests/unit/vector_stores
tests/unit/videos
python-version: "3.12"

5
Dockerfile
View file

@ -61,6 +61,7 @@ RUN apk add --no-cache \
python-3.13-dev \
rust \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \
@ -120,8 +121,8 @@ RUN sed -i 's/\r$//' docker/entrypoint.sh && chmod +x docker/entrypoint.sh && \
FROM $LITELLM_BUILD_IMAGE AS liteadmin-builder
COPY --from=uvbin /uv /usr/local/bin/uv
RUN apk add --no-cache python-3.13
ADD --checksum=sha256:2f7ae5cdd9d91731c0990e74a58239dc3e3fd2bf28dab23b55eafcdc47aaf87e \
https://github.com/BerriAI/litellm-admin-agent/archive/ef501e94bc9fbacb9233b922abf71427f030408c.tar.gz /tmp/liteadmin.tar.gz
ADD --checksum=sha256:4fb1847e28e2be30ba76dac94f02c4903291326516a0f4ba772f6ffb8b25d05c \
https://github.com/BerriAI/litellm-admin-agent/archive/5661c805ce479a0cf11b842ea133cf8d22cd7e34.tar.gz /tmp/liteadmin.tar.gz
RUN mkdir /tmp/liteadmin && tar xzf /tmp/liteadmin.tar.gz --strip-components=1 -C /tmp/liteadmin && \
uv venv /opt/liteadmin --python python3.13 && \
uv pip install --python /opt/liteadmin/bin/python --require-hashes -r /tmp/liteadmin/requirements.txt && \

6
README.md
View file

@ -616,15 +616,15 @@ The Terraform modules live at [`terraform/litellm/aws/`](./terraform/litellm/aws
### Verify Docker Image Signatures
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Every release is signed with the same key introduced in [commit `0112e53`](https://github.com/BerriAI/litellm/commit/0112e53046018d726492c814b3644b7d376029d0).
All LiteLLM Docker images published to GHCR are signed with [cosign](https://docs.sigstore.dev/cosign/overview/). Releases are signed with the key introduced in [commit `e82a3c6`](https://github.com/BerriAI/litellm/commit/e82a3c646908ba0a6ee0beddc4310d7a6cac4065).
**Verify using the pinned commit hash (recommended):**
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the original signing key:
A commit hash is cryptographically immutable, so this is the strongest way to ensure you are using the published signing key:
```bash
cosign verify \
--key https://raw.githubusercontent.com/BerriAI/litellm/0112e53046018d726492c814b3644b7d376029d0/cosign.pub \
--key https://raw.githubusercontent.com/BerriAI/litellm/e82a3c646908ba0a6ee0beddc4310d7a6cac4065/cosign.pub \
ghcr.io/berriai/litellm:<release-tag>
```

2
backend/Dockerfile
View file

@ -43,7 +43,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

4
cosign.pub
View file

@ -1,4 +1,4 @@
-----BEGIN PUBLIC KEY-----
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEKi4ivqGpE231OGH50PKbqy1Y1Kkb
POJC8+i2Wko82gBOUCe3M0Vw86H/4rhUhfoYEti4gdJ9wZbYmK0I2EE96g==
MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEG06dpKM+vEDnmp2zcjyXgVCXxzRg
JA5H2LnmwBKQafZGka6OOrWujCEWATQewiAgyD4WQdJ0pvU9RC4mvNAjvQ==
-----END PUBLIC KEY-----

1
docker/Dockerfile.database
View file

@ -59,6 +59,7 @@ RUN apk add --no-cache \
python-3.13 \
python-3.13-dev \
openssl \
libssl3 \
openssl-dev \
nodejs \
npm \

1
docker/Dockerfile.non_root
View file

@ -60,6 +60,7 @@ RUN for i in 1 2 3; do \
coreutils \
curl \
openssl \
libssl3 \
libsndfile \
nodejs \
npm && break || sleep 5; \

2
docker/build_from_pip/Dockerfile.build_from_pip
View file

@ -30,7 +30,7 @@ RUN uv venv --python python && \
"google-cloud-aiplatform==1.133.0" \
"google-genai==1.37.0" \
"anthropic[vertex]==0.84.0" \
"grpcio==1.78.0" \
"grpcio==1.84.1" \
"prometheus-client==0.20.0" \
"langfuse==2.59.7" \
"opentelemetry-api==1.28.0" \

2
gateway/Dockerfile
View file

@ -59,7 +59,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

6
litellm-rust/crates/llms-types/src/providers/anthropic/beta.rs
View file

@ -235,7 +235,10 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse],
Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[],
Self::ComputerUse20250124
| Self::ComputerUse20251124
| Self::Context1m20250807
| Self::ThinkingBindingControls20260801 => &[],
Self::DangerousToolUse20260903 => &[BetaProvider::BedrockConverse],
Self::Effort20251124 => &[BetaProvider::VertexAi],
Self::FastMode20260201 | Self::Oauth20250420 => &[
@ -272,7 +275,6 @@ impl AnthropicBeta {
BetaProvider::Databricks,
],
Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi],
Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi],
Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks],
Self::ToolSearchTool20251019 => &[
BetaProvider::Anthropic,

2
litellm/anthropic_beta_headers_config.json
View file

@ -67,7 +67,7 @@
"structured-outputs-2025-11-13": "structured-outputs-2025-11-13",
"text_editor_20241022": null,
"text_editor_20250124": null,
"thinking-binding-controls-2026-08-01": null,
"thinking-binding-controls-2026-08-01": "thinking-binding-controls-2026-08-01",
"thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18",
"token-efficient-tools-2025-02-19": null,
"web-fetch-2025-09-10": "web-fetch-2025-09-10",

3
litellm/anthropic_beta_headers_manager.py
View file

@ -36,6 +36,7 @@ import httpx
from pydantic import TypeAdapter
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
# Cache for the loaded configuration
_BETA_HEADERS_CONFIG: dict | None = None
@ -145,7 +146,7 @@ def get_beta_headers_config(url: str) -> dict:
Dict containing the beta headers configuration
"""
# Check if local-only mode is enabled
if os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS", "").lower() == "true":
# verbose_logger.debug("Using local Anthropic beta headers config (LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS=True)")
return GetAnthropicBetaHeadersConfig.load_local_beta_headers_config()

4
litellm/cost_calculator.py
View file

@ -2264,6 +2264,8 @@ def response_cost_calculator(
### VERTEX LOCATION ###
vertex_location: str | None = None, # for Vertex AI regional-endpoint uplift (e.g. "us-east5", "global")
region_name: str | None = None,
custom_cost_per_token: CostPerToken | None = None,
custom_cost_per_second: float | None = None,
) -> float:
"""
Returns
@ -2292,6 +2294,8 @@ def response_cost_calculator(
optional_params=optional_params,
custom_pricing=custom_pricing,
base_model=base_model,
custom_cost_per_token=custom_cost_per_token,
custom_cost_per_second=custom_cost_per_second,
prompt=prompt,
standard_built_in_tools_params=standard_built_in_tools_params,
litellm_model_name=litellm_model_name,

7
litellm/integrations/otel/mount.py
View file

@ -107,8 +107,8 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
Safe no-op when the V2 gate is off or ``opentelemetry-instrumentation-fastapi``
is unavailable. This MUST be called at app-creation time — once the lifespan
runs, the middleware stack is frozen and ``instrument_app`` raises "Cannot add
middleware after an application has started".
runs, the middleware stack is frozen and middleware added later never runs, so
a late call is skipped without registering anything.
No ``TracerProvider`` is passed, so the instrumentation binds to the OTel global
``ProxyTracerProvider``; the proxy publishes the real provider as the global
@ -126,6 +126,9 @@ def instrument_fastapi_app(app: "FastAPI", tracer_provider: "TracerProvider | No
try:
if not is_otel_v2_enabled():
return
if app.middleware_stack is not None:
verbose_logger.debug("Skipping OTel V2 FastAPI instrumentation: the app has already started")
return
# Lazy: only the V2-enabled path needs the optional
# ``opentelemetry-instrumentation-fastapi`` package, which is not part of the

44
litellm/integrations/otel/plumbing/otlp_json.py
View file

@ -6,6 +6,7 @@ retry loop and swaps the payload for OTLP/JSON (enums as integers, ids as hex).
import base64
import json
import logging
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import Final, TypeAlias
@ -13,13 +14,13 @@ from typing import Final, TypeAlias
import requests
from google.protobuf.json_format import MessageToDict
from opentelemetry.exporter.otlp.proto.common.trace_encoder import encode_spans
from opentelemetry.exporter.otlp.proto.http._common import RequestPayloadTooLargeError
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import ReadableSpan
from opentelemetry.sdk.trace.export import SpanExportResult
from opentelemetry.semconv.attributes.http_attributes import HTTP_RESPONSE_STATUS_CODE
from litellm._logging import verbose_logger
_logger: Final = logging.getLogger(__name__)
JSON_CONTENT_TYPE: Final = "application/json"
_HEX_ID_KEYS: Final = frozenset({"traceId", "spanId", "parentSpanId"})
@ -73,11 +74,12 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
headers: Mapping[str, str],
certificate_file: str | None = None,
session: "requests.Session | None" = None,
*,
max_request_size: int | None = None,
) -> None:
super().__init__(
endpoint=endpoint,
headers={**headers, "Content-Type": JSON_CONTENT_TYPE},
headers={**{key.lower(): value for key, value in headers.items()}, "content-type": JSON_CONTENT_TYPE},
certificate_file=certificate_file,
session=session,
max_request_size=max_request_size,
@ -85,23 +87,33 @@ class OTLPJsonSpanExporter(OTLPSpanExporter):
def export(self, spans: Sequence[ReadableSpan]) -> SpanExportResult:
if self._shutdown:
_logger.warning("Exporter already shutdown, ignoring batch")
return SpanExportResult.FAILURE
with self._metrics.export_operation(len(spans)) as operation:
with self._metrics.export_operation(len(spans)) as result:
try:
serialized: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # malformed telemetry must remain a non-blocking export failure
operation.error = error
verbose_logger.error("Could not encode OTLP/JSON span batch")
serialized_data: Final = encode_spans_json(spans)
except Exception as error: # noqa: BLE001 # SDK export contract reports encoding failures as results
_logger.error("Failed to encode span batch: %s", error)
result.error = error
return SpanExportResult.FAILURE
if 0 < self._max_request_size < len(serialized):
operation.error = ValueError("OTLP/JSON span batch exceeds the exporter request limit")
verbose_logger.error("OTLP/JSON span batch exceeds the exporter request limit")
if 0 < self._max_request_size < len(serialized_data):
result.error = RequestPayloadTooLargeError(
f"Serialized span request size {len(serialized_data)} bytes "
f"exceeds max_request_size {self._max_request_size} bytes."
)
_logger.warning("Dropping span batch: %s", result.error)
return SpanExportResult.FAILURE
exported: Final = self._client.export(serialized)
if not exported.success:
operation.error = exported.error
operation.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: exported.status_code} if exported.status_code is not None else None
export_result: Final = self._client.export(serialized_data)
if not export_result.success:
result.error = export_result.error
result.error_attrs = (
{HTTP_RESPONSE_STATUS_CODE: export_result.status_code}
if export_result.status_code is not None
else None
)
return SpanExportResult.FAILURE
return SpanExportResult.SUCCESS

1
litellm/litellm_core_utils/exception_mapping_utils.py
View file

@ -102,6 +102,7 @@ class ExceptionCheckers:
"`inputs` tokens + `max_new_tokens` must be",
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
'"code": "context_length_exceeded"',
]
for substring in known_exception_substrings:
if substring in _error_str_lowercase:

3
litellm/litellm_core_utils/get_blog_posts.py
View file

@ -19,6 +19,7 @@ import httpx
from pydantic import ConfigDict
from litellm import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.types.llms.base import LiteLLMBaseModel
BLOG_POSTS_TTL_SECONDS: Final[int] = 3600 # 1 hour
@ -129,7 +130,7 @@ class GetBlogPosts:
Uses the in-process cache if within BLOG_POSTS_TTL_SECONDS.
Fetches from ``url`` otherwise, falling back to local backup on failure.
"""
if os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_BLOG_POSTS", "").lower() == "true":
return cls.load_local_blog_posts()
now: Final = time.time()

6
litellm/litellm_core_utils/get_llm_provider_logic.py
View file

@ -972,3 +972,9 @@ def _get_openai_compatible_provider_info(
if dynamic_api_key is None and api_key is not None:
dynamic_api_key = api_key
return model, custom_llm_provider, dynamic_api_key, api_base
def get_provider_affinity_header(api_base: str | None) -> str | None:
from litellm.llms.opencode.transformation import OpenCodeProviderAdapter
return OpenCodeProviderAdapter().get_provider_affinity_header(api_base)

5
litellm/litellm_core_utils/get_model_cost_map.py
View file

@ -39,6 +39,7 @@ from litellm.constants import (
from litellm.litellm_core_utils.fallback_generalizations import (
set_fallback_generalizations,
)
from litellm.litellm_core_utils.offline_mode import is_offline_mode
FALLBACK_GENERALIZATIONS_KEY: Final = "fallback_generalizations"
_CATALOG_ADAPTER: Final = TypeAdapter(dict[str, dict[str, object]])
@ -456,7 +457,7 @@ async def refetch_model_cost_map(
on failure it returns ``ModelCostMapReloadUnavailable`` so callers keep the
map they already have.
"""
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true":
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
@ -740,7 +741,7 @@ def get_model_cost_map(
_cost_map_source_info.loaded_at = datetime.now(timezone.utc)
# Note: can't use get_secret_bool here — this runs during litellm.__init__
# before litellm._key_management_settings is set.
if os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
if is_offline_mode() or os.getenv("LITELLM_LOCAL_MODEL_COST_MAP", "").lower() == "true" or _is_cli_process():
_cost_map_source_info.source = "local"
_cost_map_source_info.url = None
_cost_map_source_info.is_env_forced = True

5
litellm/litellm_core_utils/litellm_logging.py
View file

@ -164,6 +164,7 @@ from litellm.types.utils import (
CachingDetails,
CallTypes,
CostBreakdown,
CostPerToken,
CustomPricingLiteLLMParams,
DynamicPromptManagementParamLiteral,
EmbeddingResponse,
@ -624,6 +625,8 @@ class Logging(LiteLLMLoggingBaseClass):
prometheusLogger, \
slack_app
custom_pricing: bool = False
custom_cost_per_token: CostPerToken | None = None
custom_cost_per_second: float | None = None
replaced_global_callback_keys: frozenset[str] = frozenset()
stream_options = None
litellm_request_debug: bool = False
@ -2099,6 +2102,8 @@ class Logging(LiteLLMLoggingBaseClass):
"call_type": self.call_type,
"optional_params": self.optional_params,
"custom_pricing": custom_pricing,
"custom_cost_per_token": self.custom_cost_per_token if custom_pricing else None,
"custom_cost_per_second": self.custom_cost_per_second if custom_pricing else None,
"prompt": prompt,
"standard_built_in_tools_params": self.standard_built_in_tools_params,
"router_model_id": router_model_id,

7
litellm/litellm_core_utils/offline_mode.py Normal file
View file

@ -0,0 +1,7 @@
import os
def is_offline_mode() -> bool:
"""True when ``LITELLM_OFFLINE_MODE=true``. A catalog, check or tokenizer LiteLLM fetches on its own, rather
than from a destination the operator configured, must check this and use its bundled copy or skip instead"""
return os.getenv("LITELLM_OFFLINE_MODE", "").strip().lower() == "true"

8
litellm/litellm_core_utils/provider_affinity.py
View file

@ -44,10 +44,10 @@ def _get_value(value: object, key: str) -> object | None:
return getattr(value, key, None)
def _get_provider_affinity_header_name(litellm_params: object | None) -> str | None:
def _get_provider_affinity_header_name(litellm_params: object | None, default_header: str | None = None) -> str | None:
header: Final = _get_value(litellm_params, "provider_affinity_header") if litellm_params is not None else None
if header is None:
return None
return default_header
if not isinstance(header, str):
raise TypeError("provider_affinity_header must be a string")
return validate_provider_affinity_header_name(header)
@ -84,9 +84,9 @@ def get_stable_session_id(litellm_params: object | None) -> str | None:
def add_provider_affinity_header(
headers: Mapping[str, object], litellm_params: object | None
headers: Mapping[str, object], litellm_params: object | None, default_header: str | None = None
) -> dict[str, object]: # mutable-ok: downstream handlers add auth and signing headers
header_name: Final = _get_provider_affinity_header_name(litellm_params)
header_name: Final = _get_provider_affinity_header_name(litellm_params, default_header)
if header_name is None or any(key.lower() == header_name.lower() for key in headers):
return dict(headers)

12
litellm/llms/anthropic/pass_through/messages/transformation.py
View file

@ -75,6 +75,12 @@ def messages_carry_output_config(messages: Sequence[object]) -> bool:
_messages_carry_output_config = messages_carry_output_config
def _ensure_stop_sequence(response: AnthropicMessagesResponse) -> AnthropicMessagesResponse:
if "stop_sequence" not in response:
response["stop_sequence"] = None # rebind-ok: normalize the request-owned response in place
return response
class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
_workload_identity_eligible: ClassVar[bool] = True
@ -715,14 +721,12 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
raw_response: httpx.Response,
logging_obj: LiteLLMLoggingObj,
) -> AnthropicMessagesResponse:
"""
No transformation is needed for Anthropic messages, since we want the response in the Anthropic /v1/messages API spec
"""
try:
raw_response_json: Final = raw_response.json()
except Exception:
raise AnthropicError(message=raw_response.text, status_code=raw_response.status_code)
return AnthropicMessagesResponse(**raw_response_json)
response: Final = AnthropicMessagesResponse(**raw_response_json)
return _ensure_stop_sequence(response)
def get_async_streaming_response_iterator(
self,

17
litellm/llms/aws_polly/text_to_speech/transformation.py
View file

@ -178,7 +178,7 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
else:
mapped_params["output_format"] = self.DEFAULT_OUTPUT_FORMAT
# Extract engine from model name (e.g., "aws_polly/neural" -> "neural")
# model arrives without the "aws_polly/" prefix, e.g. "standard"
engine: Final = self._extract_engine_from_model(model)
mapped_params["engine"] = engine
@ -196,18 +196,17 @@ class AWSPollyTextToSpeechConfig(BaseTextToSpeechConfig, BaseAWSLLM):
"""
Extract engine from model name.
speech() strips the "aws_polly/" prefix before the config is called,
so the bare engine name is what normally arrives here.
Examples:
- aws_polly/neural -> neural
- aws_polly/standard -> standard
- standard -> standard
- aws_polly/long-form -> long-form
- aws_polly -> neural (default)
"""
if "/" in model:
parts: Final = model.split("/")
if len(parts) >= 2:
engine: Final = parts[1].lower()
if engine in self.VALID_ENGINES:
return engine
engine: Final = model.rsplit("/", 1)[-1].lower()
if engine in self.VALID_ENGINES:
return engine
return self.DEFAULT_ENGINE
def validate_environment(

15
litellm/llms/azure/responses/transformation.py
View file

@ -3,6 +3,7 @@ from collections.abc import Mapping
from copy import deepcopy
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal
from urllib.parse import urlsplit
import httpx
from openai.types.responses import ResponseReasoningItem
@ -211,9 +212,21 @@ class AzureOpenAIResponsesAPIConfig(OpenAIResponsesAPIConfig):
"https://litellm8397336933.openai.azure.com/openai/responses?api-version=2024-05-01-preview"
"""
resolved_api_base: Final = BaseAzureLLM.resolve_azure_api_base(api_base)
responses_params: Final = _with_responses_api_version(resolved_api_base, litellm_params)
parsed_base: Final = urlsplit(resolved_api_base or "")
path: Final = parsed_base.path.rstrip("/")
if path.endswith(("/openai/v1", "/openai/v1/responses")):
responses_path: Final = f"{path}/responses" if path.endswith("/openai/v1") else path
return BaseAzureLLM.get_base_azure_url(
api_base=parsed_base._replace(path=responses_path).geturl(),
litellm_params=responses_params,
route="/openai/v1/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)
return BaseAzureLLM.get_base_azure_url(
api_base=resolved_api_base,
litellm_params=_with_responses_api_version(resolved_api_base, litellm_params),
litellm_params=responses_params,
route="/openai/responses",
default_api_version=AZURE_DEFAULT_RESPONSES_API_VERSION,
)

3
litellm/llms/azure_ai/chat/transformation.py
View file

@ -12,6 +12,7 @@ from litellm._logging import verbose_logger
from litellm.litellm_core_utils.prompt_templates.common_utils import (
audio_or_image_in_message_content,
convert_content_list_to_str,
drop_tool_reference_parts_from_tool_messages,
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
@ -250,7 +251,7 @@ class AzureAIStudioConfig(OpenAIConfig):
Operates on a deep copy so the caller's messages keep their thinking blocks
and provider metadata, which a fallback to another provider still needs.
"""
stripped_messages: Final = copy.deepcopy(messages)
stripped_messages: Final = copy.deepcopy(drop_tool_reference_parts_from_tool_messages(messages))
for message in stripped_messages:
message_dict = cast(dict, message) # cast-ok: TypedDict is a runtime dict stripped on our copy
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:

15
litellm/llms/bedrock/count_tokens/kimi_k3.py
View file

@ -27,6 +27,7 @@ from typing_extensions import Self
import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.prompt_templates.factory import (
chat_completion_tools_for,
is_flat_responses_tool,
@ -67,12 +68,14 @@ class KimiK3TokenizerSource:
def _hugging_face_ranks_path() -> str:
import huggingface_hub
return huggingface_hub.hf_hub_download(
repo_id=KIMI_K3_REPO,
filename=KIMI_K3_RANKS_FILE,
revision=KIMI_K3_REVISION,
local_files_only=litellm.disable_hf_tokenizer_download is True,
)
if litellm.disable_hf_tokenizer_download is True or is_offline_mode():
cached: Final = huggingface_hub.try_to_load_from_cache(
repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION
)
if not isinstance(cached, str):
raise FileNotFoundError(f"{KIMI_K3_RANKS_FILE} is not in the Hugging Face cache")
return cached
return huggingface_hub.hf_hub_download(repo_id=KIMI_K3_REPO, filename=KIMI_K3_RANKS_FILE, revision=KIMI_K3_REVISION)
HUGGING_FACE_SOURCE: Final = KimiK3TokenizerSource(ranks_path=_hugging_face_ranks_path, sha256=KIMI_K3_RANKS_SHA256)

27
litellm/llms/custom_httpx/llm_http_handler.py
View file

@ -52,7 +52,9 @@ from litellm.litellm_core_utils.audio_utils.subtitle_utils import (
from litellm.litellm_core_utils.core_helpers import set_provider_response_headers_in_hidden_params
from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value
from litellm.litellm_core_utils.get_litellm_params import AWS_CREDENTIAL_KWARGS_KEYS
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.llm_request_utils import serialize_multipart_form_fields
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.litellm_core_utils.realtime_errors import (
close_after_upstream_handshake_refusal,
realtime_error_event,
@ -60,6 +62,7 @@ from litellm.litellm_core_utils.realtime_errors import (
)
from litellm.litellm_core_utils.realtime_streaming import RealTimeStreaming
from litellm.litellm_core_utils.url_utils import encode_url_path_segment, join_api_base
from litellm.llms.anthropic.common_utils import AnthropicModelInfo
from litellm.llms.base_llm.anthropic_messages.transformation import (
BaseAnthropicMessagesConfig,
)
@ -2435,13 +2438,13 @@ class BaseLLMHTTPHandler:
# Also check for extra_headers in kwargs (from config or direct calls)
extra_headers_from_kwargs: Final = kwargs.get("extra_headers", None)
# Merge all header sources: forwarded < extra_headers < provider_specific
merged_headers: Final = {}
if forwarded_headers:
merged_headers.update(forwarded_headers)
if extra_headers_from_kwargs:
merged_headers.update(extra_headers_from_kwargs)
if provider_specific_headers:
merged_headers.update(provider_specific_headers)
merged_headers: Final[Mapping[str, object]] = MappingProxyType(
{
**(forwarded_headers or {}),
**(extra_headers_from_kwargs or {}),
**(provider_specific_headers or {}),
}
)
request_params: Final = reduce(
delete_nested_value,
litellm_params.get("additional_drop_params") or (),
@ -2452,11 +2455,19 @@ class BaseLLMHTTPHandler:
and anthropic_messages_provider_config.should_filter_anthropic_beta_headers()
):
await aload_beta_headers_config()
affinity_api_base: Final = (
AnthropicModelInfo.get_api_base(api_base) if custom_llm_provider == "anthropic" else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
(
headers,
api_base,
) = await anthropic_messages_provider_config.avalidate_anthropic_messages_environment(
headers=merged_headers or {},
headers=(
add_provider_affinity_header(merged_headers, litellm_params, default_affinity_header)
if default_affinity_header is not None
else dict(merged_headers)
),
model=model,
messages=messages,
optional_params=request_params,

10
litellm/llms/opencode/common_utils.py Normal file
View file

@ -0,0 +1,10 @@
import re
def is_opencode_api_base(api_base: str | None) -> bool:
return bool(
api_base
and re.fullmatch(
r"https://opencode\.ai/(?:zen(?:/go)?|inference/(?:go/)?(?:openai|anthropic))(?:/v1)?/?", api_base
)
)

6
litellm/llms/opencode/transformation.py Normal file
View file

@ -0,0 +1,6 @@
from litellm.llms.opencode.common_utils import is_opencode_api_base
class OpenCodeProviderAdapter:
def get_provider_affinity_header(self, api_base: str | None) -> str | None:
return "x-opencode-session" if is_opencode_api_base(api_base) else None

17
litellm/main.py
View file

@ -89,6 +89,7 @@ from litellm.litellm_core_utils.get_litellm_params import (
with_control_options,
)
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_provider_affinity_header,
invalid_provider_routing_hints_message,
)
from litellm.litellm_core_utils.get_provider_specific_headers import (
@ -130,6 +131,7 @@ from litellm.llms.bedrock_mantle.chat.claude_transformation import bedrock_mantl
from litellm.llms.cohere.common_utils import CohereModelInfo
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler, http2_enabled
from litellm.llms.openai.chat.gpt_5_transformation import OpenAIGPT5Config
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.json_loader import JSONProviderRegistry
from litellm.llms.vertex_ai.common_utils import (
VertexAIModelRoute,
@ -143,6 +145,7 @@ from litellm.router_utils.reasoning_effort_capability import (
resolve_chat_reasoning,
)
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper
from litellm.types.completion import (
CompletionDispatchContext,
_CompletionDispatchResult,
@ -169,7 +172,6 @@ from litellm.utils import (
ProviderConfigManager,
TextChoices,
TextCompletionResponse,
TextCompletionStreamWrapper,
TranscriptionResponse,
Usage,
add_provider_specific_params_to_optional_params,
@ -5990,19 +5992,28 @@ def completion(
},
)
litellm_params: Final = with_control_options(requested_litellm_params, control_options)
if litellm_params.get("provider_affinity_header") is not None:
affinity_api_base: Final = (
str(client.base_url)
if isinstance(client, openai.OpenAI | openai.AsyncOpenAI)
else get_openai_credentials(api_base=api_base).api_base
if custom_llm_provider == "openai"
else api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
if litellm_params.get("provider_affinity_header") is not None or default_affinity_header is not None:
try:
headers = add_provider_affinity_header(
headers=headers or litellm.headers or MappingProxyType({}),
litellm_params=MappingProxyType(
{
"provider_affinity_header": litellm_params["provider_affinity_header"],
"provider_affinity_header": litellm_params.get("provider_affinity_header"),
"litellm_session_id": kwargs.get("litellm_session_id"),
"session_id": kwargs.get("session_id"),
"metadata": metadata,
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
except ValueError as affinity_error:
raise litellm.BadRequestError(

1
litellm/messages/dispatch.py
View file

@ -103,6 +103,7 @@ def _target(request: NativeCall, model: str, provider: str | None) -> Target:
_FEATURES: Final[detectors.Detectors] = MappingProxyType(
{
**detectors.COMMON,
Feature.PROVIDER_AFFINITY: route_detectors.provider_affinity,
Feature.SAFEGUARDS_CLASSIFIER: route_detectors.safeguards_classifier,
Feature.RESPONSE_CACHE: detectors.response_cache,
Feature.MODIFY_PARAMS: detectors.modify_params,

276
litellm/model_prices_and_context_window_backup.json
View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -36861,17 +36957,17 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-08,
"cache_creation_input_token_cost_priority": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_decisions": 1e-07,
"cache_read_input_token_cost_decisions": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_read_input_token_cost_batches": 5e-09,
@ -42337,6 +42433,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -45623,6 +45847,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45648,6 +45873,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -48499,6 +48725,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -49352,6 +49589,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49371,6 +49609,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -52906,12 +53145,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53167,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -59917,6 +60160,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -59942,6 +60186,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -59967,6 +60212,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -59990,6 +60236,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -67841,6 +68088,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -67860,6 +68108,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68138,6 +68387,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68157,6 +68407,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68545,6 +68796,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68564,6 +68816,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68704,6 +68957,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68723,6 +68977,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -73330,6 +73585,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,

BIN
litellm/proxy/_experimental/out/assets/aria-agent.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 685 KiB

27
litellm/proxy/_experimental/out/assets/logos/claude.svg Normal file
View file

@ -0,0 +1,27 @@
<!--
Source: https://github.com/lobehub/lobe-icons/blob/master/packages/static-svg/icons/claude-color.svg
Retrieved: 2026-10-22
MIT License
Copyright (c) 2023 LobeHub
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
-->
<svg height="1em" style="flex:none;line-height:1" viewBox="0 0 24 24" width="1em" xmlns="http://www.w3.org/2000/svg"><title>Claude</title><path d="M4.709 15.955l4.72-2.647.08-.23-.08-.128H9.2l-.79-.048-2.698-.073-2.339-.097-2.266-.122-.571-.121L0 11.784l.055-.352.48-.321.686.06 1.52.103 2.278.158 1.652.097 2.449.255h.389l.055-.157-.134-.098-.103-.097-2.358-1.596-2.552-1.688-1.336-.972-.724-.491-.364-.462-.158-1.008.656-.722.881.06.225.061.893.686 1.908 1.476 2.491 1.833.365.304.145-.103.019-.073-.164-.274-1.355-2.446-1.446-2.49-.644-1.032-.17-.619a2.97 2.97 0 01-.104-.729L6.283.134 6.696 0l.996.134.42.364.62 1.414 1.002 2.229 1.555 3.03.456.898.243.832.091.255h.158V9.01l.128-1.706.237-2.095.23-2.695.08-.76.376-.91.747-.492.584.28.48.685-.067.444-.286 1.851-.559 2.903-.364 1.942h.212l.243-.242.985-1.306 1.652-2.064.73-.82.85-.904.547-.431h1.033l.76 1.129-.34 1.166-1.064 1.347-.881 1.142-1.264 1.7-.79 1.36.073.11.188-.02 2.856-.606 1.543-.28 1.841-.315.833.388.091.395-.328.807-1.969.486-2.309.462-3.439.813-.042.03.049.061 1.549.146.662.036h1.622l3.02.225.79.522.474.638-.079.485-1.215.62-1.64-.389-3.829-.91-1.312-.329h-.182v.11l1.093 1.068 2.006 1.81 2.509 2.33.127.578-.322.455-.34-.049-2.205-1.657-.851-.747-1.926-1.62h-.128v.17l.444.649 2.345 3.521.122 1.08-.17.353-.608.213-.668-.122-1.374-1.925-1.415-2.167-1.143-1.943-.14.08-.674 7.254-.316.37-.729.28-.607-.461-.322-.747.322-1.476.389-1.924.315-1.53.286-1.9.17-.632-.012-.042-.14.018-1.434 1.967-2.18 2.945-1.726 1.845-.414.164-.717-.37.067-.662.401-.589 2.388-3.036 1.44-1.882.93-1.086-.006-.158h-.055L4.132 18.56l-1.13.146-.487-.456.061-.746.231-.243 1.908-1.312-.006.006z" fill="#D97757" fill-rule="nonzero"></path></svg>

5
litellm/proxy/_lazy_features.py
View file

@ -299,11 +299,6 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = (
module_path="litellm.proxy.spend_tracking.vantage_endpoints",
path_prefixes=("/vantage",),
),
LazyFeature(
name="usage_ai",
module_path="litellm.proxy.management_endpoints.usage_endpoints",
path_prefixes=("/usage/ai",),
),
LazyFeature(
name="prompts",
module_path="litellm.proxy.prompts.prompt_endpoints",

163
litellm/proxy/_lazy_openapi_snapshot.json
View file

@ -11407,7 +11407,7 @@
},
"/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_systemone_post",
"requestBody": {
"content": {
@ -11813,7 +11813,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -12814,7 +12813,7 @@
},
"/v1/systemone": {
"post": {
"description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"description": "Judge a `state` against one or more named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`) and `box` (the image region the `instructions` describe, with `confidence`).\nTypeSafe models also take images in `assets`, each one referenced by its key in `state`. Requests that break\nthese rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)",
"operationId": "systemone_v1_systemone_post",
"requestBody": {
"content": {
@ -13220,7 +13219,6 @@
]
},
"description": "Named noul, choice and score questions. Each key becomes a key in the answers",
"maxProperties": 128,
"minProperties": 1,
"propertyNames": {
"minLength": 1
@ -57900,163 +57898,6 @@
}
}
},
"usage_ai": {
"components": {
"schemas": {
"ChatMessage": {
"properties": {
"content": {
"title": "Content",
"type": "string"
},
"role": {
"enum": [
"user",
"assistant"
],
"title": "Role",
"type": "string"
}
},
"required": [
"role",
"content"
],
"title": "ChatMessage",
"type": "object"
},
"HTTPValidationError": {
"properties": {
"detail": {
"items": {
"$ref": "#/components/schemas/ValidationError"
},
"title": "Detail",
"type": "array"
}
},
"title": "HTTPValidationError",
"type": "object"
},
"UsageAIChatRequest": {
"properties": {
"messages": {
"description": "Chat messages (user/assistant history)",
"items": {
"$ref": "#/components/schemas/ChatMessage"
},
"title": "Messages",
"type": "array"
},
"model": {
"anyOf": [
{
"type": "string"
},
{
"type": "null"
}
],
"description": "Model to use for AI chat",
"title": "Model"
}
},
"required": [
"messages"
],
"title": "UsageAIChatRequest",
"type": "object"
},
"ValidationError": {
"properties": {
"ctx": {
"title": "Context",
"type": "object"
},
"input": {
"title": "Input"
},
"loc": {
"items": {
"anyOf": [
{
"type": "string"
},
{
"type": "integer"
}
]
},
"title": "Location",
"type": "array"
},
"msg": {
"title": "Message",
"type": "string"
},
"type": {
"title": "Error Type",
"type": "string"
}
},
"required": [
"loc",
"msg",
"type"
],
"title": "ValidationError",
"type": "object"
}
}
},
"paths": {
"/usage/ai/chat": {
"post": {
"description": "AI chat about usage data. Streams SSE events with the AI response.\nThe AI agent has access to tools that query aggregated daily activity data.",
"operationId": "usage_ai_chat_usage_ai_chat_post",
"requestBody": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/UsageAIChatRequest"
}
}
},
"required": true
},
"responses": {
"200": {
"content": {
"application/json": {
"schema": {}
}
},
"description": "Successful Response"
},
"422": {
"content": {
"application/json": {
"schema": {
"$ref": "#/components/schemas/HTTPValidationError"
}
}
},
"description": "Validation Error"
}
},
"security": [
{
"APIKeyHeader": []
}
],
"summary": "Usage Ai Chat",
"tags": [
"usage_ai"
]
}
}
}
},
"vantage": {
"components": {
"schemas": {

7
litellm/proxy/_types.py
View file

@ -2717,6 +2717,13 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
"hidden model can still be called."
),
)
return_wildcard_routes: bool | None = Field(
None,
description=(
"When true, `/v1/models` lists wildcard routes like `openai/*` unless the request passes "
"`return_wildcard_routes=false`."
),
)
alerting: list | None = Field(
None,
description="List of alerting integrations - e.g. `alerting: ['slack', 'webhook', 'email']`. 'slack' posts Slack-format messages to any Slack-compatible webhook (Slack, Rocket.Chat, Mattermost); 'webhook' posts structured JSON budget alerts to WEBHOOK_URL",

3
litellm/proxy/auth/password_policy.py
View file

@ -21,6 +21,7 @@ from typing import Final
from litellm._logging import verbose_proxy_logger
from litellm._version import version
from litellm.constants import HIBP_RANGE_API_BASE
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, get_async_httpx_client
from litellm.proxy._types import ProxyErrorTypes, ProxyException
from litellm.types.llms.custom_http import httpxSpecialProvider
@ -143,7 +144,7 @@ async def _is_password_breached(password: str, client: AsyncHTTPHandler) -> bool
def is_breach_check_enabled(general_settings: Mapping[str, object]) -> bool:
return general_settings.get("password_policy_check_breached_passwords", True) is not False
return not is_offline_mode() and general_settings.get("password_policy_check_breached_passwords", True) is not False
async def is_password_breached(

300
litellm/proxy/db/db_spend_update_writer.py
View file

@ -12,7 +12,6 @@ import json
import os
import random
import time
import traceback
from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable, Mapping, Sequence
from contextlib import asynccontextmanager
from contextvars import ContextVar
@ -90,6 +89,7 @@ from litellm.proxy.spend_tracking.savings import (
)
from litellm.proxy.spend_tracking.spend_log_error_logger import spend_log_error
from litellm.repositories.prisma_protocols import BatchTable
from litellm.responses.streaming_iterator import spawn_logging_task
from litellm.types.utils import CallTypes
if TYPE_CHECKING:
@ -313,6 +313,38 @@ def _spend_tables_left_to_send(
return cast(DBSpendUpdateTransactions, remaining) if any(remaining.values()) else None
def _report_spend_update_step_failure(step_name: str, *, payload: Mapping[str, object], exc: Exception) -> None:
spend_log_error(
"Spend tracking - %s failed, this request is missing from that spend rollup. request_id=%s api_key=%s",
step_name,
payload.get("request_id"),
payload.get("api_key"),
exc=exc,
)
_EMPTY_SPEND_LOG_METADATA: Final = cast(SpendLogsMetadata, MappingProxyType({})) # cast-ok: empty read-only metadata
def _parse_spend_log_metadata(payload: Mapping[str, object]) -> SpendLogsMetadata:
"""Malformed metadata costs the request its metadata-derived fields, never its daily rollup row."""
raw: Final = payload.get("metadata")
if not isinstance(raw, str):
return _EMPTY_SPEND_LOG_METADATA
try:
parsed: Final[SpendLogsMetadata] = json.loads(raw)
except json.JSONDecodeError as e:
spend_log_error(
"Spend tracking - unreadable spend log metadata, rollups for this request are written without "
"metadata-derived fields. request_id=%s api_key=%s",
payload.get("request_id"),
payload.get("api_key"),
exc=e,
)
return _EMPTY_SPEND_LOG_METADATA
return parsed
def _is_billed_retry_attempt(metadata: SpendLogsMetadata) -> bool:
"""An earlier billed retry attempt's own spend row, not a request the caller sent."""
return BILLED_RETRY_ATTEMPT_METADATA_KEY in metadata and metadata[BILLED_RETRY_ATTEMPT_METADATA_KEY] is True
@ -647,8 +679,8 @@ class DBSpendUpdateWriter:
"disable_spend_logs=True. Skipping writing spend logs to db. Other spend updates - Key/User/Team table will still occur."
)
# Single task replaces 11 create_task() calls
asyncio.create_task(
spawn_logging_task(
asyncio.get_running_loop(),
self._batch_database_updates(
response_cost=response_cost,
user_id=user_id,
@ -661,7 +693,8 @@ class DBSpendUpdateWriter:
litellm_proxy_budget_name=litellm_proxy_budget_name,
payload=payload,
request_model_access_groups=get_request_model_access_groups(kwargs),
)
),
task_name="spend rollups",
)
self._enqueue_tool_registry_upsert(
@ -1086,91 +1119,52 @@ class DBSpendUpdateWriter:
project_id: str | None = None,
):
"""
Runs all 13 spend-update helpers sequentially inside a single asyncio task.
Each helper is wrapped in try/except so one failure doesn't prevent the others.
The deepcopy runs here, off the awaited request path, so the daily spend
helpers get a payload isolated from the spend-log queue entry and the caller.
Runs every spend-update helper for one request. Each helper goes through
_run_spend_update_step so one failure never skips the sibling writes. The deepcopy
isolates the daily spend helpers' payload from the spend-log queue entry and the caller.
"""
payload_copy: Final = copy.deepcopy(payload)
request_tags: Final = payload_copy.get("request_tags")
try:
await self._update_user_db(
await self._run_spend_update_step(
"_update_user_db",
self._update_user_db(
response_cost=response_cost,
user_id=user_id,
prisma_client=prisma_client,
litellm_proxy_budget_name=litellm_proxy_budget_name,
end_user_id=end_user_id,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_user_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_key_db(
response_cost=response_cost,
hashed_token=hashed_token,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_key_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_team_db(
response_cost=response_cost,
team_id=team_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_team_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_org_db(
response_cost=response_cost,
org_id=org_id,
user_id=user_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_org_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_project_db(
response_cost=response_cost,
project_id=project_id,
prisma_client=prisma_client,
)
except Exception: # noqa: BLE001 # a project enqueue failure must not skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _update_project_db failed: %s",
traceback.format_exc(),
)
try:
await self._update_tag_db(
response_cost=response_cost,
request_tags=request_tags,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_tag_db failed: %s",
traceback.format_exc(),
)
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_key_db",
self._update_key_db(response_cost=response_cost, hashed_token=hashed_token, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_team_db",
self._update_team_db(
response_cost=response_cost, team_id=team_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_org_db",
self._update_org_db(
response_cost=response_cost, org_id=org_id, user_id=user_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_project_db",
self._update_project_db(response_cost=response_cost, project_id=project_id, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"_update_tag_db",
self._update_tag_db(response_cost=response_cost, request_tags=request_tags, prisma_client=prisma_client),
payload=payload_copy,
)
await self._update_model_access_group_db(
response_cost=response_cost,
request_model_access_groups=request_model_access_groups,
@ -1178,96 +1172,64 @@ class DBSpendUpdateWriter:
prisma_client=prisma_client,
router=get_llm_router(),
)
_agent_id_for_spend: Final = payload_copy.get("agent_id")
try:
await self._update_agent_db(
response_cost=response_cost,
agent_id=_agent_id_for_spend,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: _update_agent_db failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_end_user_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_agent_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_team_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_team_transaction failed: %s",
traceback.format_exc(),
)
await self._run_spend_update_step(
"_update_agent_db",
self._update_agent_db(
response_cost=response_cost, agent_id=payload_copy.get("agent_id"), prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_user_transaction",
self.add_spend_log_transaction_to_daily_user_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_end_user_transaction",
self.add_spend_log_transaction_to_daily_end_user_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_agent_transaction",
self.add_spend_log_transaction_to_daily_agent_transaction(
payload=payload_copy, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_team_transaction",
self.add_spend_log_transaction_to_daily_team_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
try:
self._record_request_error(payload=payload_copy, prisma_client=prisma_client)
except Exception: # noqa: BLE001 # the rollup must never skip the sibling spend writes
verbose_proxy_logger.debug(
"_batch_database_updates: _record_request_error failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy,
org_id=org_id,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_org_transaction failed: %s",
traceback.format_exc(),
)
try:
await self.add_spend_log_transaction_to_daily_tag_transaction(
payload=payload_copy,
prisma_client=prisma_client,
)
except Exception:
verbose_proxy_logger.debug(
"_batch_database_updates: add_spend_log_transaction_to_daily_tag_transaction failed: %s",
traceback.format_exc(),
)
except Exception as e: # noqa: BLE001 # the rollup must never skip the sibling spend writes
_report_spend_update_step_failure("_record_request_error", payload=payload_copy, exc=e)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_org_transaction",
self.add_spend_log_transaction_to_daily_org_transaction(
payload=payload_copy, org_id=org_id, prisma_client=prisma_client
),
payload=payload_copy,
)
await self._run_spend_update_step(
"add_spend_log_transaction_to_daily_tag_transaction",
self.add_spend_log_transaction_to_daily_tag_transaction(payload=payload_copy, prisma_client=prisma_client),
payload=payload_copy,
)
await self._enqueue_model_usage_transaction(payload=payload_copy, prisma_client=prisma_client)
@staticmethod
async def _run_spend_update_step(
step_name: str, step: Coroutine[object, object, object], *, payload: SpendLogsPayload
) -> None:
try:
await step
except Exception as e: # noqa: BLE001 # one helper failing must not skip the sibling spend writes
_report_spend_update_step_failure(step_name, payload=payload, exc=e)
async def _update_key_db(
self,
response_cost: float | None,
@ -2810,7 +2772,7 @@ class DBSpendUpdateWriter:
date: Final = start_time.isoformat() if isinstance(start_time, datetime) else str(start_time or "")
if not date:
return
metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
metadata: Final = _parse_spend_log_metadata(payload)
internal_origin: Final = bool(metadata.get(INTERNAL_CALL_ORIGIN_METADATA_KEY)) # pyright: ignore[reportUnknownMemberType] # TypedDict.get overloads carry Any defaults
request_error_accumulator.record(
payload=payload,
@ -2861,7 +2823,7 @@ class DBSpendUpdateWriter:
# spend, token and per-entity columns written here stay either way.
request_status: Final = prisma_client.get_request_status(payload)
verbose_proxy_logger.debug("Logged request status: %s", request_status)
_metadata: Final[SpendLogsMetadata] = json.loads(payload["metadata"])
_metadata: Final = _parse_spend_log_metadata(payload)
usage_obj: Final = _metadata.get("usage_object", {}) or {}
if isinstance(payload["startTime"], datetime):
start_time: Final = payload["startTime"].isoformat()

2
litellm/proxy/decisions_endpoints/endpoints.py
View file

@ -228,7 +228,7 @@ async def systemone(
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
):
"""
Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.
Judge a `state` against one or more named `questions` and get one calibrated answer per question name.
Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with
`confidence` and `probabilities`), `score` (an index into the `criteria` levels, with `confidence`,

6
litellm/proxy/litellm_pre_call_utils.py
View file

@ -334,6 +334,7 @@ LITELLM_METADATA_ROUTES: Final = (
"/v1/messages",
"responses",
"files",
"interactions",
)
LITELLM_TRACE_CONTROL_METADATA_FIELDS: Final = frozenset(
@ -4277,6 +4278,9 @@ _ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
LlmProviders.VERTEX_AI.value,
)
)
_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS: Final = ",".join(
(LlmProviders.AZURE_AI.value, LlmProviders.DATABRICKS.value)
)
_ANTHROPIC_OAUTH_CREDENTIAL_PROVIDERS: Final = ",".join(sorted(ANTHROPIC_OAUTH_FORWARD_PROVIDERS))
@ -4305,7 +4309,7 @@ def add_provider_specific_headers_to_request(
custom_llm_provider=_ANTHROPIC_API_HEADER_PROVIDERS, extra_headers=anthropic_api_headers
),
ProviderSpecificHeader(
custom_llm_provider=LlmProviders.DATABRICKS.value,
custom_llm_provider=_NATIVE_MESSAGES_ANTHROPIC_API_HEADER_PROVIDERS,
extra_headers=anthropic_api_headers,
native_anthropic_messages_only=True,
),

50
litellm/proxy/management_endpoints/internal_user_endpoints.py
View file

@ -2093,7 +2093,9 @@ async def get_user_key_counts(
return result
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
order_by: Final[dict[str, str]] = {}
if sort_by is None:
@ -2106,6 +2108,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
"spend",
"user_alias",
"user_role",
"max_budget",
"updated_at",
"sso_user_id",
"status",
"key_count",
]
if sort_by not in valid_columns:
raise HTTPException(
@ -2120,6 +2127,11 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
if sort_by in {"status", "key_count"}:
return None
if sort_by in {"max_budget", "updated_at", "sso_user_id"}:
nulls: Final = "first" if sort_by == "max_budget" and sort_order.lower() == "desc" else "last"
return ({sort_by: {"sort": sort_order.lower(), "nulls": nulls}}, {"user_id": "asc"})
order_by[sort_by] = sort_order.lower()
return order_by
@ -2338,16 +2350,40 @@ async def get_users(
# Build order_by conditions
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)
users: Final[Sequence[prisma_models.LiteLLM_UserTable]] = await UserRepository(prisma_client).table.find_many(
where=where,
skip=skip,
take=page_size,
order=(order_by if order_by else {"created_at": "desc"}), # Default to created_at desc if no sort specified
from litellm.proxy.management_helpers.user_list_sorting import sorted_user_ids
computed_sort: Final = sort_by in ("status", "key_count")
sorted_ids: Final = (
await sorted_user_ids(
prisma_client,
sort_by="status" if sort_by == "status" else "key_count",
descending=sort_order.lower() == "desc",
skip=skip,
take=page_size,
role=role,
user_ids=user_ids,
user_email=user_email,
team=team,
sso_user_ids=sso_user_ids,
organization_ids=organization_ids,
search=search,
)
if computed_sort
else ()
)
page_users: Final = await UserRepository(prisma_client).table.find_many(
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where, {"user_id": {"in": sorted_ids}})} if computed_sort else where,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"},
)
positions: Final = {user_id: index for index, user_id in enumerate(sorted_ids)}
users: Final = sorted(page_users, key=lambda user: positions[user.user_id]) if computed_sort else page_users
# Get total count of user rows
total_count: Final[int] = await UserRepository(prisma_client).table.count(where=where)

29
litellm/proxy/management_endpoints/key_management_endpoints.py
View file

@ -7397,20 +7397,26 @@ async def key_aliases(
)
def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str] | None:
order_by: Final[dict[str, str]] = {}
def _validate_sort_params(
sort_by: str | None, sort_order: str
) -> Mapping[str, object] | Sequence[Mapping[str, object]] | None:
if sort_by is None:
return None
# Validate sort_by is a valid column
valid_columns: Final = [
valid_columns: Final = (
"spend",
"max_budget",
"created_at",
"updated_at",
"token",
"key_alias",
]
"expires",
"last_active",
"budget_reset_at",
"total_spend",
"tpm_limit",
"rpm_limit",
)
if sort_by not in valid_columns:
raise HTTPException(
status_code=400,
@ -7424,9 +7430,14 @@ def _validate_sort_params(sort_by: str | None, sort_order: str) -> dict[str, str
detail={"error": "Invalid sort order. Must be 'asc' or 'desc'"},
)
order_by[sort_by] = sort_order.lower()
return order_by
direction: Final = sort_order.lower()
nullable_sort_columns: Final = ("expires", "last_active", "budget_reset_at", "tpm_limit", "rpm_limit")
if sort_by not in (*nullable_sort_columns, "total_spend"):
return {sort_by: direction}
return [
{sort_by: {"sort": direction, "nulls": "last"}} if sort_by in nullable_sort_columns else {sort_by: direction},
{"token": "asc"},
]
def _build_expires_where_clause(expires_filter: str, now: datetime) -> dict[str, object]:
@ -7668,7 +7679,7 @@ async def list_key_helper(
verbose_proxy_logger.debug("Pagination: skip=%s, take=%s", skip, size)
order_by: Final[dict[str, str] | None] = (
order_by: Final = (
_validate_sort_params(sort_by, sort_order) if sort_by is not None and isinstance(sort_by, str) else None
)

46
litellm/proxy/management_endpoints/management_v1/budgets.py
View file

@ -3,6 +3,7 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime
from itertools import chain
from types import MappingProxyType
from typing import Annotated, Final
@ -76,6 +77,32 @@ _ROW_COUNTS: Final = TypeAdapter(tuple[_RowCount, ...])
SELECTED_COLUMNS: Final = ", ".join(f'"{name}"' for name in BudgetListItem.model_fields)
# Unicode 17.0 Nd zeroes, each followed by its other nine digits in UnicodeData.txt.
_DECIMAL_ZEROES: Final = "0٠۰߀०০੦૦୦௦౦೦൦෦๐໐༠၀႐០᠐᥆᧐᪀᪐᭐᮰᱀᱐꘠꣐꤀꧐꧰꩐꯰０𐒠𐴰𐵀𑁦𑃰𑄶𑇐𑋰𑑐𑓐𑙐𑛀𑛐𑛚𑜰𑣠𑥐𑯰𑱐𑵐𑶠\U00011de0𑽐𖄰𖩠𖫀𖭐𖵰𜳰𝟎𝟘𝟢𝟬𝟶𞅀𞋰𞓰𞗱𞥐🯰"
_DECIMAL_DIGITS: Final = "".join(
chain.from_iterable(tuple(chr(ord(zero) + digit) for digit in range(10)) for zero in _DECIMAL_ZEROES)
)
_ASCII_DIGITS: Final = "0123456789" * len(_DECIMAL_ZEROES)
_BUDGET_DURATION_SECONDS: Final = f"""(
SELECT EXTRACT(EPOCH FROM (
(CURRENT_TIMESTAMP AT TIME ZONE 'UTC') +
(parts[1] || ' ' || CASE parts[2]
WHEN 's' THEN 'seconds' WHEN 'm' THEN 'minutes' WHEN 'h' THEN 'hours'
WHEN 'd' THEN 'days' WHEN 'w' THEN 'weeks' WHEN 'mo' THEN 'months' END)::interval
- (CURRENT_TIMESTAMP AT TIME ZONE 'UTC')
)) FROM regexp_match(
CASE lower(trim(budget_duration))
WHEN 'hourly' THEN '1h' WHEN 'daily' THEN '24h'
WHEN 'weekly' THEN '7d' WHEN 'monthly' THEN '30d'
ELSE CASE WHEN octet_length(budget_duration) = length(budget_duration) THEN budget_duration
ELSE translate(budget_duration, '{_DECIMAL_DIGITS}', '{_ASCII_DIGITS}') END END,
'^([0-9]+)(mo|[smhdw])'
) AS duration(parts)
)"""
@dataclass(frozen=True, slots=True)
class PrismaBudgetListExecutor:
"""The database half of the budgets list. Every caller-supplied value is bound to a
@ -91,10 +118,16 @@ class PrismaBudgetListExecutor:
async def find_many(self, plan: QueryPlan) -> Sequence[BudgetListItem]:
clauses, params = where_sql(plan.where)
ordering: Final = ", ".join(
order_by_sql((key,)).replace("NULLS LAST", "NULLS FIRST")
if key.descending and key.field in ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit")
else order_by_sql((key,))
for key in plan.order
).replace('"budget_duration"', _BUDGET_DURATION_SECONDS)
sql: Final = (
f"SELECT {SELECTED_COLUMNS} FROM {BUDGET_TABLE}"
+ (f" WHERE {clauses}" if clauses else "")
+ f" ORDER BY {order_by_sql(plan.order)}"
+ f" ORDER BY {ordering}"
+ f" LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
return _BUDGET_ROWS.validate_python(await self.prisma_client.db.query_raw(sql, *params, plan.take, plan.skip))
@ -111,8 +144,6 @@ def _scope(caller: UserAPIKeyAuth) -> Scope:
return ScopeDenied(reason=f"Only proxy admins can list budgets, your role={caller.user_role}")
# budget_duration is deliberately absent from `sortable`: the column holds strings
# like "7d" and "30d", so a lexicographic ORDER BY puts "30d" ahead of "7d".
BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
{
"budget_duration": FilterSpec(type=str, ops=frozenset(("in", "is_null"))),
@ -123,7 +154,9 @@ BUDGET_FILTERS: Final[Mapping[str, FilterSpec]] = MappingProxyType(
BUDGETS_LIST_SPEC: Final[ListSpec[BudgetListItem, BudgetListItem]] = ListSpec(
resource="budgets",
sortable=frozenset(("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at")),
sortable=frozenset(
("budget_id", "max_budget", "tpm_limit", "rpm_limit", "tpd_limit", "created_at", "budget_duration")
),
searchable=frozenset(("budget_id",)),
filters=BUDGET_FILTERS,
default_sort=(SortKey(field="created_at", descending=True),),
@ -154,8 +187,9 @@ async def list_budgets(
way to page, sort or filter it.
`sort` takes a comma-separated list of `budget_id`, `max_budget`, `tpm_limit`,
`rpm_limit`, `tpd_limit` or `created_at`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. `budget_id` is appended to every sort as the
`rpm_limit`, `tpd_limit`, `created_at` or `budget_duration`, each optionally prefixed with `-` for descending,
and defaults to `-created_at`. Reset periods sort by elapsed time, with calendar months
measured from the current UTC date and unset periods last. `budget_id` is appended to every sort as the
tiebreaker. `q` is a case-insensitive substring match on `budget_id`.
`page_size` defaults to 50 and is capped at 100. Filters are
`filter[budget_duration][in|is_null]`, `filter[max_budget][gte|lte|is_null]`

3
litellm/proxy/management_endpoints/policy_endpoints/endpoints.py
View file

@ -28,6 +28,7 @@ from litellm.constants import (
MAX_COMPETITOR_NAMES,
)
from litellm.integrations.custom_guardrail import CustomGuardrail
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.llms.openai.chat.guardrail_translation.handler import (
OpenAIChatCompletionsHandler,
)
@ -657,7 +658,7 @@ async def get_policy_templates(
Fetches from GitHub with automatic fallback to local backup on failure.
Set LITELLM_LOCAL_POLICY_TEMPLATES=true to skip GitHub and use local backup only.
"""
use_local: Final = os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
use_local: Final = is_offline_mode() or os.getenv("LITELLM_LOCAL_POLICY_TEMPLATES", "").strip().lower() in (
"true",
"1",
"yes",

73
litellm/proxy/management_endpoints/team_endpoints.py
View file

@ -6109,13 +6109,52 @@ async def list_team_v2(
"total_pages": 0,
}
# Build order_by conditions
valid_sort_columns: Final = ["team_id", "team_alias", "created_at"]
order_by = None
if sort_by and sort_by in valid_sort_columns:
if sort_order.lower() not in ["asc", "desc"]:
sort_order = "asc"
order_by = {sort_by: sort_order.lower()}
from litellm.proxy.management_helpers.team_list_sorting import (
COMPUTED_TEAM_SORTS,
UnsupportedTeamFilter,
sorted_team_ids,
)
valid_sort_columns: Final = (
"team_id",
"team_alias",
"created_at",
"spend",
"max_budget",
"tpm_limit",
"rpm_limit",
"updated_at",
)
sort_direction: Final = "desc" if str(sort_order).lower() == "desc" else "asc"
nullable_sort: Final = sort_by in ("max_budget", "tpm_limit", "rpm_limit")
order_by: Final = (
(
{sort_by: {"sort": sort_direction, "nulls": "first" if sort_direction == "desc" else "last"}},
{"team_id": "asc"},
)
if nullable_sort
else ({sort_by: sort_direction}, {"team_id": "asc"})
if sort_by in ("spend", "updated_at")
else {sort_by: sort_direction}
if sort_by in valid_sort_columns
else None
)
computed_sort: Final = not use_deleted_table and sort_by in COMPUTED_TEAM_SORTS
sorted_ids: Final = (
await sorted_team_ids(
prisma_client,
where=where_conditions,
sort_by=sort_by or "",
descending=sort_direction == "desc",
skip=skip,
take=page_size,
)
if computed_sort
else ()
)
if isinstance(sorted_ids, UnsupportedTeamFilter):
raise HTTPException(status_code=500, detail="Unsupported team sorting filter")
# Get teams with pagination
if use_deleted_table:
@ -6130,26 +6169,32 @@ async def list_team_v2(
total_count = await _deleted_team_db(prisma_client).count(where=where_conditions)
else:
teams = await _team_db(prisma_client).find_many(
where=where_conditions,
skip=skip,
# bounded-ok: sorted_ids comes from LIMIT page_size, capped at 100 by the route
where={"AND": (where_conditions, {"team_id": {"in": sorted_ids}})} if computed_sort else where_conditions,
skip=0 if computed_sort else skip,
take=page_size,
order=order_by if order_by else {"created_at": "desc"}, # Default sort
order=order_by if order_by else {"created_at": "desc"},
include=_INCLUDE_MODEL_TABLE,
)
# Get total count for pagination
total_count = await _team_db(prisma_client).count(where=where_conditions)
positions: Final = {team_id: index for index, team_id in enumerate(sorted_ids)}
sorted_teams: Final = sorted(teams, key=lambda team: positions[team.team_id]) if computed_sort else teams
# Calculate total pages
total_pages: Final = -(-total_count // page_size) # Ceiling division
# Aggregate virtual-key counts per team for the current page. The deleted
# table does not carry keys_count, so it is skipped.
keys_count_by_team: dict[str, int] = {}
if not use_deleted_table:
keys_count_by_team = await _get_keys_count_by_team(prisma_client, teams)
keys_count_by_team: Final = (
await _get_keys_count_by_team(prisma_client, sorted_teams) if not use_deleted_table else {}
)
# Convert Prisma models to response models with members_count and keys_count
team_list = _convert_teams_to_response_models(teams, use_deleted_table, keys_count_by_team=keys_count_by_team)
team_list: Final = _convert_teams_to_response_models(
sorted_teams, use_deleted_table, keys_count_by_team=keys_count_by_team
)
# Resolve resources inherited from access groups (single batch query)
if not use_deleted_table:

59
litellm/proxy/management_endpoints/ui_sso.py
View file

@ -152,7 +152,7 @@ from litellm.proxy.utils import (
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import SSOConfigRepository
from litellm.repositories.team_repository import TeamRepository
from litellm.repositories.user_repository import UserRepository
from litellm.repositories.user_repository import UserRepository, free_sso_user_creation
from litellm.secret_managers.main import get_secret_bool, get_secret_str
from litellm.types.llms.base import LiteLLMBaseModel
from litellm.types.proxy.management_endpoints.ui_sso import * # noqa: F403
@ -1074,25 +1074,8 @@ async def _verify_generic_sso_or_fall_back_to_id_token(
)
async def _raise_if_sso_exceeds_free_user_limit(premium_user: bool, prisma_client: PrismaClient | None) -> None:
"""Free tier allows SSO for up to 5 billable users; beyond that requires an Enterprise license."""
if premium_user is True:
return
if prisma_client is None:
raise ProxyException(
message=CommonProxyErrors.db_not_connected_error.value,
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
billable_users: Final = await UserRepository(prisma_client).count_billable_users()
if billable_users and billable_users > 5:
raise ProxyException(
message="You must be a LiteLLM Enterprise user to use SSO for more than 5 users. If you have a license please set `LITELLM_LICENSE` in your env. If you want to obtain a license meet with us here: https://enterprise.litellm.ai/demo You are seeing this error message because You configured SSO (one of `MICROSOFT_CLIENT_ID`, `GOOGLE_CLIENT_ID`, `GENERIC_CLIENT_ID`, or SAML) in your env. Please unset it",
type=ProxyErrorTypes.auth_error,
param="premium_user",
code=status.HTTP_403_FORBIDDEN,
)
class _SSOUserAdmissionError(ProxyException):
pass
@router.get("/sso/key/generate", tags=["experimental"], include_in_schema=False)
@ -1112,8 +1095,6 @@ async def google_login(
from litellm.proxy.proxy_server import (
cli_sso_session_cache,
general_settings,
premium_user,
prisma_client,
saml_sso_cache,
user_custom_ui_sso_sign_in_handler,
)
@ -1128,15 +1109,6 @@ async def google_login(
if admin_ui_is_disabled and not is_cli_sso_login:
return admin_ui_disabled()
####### Check if user is a Enterprise / Premium User #######
if (
microsoft_client_id is not None
or google_client_id is not None
or generic_client_id is not None
or SAMLAuthHandler.is_saml_configured()
):
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
####### Detect DB + MASTER KEY in .env #######
missing_env_vars: Final = show_missing_vars_in_env()
if missing_env_vars is not None:
@ -1963,6 +1935,8 @@ async def get_user_info_from_db(
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if resolved_by_database_only and PrismaDBExceptionHandler.is_database_service_unavailable_error_in_chain(e):
verbose_proxy_logger.exception("SSO login failed: the database could not resolve the account: %s", e)
@ -2328,7 +2302,6 @@ async def saml_callback(request: Request):
general_settings,
jwt_handler,
master_key,
premium_user,
prisma_client,
saml_sso_cache,
)
@ -2352,8 +2325,6 @@ async def saml_callback(request: Request):
user, retained = await SAMLAuthHandler.handle_acs(request=request, cache=saml_sso_cache, post_data=post_data)
await _raise_if_sso_exceeds_free_user_limit(premium_user, prisma_client)
ui_access_mode: Final = general_settings.get("ui_access_mode", None)
relay_state: Final = post_data.get("RelayState")
cp_return_to: Final[str | None] = (
@ -2821,12 +2792,20 @@ async def insert_sso_user(
if result_openid and hasattr(result_openid, "provider"):
new_user_request.metadata = {"auth_provider": getattr(result_openid, "provider")}
response: Final = await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
from litellm.proxy.proxy_server import premium_user
return response
with free_sso_user_creation(new_user_request.user_id, enabled=premium_user is not True):
try:
return await new_user(
data=new_user_request,
user_api_key_dict=UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN),
)
except ProxyException as error:
if premium_user is True:
raise
raise _SSOUserAdmissionError(
message=error.message, type=error.type, param=error.param, code=error.code
) from error
DefaultTeamJoin: TypeAlias = Callable[
@ -3567,6 +3546,8 @@ class SSOAuthenticationHandler:
prisma_client=prisma_client,
)
return user_info
except _SSOUserAdmissionError:
raise
except Exception as e:
if (
user_info is None

9
litellm/proxy/management_endpoints/usage_endpoints/__init__.py
View file

@ -1,9 +0,0 @@
"""
Usage endpoints package.
Re-exports the router from endpoints module.
"""
from litellm.proxy.management_endpoints.usage_endpoints.endpoints import ( # noqa: F401
router,
)

627
litellm/proxy/management_endpoints/usage_endpoints/ai_usage_chat.py
View file

@ -1,627 +0,0 @@
"""
AI Usage Chat - uses LLM tool calling to answer questions about
usage/spend data by querying the aggregated daily activity endpoints.
"""
import json
from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Mapping, Sequence
from datetime import date
from typing import Final, Literal, NamedTuple, Protocol, cast, overload
from fastapi import HTTPException
from typing_extensions import ReadOnly, TypedDict
import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import DEFAULT_COMPETITOR_DISCOVERY_MODEL
from litellm.proxy._types import CommonProxyErrors
from litellm.types.proxy.management_endpoints.common_daily_activity import (
SpendAnalyticsPaginatedResponse,
)
from litellm.types.utils import ChatCompletionMessageToolCall
# ---------------------------------------------------------------------------
# Constants
# ---------------------------------------------------------------------------
USAGE_AI_TEMPERATURE: Final = 0.2
TABLE_DAILY_USER_SPEND: Final = "litellm_dailyuserspend"
TABLE_DAILY_TEAM_SPEND: Final = "litellm_dailyteamspend"
TABLE_DAILY_TAG_SPEND: Final = "litellm_dailytagspend"
ENTITY_FIELD_USER: Final = "user_id"
ENTITY_FIELD_TEAM: Final = "team_id"
ENTITY_FIELD_TAG: Final = "tag"
PAGINATED_PAGE_SIZE: Final = 200
MAX_CHAT_MESSAGES: Final = 20
TOP_N_MODELS: Final = 15
TOP_N_PROVIDERS: Final = 10
TOP_N_KEYS: Final = 10
# ---------------------------------------------------------------------------
# Types
# ---------------------------------------------------------------------------
class SSEStatusEvent(TypedDict):
type: Literal["status"]
message: str
class SSEToolCallEvent(TypedDict, total=False):
type: Literal["tool_call"]
tool_name: str
tool_label: str
arguments: dict[str, str]
status: Literal["running", "complete", "error"]
error: str
class SSEChunkEvent(TypedDict):
type: Literal["chunk"]
content: str
class SSEDoneEvent(TypedDict):
type: Literal["done"]
class SSEErrorEvent(TypedDict):
type: Literal["error"]
message: str
SSEEvent = SSEStatusEvent | SSEToolCallEvent | SSEChunkEvent | SSEDoneEvent | SSEErrorEvent
class _EntityEntry(TypedDict, total=False):
metrics: ReadOnly[Mapping[str, float]]
metadata: ReadOnly[Mapping[str, str]]
class _DayDump(TypedDict, total=False):
breakdown: ReadOnly[Mapping[str, Mapping[str, _EntityEntry]]]
class _EntityTotal(NamedTuple):
"""Running per-entity totals accumulated while summarising a usage dump."""
alias: str
spend: float
requests: float
tokens: float
class _UsageDump(Protocol):
@overload
def get(self, key: Literal["metadata"], default: Mapping[str, float], /) -> Mapping[str, float]: ...
@overload
def get(self, key: Literal["results"], default: Sequence[_DayDump], /) -> Sequence[_DayDump]: ...
class _ToolFunctionDef(TypedDict):
name: ReadOnly[str]
description: ReadOnly[str]
parameters: ReadOnly[Mapping[str, object]]
class _ToolDef(TypedDict):
type: ReadOnly[str]
function: ReadOnly[_ToolFunctionDef]
class ToolHandler(TypedDict):
fetch: Callable[..., Awaitable[_UsageDump]]
summarise: Callable[[_UsageDump], str]
label: str
# ---------------------------------------------------------------------------
# Tool definitions (OpenAI function-calling schema)
# ---------------------------------------------------------------------------
_DATE_PARAMS: Final = {
"start_date": {"type": "string", "description": "Start date in YYYY-MM-DD format"},
"end_date": {"type": "string", "description": "End date in YYYY-MM-DD format"},
}
_TOOL_USAGE: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_usage_data",
"description": (
"Fetch aggregated global usage/spend data. Returns daily spend, "
"token counts, request counts, and breakdowns by model, provider, "
"and API key. Use for overall spend, top models, top providers."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"user_id": {
"type": "string",
"description": "Optional user ID filter. Omit for global view.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TEAM: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_team_usage_data",
"description": (
"Fetch usage/spend data broken down by team. Use for questions "
"like 'which team spends the most' or 'show me team X usage'."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"team_ids": {
"type": "string",
"description": "Optional comma-separated team IDs. Omit for all teams.",
},
},
"required": ["start_date", "end_date"],
},
},
}
_TOOL_TAG: Final[_ToolDef] = {
"type": "function",
"function": {
"name": "get_tag_usage_data",
"description": (
"Fetch usage/spend data broken down by tag. Tags are labels "
"attached to requests (features, environments, credentials)."
),
"parameters": {
"type": "object",
"properties": {
**_DATE_PARAMS,
"tags": {
"type": "string",
"description": "Optional comma-separated tag names. Omit for all tags.",
},
},
"required": ["start_date", "end_date"],
},
},
}
TOOLS_BASE: Final = [_TOOL_USAGE]
TOOLS_ADMIN: Final = [_TOOL_USAGE, _TOOL_TEAM, _TOOL_TAG]
def get_tools_for_role(is_admin: bool) -> list[_ToolDef]:
"""Return the tool list appropriate for the user's role."""
return TOOLS_ADMIN if is_admin else TOOLS_BASE
_SYSTEM_PROMPT_BASE: Final = (
"You are an AI assistant embedded in the LiteLLM Usage dashboard. "
"You help users understand their LLM API spend and usage data.\n\n"
"ALWAYS call the appropriate tool(s) first to fetch data before answering. "
"You may call multiple tools if the question spans different dimensions.\n\n"
"Guidelines:\n"
"- Be concise and specific. Use exact numbers from the data.\n"
"- Format costs as dollar amounts (e.g. $12.34).\n"
"- When comparing entities, show a ranked list.\n"
"- If data is empty or no results found, say so clearly.\n"
"- Do not hallucinate data — only use what the tools return.\n"
"- Today's date will be provided below. Use it to interpret relative dates "
"like 'this week', 'this month', 'last 7 days', etc."
)
_TOOL_DESCRIPTIONS_ADMIN: Final = (
"You have access to these tools:\n"
"- `get_usage_data`: Global/user-level usage (spend, models, providers, API keys)\n"
"- `get_team_usage_data`: Team-level usage breakdown\n"
"- `get_tag_usage_data`: Tag-level usage breakdown\n\n"
)
_TOOL_DESCRIPTIONS_BASE: Final = (
"You have access to this tool:\n- `get_usage_data`: Your usage data (spend, models, providers, API keys)\n\n"
)
def _build_system_prompt(is_admin: bool) -> str:
"""Build role-appropriate system prompt with today's date."""
tool_desc: Final = _TOOL_DESCRIPTIONS_ADMIN if is_admin else _TOOL_DESCRIPTIONS_BASE
return f"{_SYSTEM_PROMPT_BASE}\n\n{tool_desc}Today's date: {date.today().isoformat()}"
# keep a public reference for test assertions
SYSTEM_PROMPT: Final = _SYSTEM_PROMPT_BASE
# ---------------------------------------------------------------------------
# Data fetchers
# ---------------------------------------------------------------------------
def _parse_csv_ids(raw: str | None) -> list[str] | None:
if not raw:
return None
return [t.strip() for t in raw.split(",") if t.strip()]
async def _query_activity(
table_name: str,
entity_id_field: str,
entity_id: str | list[str] | None,
start_date: str,
end_date: str,
*,
use_aggregated: bool = False,
) -> SpendAnalyticsPaginatedResponse:
"""Shared helper that calls the daily activity query layer."""
from litellm.proxy.management_endpoints.common_daily_activity import (
daily_activity_repository,
daily_activity_scope,
get_daily_activity,
get_daily_activity_aggregated,
)
from litellm.proxy.proxy_server import prisma_client
if use_aggregated:
if prisma_client is None:
raise HTTPException(
status_code=500,
detail={"error": CommonProxyErrors.db_not_connected_error.value},
)
repository: Final = daily_activity_repository(prisma_client)
scope: Final = daily_activity_scope(
table_name,
entity_id_field,
entity_id,
None,
None,
start_date,
end_date,
None,
None,
)
return await get_daily_activity_aggregated(
repository,
scope,
)
return await get_daily_activity(
prisma_client=prisma_client,
table_name=table_name,
entity_id_field=entity_id_field,
entity_id=entity_id,
entity_metadata_field=None,
start_date=start_date,
end_date=end_date,
model=None,
api_key=None,
page=1,
page_size=PAGINATED_PAGE_SIZE,
)
async def _fetch_usage_data(start_date: str, end_date: str, user_id: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_USER_SPEND,
ENTITY_FIELD_USER,
user_id,
start_date,
end_date,
use_aggregated=True,
)
return resp.model_dump(mode="json")
async def _fetch_team_usage_data(start_date: str, end_date: str, team_ids: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TEAM_SPEND,
ENTITY_FIELD_TEAM,
_parse_csv_ids(team_ids),
start_date,
end_date,
)
return resp.model_dump(mode="json")
async def _fetch_tag_usage_data(start_date: str, end_date: str, tags: str | None = None) -> _UsageDump:
resp: Final = await _query_activity(
TABLE_DAILY_TAG_SPEND,
ENTITY_FIELD_TAG,
_parse_csv_ids(tags),
start_date,
end_date,
)
return resp.model_dump(mode="json")
# ---------------------------------------------------------------------------
# Summarisers — convert raw JSON to concise text the LLM can reason over
# ---------------------------------------------------------------------------
def _accumulate_breakdown(
results: Sequence[_DayDump], dimension: str, fields: Sequence[str]
) -> dict[str, dict[str, float]]:
"""Aggregate a single breakdown dimension across days."""
totals: Final[dict[str, dict[str, float]]] = {}
for day in results:
for key, entry in day.get("breakdown", {}).get(dimension, {}).items():
if key not in totals:
totals[key] = {f: 0.0 for f in fields}
m = entry.get("metrics", {})
for f in fields:
totals[key][f] += m.get(f, 0)
return totals
def _ranked_lines(
totals: dict[str, dict[str, float]],
fmt: Callable[[str, dict[str, float]], str],
limit: int,
) -> list[str]:
"""Sort by spend descending, format each entry, and truncate."""
return [fmt(name, vals) for name, vals in sorted(totals.items(), key=lambda x: -x[1].get("spend", 0))[:limit]]
def _summarise_usage_data(data: _UsageDump) -> str:
meta: Final = data.get("metadata", {})
results: Final = data.get("results", [])
header: Final = (
f"Total Spend: ${meta.get('total_spend', 0):.4f}\n"
f"Total Requests: {meta.get('total_api_requests', 0)}\n"
f"Successful: {meta.get('total_successful_requests', 0)} | "
f"Failed: {meta.get('total_failed_requests', 0)}\n"
f"Total Tokens: {meta.get('total_tokens', 0)}"
)
models: Final = _accumulate_breakdown(results, "models", ["spend", "api_requests", "total_tokens"])
providers: Final = _accumulate_breakdown(results, "providers", ["spend", "api_requests"])
model_lines: Final = _ranked_lines(
models,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs, {int(d['total_tokens'])} tokens)",
TOP_N_MODELS,
)
provider_lines: Final = _ranked_lines(
providers,
lambda n, d: f" - {n}: ${d['spend']:.4f} ({int(d['api_requests'])} reqs)",
TOP_N_PROVIDERS,
)
sections = [header, ""]
sections += ["Top Models by Spend:"] + (model_lines or [" (no data)"]) + [""]
sections += ["Top Providers by Spend:"] + (provider_lines or [" (no data)"])
return "\n".join(sections)
def _summarise_entity_data(data: _UsageDump, entity_label: str) -> str:
"""Summarise team/tag entity usage data."""
results: Final = data.get("results", [])
if not results:
return f"No {entity_label} usage data found for the given date range."
totals: Final[dict[str, _EntityTotal]] = {}
for day in results:
for eid, entry in day.get("breakdown", {}).get("entities", {}).items():
previous = totals.get(eid)
m = entry.get("metrics", {})
totals[eid] = _EntityTotal(
alias=previous.alias if previous is not None else entry.get("metadata", {}).get("alias", eid),
spend=(previous.spend if previous is not None else 0.0) + m.get("spend", 0),
requests=(previous.requests if previous is not None else 0) + m.get("api_requests", 0),
tokens=(previous.tokens if previous is not None else 0) + m.get("total_tokens", 0),
)
lines: Final = [f"{entity_label} Usage ({len(totals)} {entity_label.lower()}s):", ""]
for eid, d in sorted(totals.items(), key=lambda x: -x[1].spend):
label = d.alias if d.alias != eid else eid
lines.append(f"- {label} (ID: {eid}): ${d.spend:.4f} | {int(d.requests)} reqs | {int(d.tokens)} tokens")
return "\n".join(lines)
# ---------------------------------------------------------------------------
# Tool dispatch registry
# ---------------------------------------------------------------------------
TOOL_HANDLERS: Final[dict[str, ToolHandler]] = {
"get_usage_data": ToolHandler(
fetch=_fetch_usage_data,
summarise=_summarise_usage_data,
label="global usage data",
),
"get_team_usage_data": ToolHandler(
fetch=_fetch_team_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Team"),
label="team usage data",
),
"get_tag_usage_data": ToolHandler(
fetch=_fetch_tag_usage_data,
summarise=lambda data: _summarise_entity_data(data, "Tag"),
label="tag usage data",
),
}
# ---------------------------------------------------------------------------
# SSE streaming
# ---------------------------------------------------------------------------
def _sse(event: SSEEvent) -> str:
return f"data: {json.dumps(event)}\n\n"
def _resolve_fetch_kwargs(
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> dict[str, str]:
"""Build keyword arguments for a tool's fetch function."""
start_date: Final = fn_args.get("start_date", "")
end_date: Final = fn_args.get("end_date", "")
if not start_date or not end_date:
raise ValueError("Missing required start_date or end_date from tool arguments")
kwargs: Final[dict[str, str]] = {"start_date": start_date, "end_date": end_date}
if fn_name == "get_usage_data":
if not is_admin:
if user_id is None:
# Defense-in-depth: the endpoint guard in usage_endpoints/endpoints.py
# should have already rejected this. If we ever reach here it means
# a future caller invoked the helper without scoping — fail loudly
# rather than issuing an unfiltered global query.
raise ValueError(
"Non-admin caller has user_id=None; refusing to issue an "
"unscoped query. Endpoint-level guard missing."
)
kwargs["user_id"] = user_id
elif fn_args.get("user_id"):
kwargs["user_id"] = fn_args["user_id"]
elif fn_name == "get_team_usage_data" and fn_args.get("team_ids"):
kwargs["team_ids"] = fn_args["team_ids"]
elif fn_name == "get_tag_usage_data" and fn_args.get("tags"):
kwargs["tags"] = fn_args["tags"]
return kwargs
async def _execute_tool_call(
handler: ToolHandler,
fn_name: str,
fn_args: Mapping[str, str],
user_id: str | None,
is_admin: bool,
) -> str:
"""Run a single tool and return the summarised result text, or the reason the query
layer rejected its dates so the model can correct them."""
kwargs: Final = _resolve_fetch_kwargs(fn_name, fn_args, user_id, is_admin)
try:
raw_data: Final = await handler["fetch"](**kwargs)
except HTTPException as e:
reason: Final = _invalid_date_reason(e)
if reason is None:
raise
return f"Invalid date range: {reason}"
return handler["summarise"](raw_data)
def _invalid_date_reason(error: HTTPException) -> str | None:
"""The reason in the daily activity query layer's 400 for invalid or reversed dates."""
detail: Final = cast(object, error.detail) # cast-ok: starlette types detail as str; the query layer passes a dict
match detail:
case {"error": str() as reason} if error.status_code == 400:
return reason
case _:
return None
async def _process_tool_call(
tc: ChatCompletionMessageToolCall,
chat_messages: list[Mapping[str, object]],
user_id: str | None,
is_admin: bool,
) -> AsyncIterator[str]:
"""Execute a single tool call, yielding SSE events for status."""
fn_name: Final = tc.function.name
fn_args: Final[Mapping[str, str]] = json.loads(tc.function.arguments)
allowed_names: Final = {t["function"]["name"] for t in get_tools_for_role(is_admin)}
handler: Final = TOOL_HANDLERS.get(fn_name) if fn_name is not None else None
if fn_name is None or fn_name not in allowed_names or not handler:
chat_messages.append(
{
"role": "tool",
"tool_call_id": tc.id,
"content": f"Tool not available: {fn_name}",
}
)
return
tool_event_base: Final = {
"type": "tool_call",
"tool_name": fn_name,
"tool_label": handler["label"],
"arguments": fn_args,
}
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "running"}))
try:
tool_result = await _execute_tool_call(handler, fn_name, fn_args, user_id, is_admin)
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "complete"}))
except Exception as e:
verbose_proxy_logger.error("Tool %s failed: %s", fn_name, e)
tool_result = f"Error fetching {handler['label']}. Please try again."
yield _sse(cast(SSEToolCallEvent, {**tool_event_base, "status": "error"}))
chat_messages.append({"role": "tool", "tool_call_id": tc.id, "content": tool_result})
async def _stream_final_response(model: str, chat_messages: list[Mapping[str, object]]) -> AsyncIterator[str]:
"""Stream the final LLM response after tool results are appended."""
yield _sse({"type": "status", "message": "Analyzing results..."})
response: Final = await litellm.acompletion(
model=model,
messages=chat_messages,
stream=True,
temperature=USAGE_AI_TEMPERATURE,
)
async for chunk in response:
delta = chunk.choices[0].delta.content
if delta:
yield _sse({"type": "chunk", "content": delta})
async def stream_usage_ai_chat(
messages: list[dict[str, str]],
model: str | None = None,
user_id: str | None = None,
is_admin: bool = False,
) -> AsyncGenerator[str, None]:
"""Stream SSE events: status → tool_call → chunk → done."""
resolved_model: Final = (model or "").strip() or DEFAULT_COMPETITOR_DISCOVERY_MODEL
truncated: Final = messages[-MAX_CHAT_MESSAGES:] if len(messages) > MAX_CHAT_MESSAGES else messages
chat_messages: Final[list[Mapping[str, object]]] = [
{"role": "system", "content": _build_system_prompt(is_admin)},
*truncated,
]
try:
yield _sse({"type": "status", "message": "Thinking..."})
tools: Final = get_tools_for_role(is_admin)
response: Final = await litellm.acompletion(
model=resolved_model,
messages=chat_messages,
tools=tools,
temperature=USAGE_AI_TEMPERATURE,
)
choice: Final = response.choices[0]
if not choice.message.tool_calls:
if choice.message.content:
yield _sse({"type": "chunk", "content": choice.message.content})
yield _sse({"type": "done"})
return
chat_messages.append(choice.message.model_dump())
for tc in choice.message.tool_calls:
async for event in _process_tool_call(tc, chat_messages, user_id, is_admin):
yield event
async for event in _stream_final_response(resolved_model, chat_messages):
yield event
yield _sse({"type": "done"})
except Exception as e:
verbose_proxy_logger.error("AI usage chat failed: %s", e)
yield _sse(
{
"type": "error",
"message": "An internal error occurred. Please try again.",
}
)

77
litellm/proxy/management_endpoints/usage_endpoints/endpoints.py
View file

@ -1,77 +0,0 @@
"""
USAGE AI CHAT ENDPOINTS
/usage/ai/chat - Stream AI chat responses about usage data
"""
from typing import Final, Literal
from fastapi import APIRouter, Depends, Request
from fastapi.responses import StreamingResponse
from pydantic import Field
import litellm
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.proxy.common_utils.sse_keepalive import (
SSE_COMMENT_PING,
wrap_sse_stream_with_keepalive_pings,
)
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
class ChatMessage(LiteLLMBaseModel):
role: Literal["user", "assistant"]
content: str
class UsageAIChatRequest(LiteLLMBaseModel):
messages: list[ChatMessage] = Field(..., description="Chat messages (user/assistant history)")
model: str | None = Field(default=None, description="Model to use for AI chat")
@router.post(
"/usage/ai/chat",
tags=["Budget & Spend Tracking"],
dependencies=[Depends(user_api_key_auth)],
)
async def usage_ai_chat(
data: UsageAIChatRequest,
request: Request,
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
):
"""
AI chat about usage data. Streams SSE events with the AI response.
The AI agent has access to tools that query aggregated daily activity data.
"""
from litellm.proxy.management_endpoints.common_utils import (
require_caller_user_id_for_non_admin,
user_api_key_has_admin_view,
)
from litellm.proxy.management_endpoints.usage_endpoints.ai_usage_chat import (
stream_usage_ai_chat,
)
is_admin: Final = user_api_key_has_admin_view(user_api_key_dict)
if is_admin:
user_id = user_api_key_dict.user_id
else:
user_id = require_caller_user_id_for_non_admin(user_api_key_dict)
messages: Final = [{"role": m.role, "content": m.content} for m in data.messages]
return StreamingResponse(
wrap_sse_stream_with_keepalive_pings(
stream_usage_ai_chat(
messages=messages,
model=data.model,
user_id=user_id,
is_admin=is_admin,
),
ping_interval_seconds=litellm.sse_keepalive_ping_interval_seconds,
ping_chunk=SSE_COMMENT_PING,
),
media_type="text/event-stream",
headers={"Cache-Control": "no-cache", "X-Accel-Buffering": "no"},
)

109
litellm/proxy/management_helpers/team_list_sorting.py Normal file
View file

@ -0,0 +1,109 @@
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from itertools import chain
from types import MappingProxyType
from typing import Final
from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter
from litellm.proxy.utils import PrismaClient
COMPUTED_TEAM_SORTS: Final = MappingProxyType(
{
"organization_alias": "COALESCE((SELECT NULLIF(o.organization_alias, '') "
'FROM "LiteLLM_OrganizationTable" o WHERE o.organization_id = t.organization_id), '
"NULLIF(t.organization_id, ''))",
"members": "CASE WHEN jsonb_typeof(t.members_with_roles) = 'array' "
"THEN jsonb_array_length(t.members_with_roles) ELSE 0 END",
"models": "cardinality(t.models)",
"keys_count": '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k WHERE k.team_id = t.team_id)',
}
)
_WHERE: Final = TypeAdapter(dict[str, JsonValue])
@dataclass(frozen=True, slots=True)
class UnsupportedTeamFilter:
field: str
def _field_sql(field: str, value: JsonValue, index: int) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
if field not in ("team_id", "team_alias", "organization_id"):
return UnsupportedTeamFilter(field)
column: Final = f't."{field}"'
match value:
case str(text):
return f"{column} = ${index}", (text,)
case {"in": list(values)}:
return f"{column} = ANY(${index}::text[])", (values,)
case {"contains": str(text), "mode": "insensitive"}:
return f"{column} ILIKE ${index}", (f"%{text}%",)
case {"startsWith": str(text)}:
return f"{column} LIKE ${index}", (f"{text}%",)
case _:
return UnsupportedTeamFilter(field)
def _group_tokens(
field: str, value: Sequence[JsonValue] | Mapping[str, JsonValue]
) -> tuple[str | tuple[str, JsonValue], ...]:
children: Final = tuple(value.items()) if isinstance(value, Mapping) else tuple(("AND", item) for item in value)
if not children:
return ("FALSE" if field == "OR" else "TRUE",)
return (
"(",
*chain.from_iterable(((f" {field} " if index else ""), child) for index, child in enumerate(children)),
")",
)
def _where_sql(where: Mapping[str, JsonValue]) -> tuple[str, tuple[object, ...]] | UnsupportedTeamFilter:
pending: Final[list[str | tuple[str, JsonValue]]] = [("AND", dict(where))] # mutable-ok: iterative tree traversal
fragments: Final[list[str]] = [] # mutable-ok: append-only SQL output avoids quadratic tuple copying
params: Final[list[object]] = [] # mutable-ok: append-only bindings follow traversal order
while pending:
match pending.pop():
case str(fragment):
fragments.append(fragment)
case (field, value) if field in ("AND", "OR") and isinstance(value, (list, dict)):
pending.extend(reversed(_group_tokens(field, value)))
case (field, value):
match _field_sql(field, value, len(params) + 1):
case UnsupportedTeamFilter() as failure:
return failure
case (fragment, values):
fragments.append(fragment)
params.extend(values)
return "".join(fragments), tuple(params)
class _TeamId(BaseModel):
model_config = ConfigDict(frozen=True)
team_id: str
_TEAM_IDS: Final = TypeAdapter(tuple[_TeamId, ...])
async def sorted_team_ids(
prisma_client: PrismaClient,
*,
where: Mapping[str, object],
sort_by: str,
descending: bool,
skip: int,
take: int,
) -> Sequence[str] | UnsupportedTeamFilter:
result: Final = _where_sql(_WHERE.validate_python(where))
if isinstance(result, UnsupportedTeamFilter):
return result
predicate, params = result
expression: Final = COMPUTED_TEAM_SORTS[sort_by]
direction: Final = "DESC" if descending else "ASC"
sql: Final = (
f'SELECT t.team_id FROM "LiteLLM_TeamTable" t WHERE {predicate} '
f"ORDER BY {expression} {direction} NULLS LAST, t.team_id ASC "
f"LIMIT ${len(params) + 1} OFFSET ${len(params) + 2}"
)
rows: Final = _TEAM_IDS.validate_python(await prisma_client.db.query_raw(sql, *params, take, skip))
return tuple(row.team_id for row in rows)

77
litellm/proxy/management_helpers/user_list_sorting.py Normal file
View file

@ -0,0 +1,77 @@
from collections.abc import Sequence
from typing import Final, Literal
from pydantic import BaseModel, ConfigDict, TypeAdapter
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from litellm.proxy.utils import PrismaClient
class _UserId(BaseModel):
model_config = ConfigDict(frozen=True)
user_id: str
_USER_IDS: Final = TypeAdapter(tuple[_UserId, ...])
async def sorted_user_ids(
prisma_client: PrismaClient,
*,
sort_by: Literal["status", "key_count"],
descending: bool,
skip: int,
take: int,
role: str | None,
user_ids: str | None,
user_email: str | None,
team: str | None,
sso_user_ids: str | None,
organization_ids: str | None,
search: str | None,
) -> Sequence[str]:
ids: Final = tuple(value.strip() for value in (user_ids or "").split(",") if value.strip())
sso_ids: Final = tuple(value.strip() for value in (sso_user_ids or "").split(",") if value.strip())
org_ids: Final = tuple(value.strip() for value in (organization_ids or "").split(",") if value.strip())
expression: Final = (
"CASE WHEN u.metadata->'scim_active' = 'false'::jsonb THEN 1 ELSE 0 END"
if sort_by == "status"
else '(SELECT COUNT(*) FROM "LiteLLM_VerificationToken" k '
"WHERE k.user_id = u.user_id AND k.team_id IS DISTINCT FROM $11::text)"
)
direction: Final = "DESC" if descending else "ASC"
sql: Final = f"""
SELECT u.user_id FROM "LiteLLM_UserTable" u
WHERE ($1::text IS NULL OR u.user_role = $1)
AND ($2::text[] IS NULL
OR CASE WHEN cardinality($2::text[]) = 1
THEN u.user_id ILIKE '%' || ($2::text[])[1] || '%'
ELSE u.user_id = ANY($2::text[]) END)
AND ($3::text IS NULL OR u.user_email ILIKE '%' || $3 || '%')
AND ($4::text IS NULL OR $4 = ANY(u.teams))
AND ($5::boolean OR u.sso_user_id = ANY($6::text[]))
AND (cardinality($7::text[]) = 0 OR EXISTS (
SELECT 1 FROM "LiteLLM_OrganizationMembership" m
WHERE m.user_id = u.user_id AND m.organization_id = ANY($7::text[])))
AND ($8::text IS NULL OR u.user_id ILIKE '%' || $8 || '%'
OR u.user_email ILIKE '%' || $8 || '%')
ORDER BY {expression} {direction}, u.user_id ASC
LIMIT $9 OFFSET $10
"""
rows: Final = _USER_IDS.validate_python(
await prisma_client.db.query_raw(
sql,
role or None,
ids if user_ids else None,
user_email,
team,
sso_user_ids is None,
sso_ids,
org_ids,
search or None,
take,
skip,
*((UI_SESSION_TOKEN_TEAM_ID,) if sort_by == "key_count" else ()),
)
)
return tuple(row.user_id for row in rows)

13
litellm/proxy/proxy_server.py
View file

@ -12210,7 +12210,7 @@ async def _deployment_hidden_by_listing_callbacks(deployment: Deployment, user_a
async def model_list(
request: Request = None, # pyright: ignore[reportArgumentType] # FastAPI always injects the Request; the None default only serves direct in-process callers
user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
return_wildcard_routes: bool | None = False,
return_wildcard_routes: bool | None = None,
team_id: str | None = None,
include_model_access_groups: bool | None = False,
only_model_access_groups: bool | None = False,
@ -12226,6 +12226,9 @@ async def model_list(
This is just for compatibility with openai projects like aider.
Query Parameters:
- return_wildcard_routes: When true, also list wildcard routes such as `openai/*`.
Defaults to `general_settings.return_wildcard_routes`, and an explicit
value on the request wins over it
- client_version: Sent by Codex CLI (`?client_version=0.159.3`) when it fetches a
provider's model catalog. When present, the response is Codex's own
catalog shape (`{"models": [...]}`) built from the same listing, with
@ -12254,6 +12257,9 @@ async def model_list(
global llm_model_list, general_settings, llm_router, prisma_client, user_api_key_cache, proxy_logging_obj
settings: Final = cast(dict[str, object], general_settings) # any-ok: legacy settings
include_wildcard_routes: Final = (
return_wildcard_routes if return_wildcard_routes is not None else settings.get("return_wildcard_routes") is True
)
from litellm.llms.anthropic.common_utils import (
create_anthropic_model_list_response,
@ -12343,7 +12349,7 @@ async def model_list(
proxy_model_list=proxy_model_list,
user_model=None,
infer_model_from_keys=False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
llm_router=llm_router,
model_access_groups=model_access_groups,
include_model_access_groups=include_model_access_groups or False,
@ -12411,7 +12417,7 @@ async def model_list(
team_id=team_id,
include_model_access_groups=include_model_access_groups or False,
only_model_access_groups=only_model_access_groups or False,
return_wildcard_routes=return_wildcard_routes or False,
return_wildcard_routes=include_wildcard_routes,
user_api_key_cache=user_api_key_cache,
)
@ -19657,6 +19663,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro
"proxy_config_reload_interval_seconds": "Integer",
"pass_through_endpoints": "PydanticModel",
"store_model_in_db": "Boolean",
"return_wildcard_routes": "Boolean",
"store_prompts_in_spend_logs": "Boolean",
"disable_requester_ip_address_logging": "Boolean",
"disable_fallbacks_on_per_model_rate_limits": "Boolean",

3
litellm/proxy/public_endpoints/public_endpoints.py
View file

@ -19,6 +19,7 @@ from litellm.litellm_core_utils.get_blog_posts import (
get_blog_posts,
)
from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy._types import (
CommonProxyErrors,
)
@ -564,7 +565,7 @@ async def _resolve_autorouter_presets(
url: str,
fetch: Callable[[str], Awaitable[Mapping[str, AutoRouterPresetRecord]]],
) -> Mapping[str, AutoRouterPresetRecord]:
if os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
if is_offline_mode() or os.getenv("LITELLM_LOCAL_AUTOROUTER_PRESETS", "").lower() == "true":
return _load_bundled_autorouter_presets()
try:
return await fetch(url)

78
litellm/proxy/ui_crud_endpoints/latest_release_endpoints.py
View file

@ -1,5 +1,7 @@
import asyncio
import os
import re
import xml.etree.ElementTree as ET
from collections import Counter
from collections.abc import Awaitable, Mapping
from dataclasses import dataclass
@ -12,12 +14,14 @@ from pydantic import ValidationError
from litellm._logging import verbose_proxy_logger
from litellm.caching.in_memory_cache import InMemoryCache
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.proxy.auth.user_api_key_auth import user_api_key_auth
from litellm.types.llms.base import LiteLLMBaseModel
router: Final = APIRouter()
LATEST_RELEASE_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/latest"
CHANGELOG_URL: Final = "https://docs.litellm.ai/release_notes/"
CHANGELOG_FEED_URL: Final = f"{CHANGELOG_URL}rss.xml"
RELEASE_BY_TAG_URL: Final = "https://api.github.com/repos/BerriAI/litellm/releases/tags/{tag}"
LATEST_RELEASE_FETCH_TIMEOUT_SECONDS: Final = 5
LATEST_RELEASE_CACHE_TTL_SECONDS: Final = 60 * 60
@ -26,7 +30,7 @@ LATEST_RELEASE_CACHE_KEY: Final = "latest_release_info"
_RELEASE_BULLET_PATTERN: Final = re.compile(r"^\*\s+(?:([A-Za-z]+)(?:\([^)]*\))?!?:\s)?\S")
_NEW_CONTRIBUTOR_PATTERN: Final = re.compile(r"^\*\s+@\S+ made their first contribution\b")
_STABLE_TAG_PATTERN: Final = re.compile(r"^v?(\d+)\.(\d+)\.(\d+)$")
_CHANGELOG_PAGE_PATTERN: Final = re.compile(rf"{re.escape(CHANGELOG_URL)}v(\d+)\.(\d+)\.(\d+)/")
_Bucket: TypeAlias = Literal["new_features", "bug_fixes", "other_updates"]
_PREFIX_BUCKETS: Final[Mapping[str, _Bucket]] = MappingProxyType({"feat": "new_features", "fix": "bug_fixes"})
@ -45,6 +49,16 @@ class LatestReleaseUnavailable:
reason: str
@dataclass(frozen=True, slots=True)
class ChangelogEntry:
version: tuple[int, int, int]
url: str
@property
def tag(self) -> str:
return "v" + ".".join(str(part) for part in self.version)
class _GitHubRelease(LiteLLMBaseModel):
tag_name: str
html_url: str
@ -76,6 +90,10 @@ def _default_fetch_lock() -> asyncio.Lock:
return _latest_release_fetch_lock
def _default_release_check_enabled() -> bool:
return not is_offline_mode() and os.getenv("LITELLM_DISABLE_LATEST_RELEASE_CHECK", "").strip().lower() != "true"
def _bucket_for(line: str) -> _Bucket | None:
if _NEW_CONTRIBUTOR_PATTERN.match(line) is not None:
return None
@ -91,9 +109,18 @@ def count_release_bullets(body: str) -> Mapping[_Bucket, int]:
return MappingProxyType(Counter(bucket for line in body.splitlines() if (bucket := _bucket_for(line)) is not None))
def _stable_version(tag_name: str) -> tuple[int, int, int] | None:
match: Final = _STABLE_TAG_PATTERN.match(tag_name)
return None if match is None else (int(match.group(1)), int(match.group(2)), int(match.group(3)))
def _changelog_entry(link: str) -> ChangelogEntry | None:
match: Final = _CHANGELOG_PAGE_PATTERN.match(link)
if match is None:
return None
return ChangelogEntry(version=(int(match.group(1)), int(match.group(2)), int(match.group(3))), url=link)
def latest_changelog_entry(feed: bytes) -> ChangelogEntry | None:
"""The highest stable release with a page in the changelog feed. Release candidate pages don't count."""
links: Final = (link.text or "" for link in ET.fromstring(feed).iterfind("channel/item/link"))
entries: Final = (entry for link in links if (entry := _changelog_entry(link)) is not None)
return max(entries, key=lambda entry: entry.version, default=None)
def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
@ -102,7 +129,9 @@ def earlier_patch_tags(version: tuple[int, int, int]) -> tuple[str, ...]:
return tuple(f"v{major}.{minor}.{earlier}" for earlier in range(patch))
def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]) -> LatestReleaseInfo:
def summarize_release_line(
entry: ChangelogEntry, latest: _GitHubRelease, earlier: tuple[_GitHubRelease, ...]
) -> LatestReleaseInfo:
"""A patch release's notes only list its backports, so count the whole minor line it belongs to."""
counts: Final = sum(
(Counter[_Bucket](count_release_bullets(release.body or "")) for release in (latest, *earlier)),
@ -113,7 +142,7 @@ def summarize_release_line(latest: _GitHubRelease, earlier: tuple[_GitHubRelease
new_features=counts.get("new_features", 0),
bug_fixes=counts.get("bug_fixes", 0),
other_updates=counts.get("other_updates", 0),
release_url=latest.html_url,
release_url=entry.url,
)
@ -137,25 +166,38 @@ async def _fetch_earlier_release(client: _AsyncGetClient, tag: str) -> _GitHubRe
return release
async def _fetch_changelog_entry(client: _AsyncGetClient) -> ChangelogEntry | LatestReleaseUnavailable:
response: Final = await client.get(CHANGELOG_FEED_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
if response.status_code != 200:
return LatestReleaseUnavailable(reason=f"Changelog feed responded with status {response.status_code}")
try:
entry: Final = latest_changelog_entry(response.content)
except ET.ParseError as e:
return LatestReleaseUnavailable(reason=f"Changelog feed was not valid XML: {e}")
return LatestReleaseUnavailable(reason="Changelog feed lists no stable release") if entry is None else entry
async def fetch_latest_release(client: _AsyncGetClient) -> LatestReleaseInfo | LatestReleaseUnavailable:
try:
entry: Final = await _fetch_changelog_entry(client)
if isinstance(entry, LatestReleaseUnavailable):
return entry
latest: Final = parse_github_release(
await client.get(LATEST_RELEASE_URL, timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
await client.get(RELEASE_BY_TAG_URL.format(tag=entry.tag), timeout=LATEST_RELEASE_FETCH_TIMEOUT_SECONDS)
)
if isinstance(latest, LatestReleaseUnavailable):
return latest
version: Final = _stable_version(latest.tag_name)
if version is None:
return summarize_release_line(latest, ())
earlier: Final = await asyncio.gather(
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(version))
*(_fetch_earlier_release(client, tag) for tag in earlier_patch_tags(entry.version))
)
except httpx.HTTPError as e:
return LatestReleaseUnavailable(reason=f"{type(e).__name__}: {e}")
unavailable: Final = next((release for release in earlier if isinstance(release, LatestReleaseUnavailable)), None)
if unavailable is not None:
return unavailable
return summarize_release_line(latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease)))
return summarize_release_line(
entry, latest, tuple(release for release in earlier if isinstance(release, _GitHubRelease))
)
async def get_latest_release_info(
@ -188,12 +230,16 @@ async def latest_release_info(
client: Annotated[_AsyncGetClient, Depends(_default_client)],
cache: Annotated[InMemoryCache, Depends(_default_cache)],
fetch_lock: Annotated[asyncio.Lock, Depends(_default_fetch_lock)],
release_check_enabled: Annotated[bool, Depends(_default_release_check_enabled)],
) -> LatestReleaseInfo | None:
"""
Latest stable LiteLLM GitHub release with the PR count of its whole minor line (e.g. v1.105.0 + v1.105.1)
split into new features, bug fixes and other updates.
Returns null when GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Newest stable release on the LiteLLM changelog, linked to its changelog page, with the PR count of its whole
minor line (e.g. v1.105.0 + v1.105.1) from GitHub split into new features, bug fixes and other updates.
Returns null when the changelog or GitHub can't be reached so the dashboard upgrade banner simply doesn't render.
Set LITELLM_DISABLE_LATEST_RELEASE_CHECK=true to return null without making any outbound request.
"""
if not release_check_enabled:
return None
result: Final = await get_latest_release_info(client=client, cache=cache, fetch_lock=fetch_lock)
if isinstance(result, LatestReleaseUnavailable):
verbose_proxy_logger.warning("LiteLLM: latest release info unavailable: %s", result.reason)

13
litellm/proxy/utils.py
View file

@ -6032,13 +6032,12 @@ class PrismaClient:
elif table_name == "user":
db_data = self.jsonify_object(data=data)
try:
new_user_row: Final = await UserRepository(self).table.upsert(
where={"user_id": data["user_id"]},
data={
"create": {**db_data},
"update": {}, # don't do anything if it already exists
},
)
new_user_row: Final = await UserRepository(self).upsert_user(db_data)
if new_user_row is None:
raise HTTPException(
status_code=403,
detail="SSO for more than 5 users requires a LiteLLM Enterprise license. Existing users can still sign in.",
)
except Exception as e:
if (
"Foreign key constraint failed on the field: `LiteLLM_UserTable_organization_id_fkey (index)`"

60
litellm/repositories/user_repository.py
View file

@ -2,8 +2,11 @@
User repository for database operations on LiteLLM_UserTable.
"""
import asyncio
import json
from collections.abc import Mapping, Sequence
from collections.abc import Generator, Mapping, Sequence
from contextlib import AbstractAsyncContextManager, contextmanager
from contextvars import ContextVar
from itertools import chain
from typing import TYPE_CHECKING, Final, Protocol
@ -35,6 +38,16 @@ ORDER BY p.user_id
"""
_PLACEHOLDER_ROWS_ADAPTER: Final = TypeAdapter(tuple[SCIMPlaceholder, ...])
_FREE_SSO_CREATION: Final[ContextVar[tuple[str | None, object] | None]] = ContextVar("free_sso_creation", default=None)
@contextmanager
def free_sso_user_creation(user_id: str | None, *, enabled: bool = True) -> Generator[None]:
token: Final = _FREE_SSO_CREATION.set((user_id, asyncio.current_task()) if enabled else None)
try:
yield
finally:
_FREE_SSO_CREATION.reset(token)
class _UserDb(Protocol):
@ -42,6 +55,12 @@ class _UserDb(Protocol):
def litellm_usertable(self) -> TableActions["prisma_models.LiteLLM_UserTable"]: ...
class _UserTransaction(_UserDb, Protocol):
async def execute_raw(self, query: str) -> int: ...
async def query_raw(self, query: str) -> object: ...
class _PrismaClientView(Protocol):
@property
def db(self) -> _UserDb: ...
@ -49,6 +68,16 @@ class _PrismaClientView(Protocol):
@property
def writer_db(self) -> _UserDb: ...
def tx(self) -> AbstractAsyncContextManager[_UserTransaction]: ...
async def _count_billable_users(table: TableActions["prisma_models.LiteLLM_UserTable"]) -> int:
from prisma import Json
total: Final = await table.count()
deactivated: Final = await table.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
class UserRepository(BaseRepository[LiteLLM_UserTable]):
"""Repository for user database operations."""
@ -137,11 +166,32 @@ class UserRepository(BaseRepository[LiteLLM_UserTable]):
null, or true all count, so seats are counted as total users minus
the deactivated ones.
"""
from prisma import Json # pyright: ignore[reportUnknownVariableType]
return await _count_billable_users(self.table)
total: Final = await self.count()
deactivated: Final = await self.count(where={"metadata": {"path": ["scim_active"], "equals": Json(False)}})
return max(0, total - deactivated)
async def upsert_user(self, data: Mapping[str, object]) -> "prisma_models.LiteLLM_UserTable | None":
"""Return the user row, or None when a new free SSO user would exceed capacity."""
scope: Final = _FREE_SSO_CREATION.get()
if (
scope is None
or (scope[0] is not None and scope[0] != data["user_id"])
or scope[1] is not asyncio.current_task()
):
return await self.table.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
client: Final[_PrismaClientView] = self.prisma_client
async with client.tx() as transaction:
await transaction.execute_raw("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
await transaction.query_raw("SELECT 1 AS locked FROM pg_advisory_xact_lock(468530001)")
existing: Final = await transaction.litellm_usertable.find_unique(where={"user_id": data["user_id"]})
if existing is not None:
return existing
if await _count_billable_users(transaction.litellm_usertable) >= 5:
return None
return await transaction.litellm_usertable.upsert(
where={"user_id": data["user_id"]}, data={"create": dict(data), "update": {}}
)
async def create_user(
self,

68
litellm/responses/main.py
View file

@ -6,10 +6,10 @@ from contextlib import contextmanager
from dataclasses import dataclass
from functools import partial
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NoReturn, Optional, TypeAlias, cast
import httpx
from pydantic import BaseModel, TypeAdapter, ValidationError
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError
from typing_extensions import (
TypeIs, # noqa: TID251 # narrows a client input item before its mark is read
assert_never,
@ -30,6 +30,7 @@ from litellm.integrations.anthropic_cache_control_hook import (
from litellm.litellm_core_utils.asyncify import run_async_function
from litellm.litellm_core_utils.completion_timeout import CompletionTimeout, StreamKind
from litellm.litellm_core_utils.core_helpers import normalize_drop_params
from litellm.litellm_core_utils.get_llm_provider_logic import get_provider_affinity_header
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup
from litellm.litellm_core_utils.prompt_templates.common_utils import (
@ -39,6 +40,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai_like.responses.transformation import OpenAILikeResponsesConfig
from litellm.responses.litellm_completion_transformation.compaction_items import (
with_compaction_items_as_summaries,
@ -85,7 +87,13 @@ from litellm.llms.openai.data_residency import infer_openai_data_residency
from litellm.types.responses.main import *
from litellm.types.responses.streaming_websocket import ResponsesWebSocketRequestDefaults
from litellm.types.router import GenericLiteLLMParams
from litellm.types.utils import ADDRESSED_RESPONSE_ID_FIELD, CallTypes, all_litellm_params
from litellm.types.utils import (
ADDRESSED_RESPONSE_ID_FIELD,
CallTypes,
CostPerToken,
CustomPricingLiteLLMParams,
all_litellm_params,
)
from litellm.utils import (
ProviderConfigManager,
client,
@ -544,6 +552,47 @@ async def aresponses_api_with_mcp(
return response
def _per_call_custom_pricing(litellm_params: GenericLiteLLMParams) -> Mapping[str, float]:
return MappingProxyType(
litellm_params.model_dump(include=set(CustomPricingLiteLLMParams.model_fields), exclude_none=True)
)
class _DeploymentModelInfo(BaseModel):
model_config = ConfigDict(frozen=True)
id: object = None
_DEPLOYMENT_MODEL_INFO: Final[TypeAdapter[_DeploymentModelInfo | object]] = TypeAdapter(
Annotated[_DeploymentModelInfo | object, Field(union_mode="left_to_right")]
)
def _is_router_deployment(deployment_model_info: object) -> bool:
"""A Router stamps the deployment's model_info (with its id) onto every call it routes."""
parsed: Final = _DEPLOYMENT_MODEL_INFO.validate_python(deployment_model_info)
return isinstance(parsed, _DeploymentModelInfo) and parsed.id is not None
_COST_PER_TOKEN: Final = TypeAdapter(CostPerToken)
def _per_call_cost_per_token(litellm_params: GenericLiteLLMParams) -> CostPerToken | None:
if litellm_params.input_cost_per_token is None or litellm_params.output_cost_per_token is None:
return None
return _COST_PER_TOKEN.validate_python(
litellm_params.model_dump(
include={
"input_cost_per_token",
"output_cost_per_token",
"cache_read_input_token_cost",
"cache_creation_input_token_cost",
},
exclude_none=True,
)
)
def _bridges_to_chat_completions(
responses_api_provider_config: BaseResponsesAPIConfig | None, use_chat_completions_api: bool
) -> bool:
@ -1466,6 +1515,12 @@ def responses(
# get llm provider logic
litellm_params: Final = GenericLiteLLMParams(**kwargs)
affinity_api_base: Final = (
get_openai_credentials(api_base=litellm_params.api_base).api_base
if custom_llm_provider == "openai"
else litellm_params.api_base
)
default_affinity_header: Final = get_provider_affinity_header(affinity_api_base)
try:
effective_extra_headers: Final = (
add_provider_affinity_header(
@ -1479,8 +1534,9 @@ def responses(
"litellm_metadata": kwargs.get("litellm_metadata"),
}
),
default_header=default_affinity_header,
)
if litellm_params.provider_affinity_header is not None
if litellm_params.provider_affinity_header is not None or default_affinity_header is not None
else extra_headers
)
except ValueError as affinity_error:
@ -1664,6 +1720,7 @@ def responses(
optional_params=dict(responses_api_request_params),
litellm_params={
**responses_api_request_params,
**_per_call_custom_pricing(litellm_params),
"aresponses": _is_async,
"litellm_call_id": litellm_call_id,
"model_info": kwargs.get("model_info"),
@ -1674,6 +1731,9 @@ def responses(
custom_llm_provider=custom_llm_provider,
)
if not _is_router_deployment(deployment_model_info):
litellm_logging_obj.custom_cost_per_token = _per_call_cost_per_token(litellm_params)
litellm_logging_obj.custom_cost_per_second = litellm_params.cost_per_second
# Decode any litellm-encoded encrypted-content item IDs back to their original IDs
input = with_compaction_items_as_summaries(
with_compaction_items_as_params(ResponsesAPIRequestUtils.restore_encrypted_content_item_ids_in_input(input))

55
litellm/router.py
View file

@ -14186,6 +14186,43 @@ class Router:
deployment, group, Router._params_neither_dropped_nor_allowed(call_settings, params)
)
@staticmethod
def _deployment_declares_no_vision(deployment: DeploymentTypedDict, group: str) -> bool:
plain_deployment: Final = cast(Mapping[str, object], deployment) # cast-ok: router deployments are plain dicts
model_info: Final = cast(Mapping[str, object], plain_deployment.get("model_info") or {}) # cast-ok: plain dict
declared: Final = model_info.get("supports_vision")
if declared is not None:
return declared is False
raw_params: Final = plain_deployment.get("litellm_params") or {}
params: Final = cast(Mapping[str, object], raw_params) # cast-ok: router deployments are plain dicts
if not params:
return False
base_model: Final = model_info.get("base_model") or params.get("base_model")
try:
model, custom_llm_provider = Router._capability_model_and_provider(deployment["litellm_params"], group)
return litellm.utils.is_vision_explicitly_disabled(
base_model if isinstance(base_model, str) else model, custom_llm_provider
)
except Exception as e: # noqa: BLE001 # best-effort filter: an unresolvable provider must not narrow the request
verbose_router_logger.debug(
"litellm.router.py::_deployment_declares_no_vision: keeping deployment for model=%s. Got - %s", group, e
)
return False
@staticmethod
def _request_carries_image(
messages: list[dict[str, str]] | None, # mutable-ok: handed to resolve_structured_messages as-is
request_kwargs: Mapping[str, object],
) -> bool:
from litellm.litellm_core_utils.prompt_templates.common_utils import request_contains_image_content
from litellm.litellm_core_utils.prompt_templates.factory import resolve_structured_messages
structured: Final = resolve_structured_messages(
messages=cast(list[dict[str, object]] | None, messages), # cast-ok: chat messages are plain dicts
request_kwargs=cast(dict[str, object], request_kwargs), # cast-ok: the resolver only reads the mapping
)
return bool(structured) and request_contains_image_content(structured)
@staticmethod
def _pre_call_check_requested_chat_params(request_kwargs: Mapping[str, object]) -> frozenset[str]:
requested: Final = litellm.utils.get_non_default_params(passed_params=dict(request_kwargs))
@ -14212,6 +14249,7 @@ class Router:
- model context window < message length. For azure openai models, requires 'base_model' is set. - https://docs.litellm.ai/docs/proxy/cost_tracking#spend-tracking-for-azure-openai-models
- filter models above rpm limits
- if region given, filter out models not in that region / unknown region
- request carries an image and the deployment declares supports_vision false, unless every candidate does
- [TODO] function call and model doesn't support function calling
"""
@ -14221,7 +14259,7 @@ class Router:
# We only pop from the list, not modify deployment dicts - 100x+ faster on hot path (every request)
_returned_deployments = list(healthy_deployments)
invalid_model_indices: Final = set() # Use set for O(1) membership checks
invalid_model_indices: Final = set[int]() # Use set for O(1) membership checks
_context_window_error = False
_potential_error_str = ""
@ -14371,9 +14409,20 @@ class Router:
model=model,
llm_provider="",
)
if len(excluded_indices) > 0:
remaining: Final = tuple(
(idx, deployment) for idx, deployment in param_candidates if idx not in excluded_indices
)
text_only_indices: Final[frozenset[int]] = (
frozenset(idx for idx, deployment in remaining if self._deployment_declares_no_vision(deployment, model))
if remaining and self._request_carries_image(messages, request_settings)
else frozenset()
)
dropped_indices: Final[frozenset[int]] = excluded_indices | (
text_only_indices if len(text_only_indices) < len(remaining) else frozenset()
)
if len(dropped_indices) > 0:
# Single-pass filter using set for O(1) lookups (avoids O(n^2) from repeated pops)
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in excluded_indices]
_returned_deployments = [d for i, d in enumerate(_returned_deployments) if i not in dropped_indices]
return _returned_deployments

1
litellm/rust_bridge/catalog.py
View file

@ -165,6 +165,7 @@ CHAT_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
}
MESSAGES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | {
Feature.PROVIDER_AFFINITY,
Feature.SAFEGUARDS_CLASSIFIER,
Feature.PRE_REQUEST_HOOK,
Feature.WEBSEARCH_INTERCEPTION,

7
litellm/rust_bridge/messages/detectors.py
View file

@ -5,10 +5,15 @@ from __future__ import annotations
from typing import Final
import litellm
from litellm.rust_bridge.detectors import overridden, present, streaming
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, overridden, present, streaming
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
def provider_affinity(request: NativeCall) -> bool:
return is_opencode_api_base(api_base(request))
def anthropic_workload_identity(request: NativeCall) -> bool:
"""Python mints an Anthropic federation token when no static credential is configured."""
from litellm.llms.anthropic.common_utils import AnthropicModelInfo

3
litellm/rust_bridge/responses/detectors.py
View file

@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Final, cast # noqa: TID251 # tool lists stay
from pydantic import TypeAdapter
import litellm
from litellm.llms.opencode.common_utils import is_opencode_api_base
from litellm.rust_bridge.detectors import api_base, call_callbacks, overridden, present
from litellm.rust_bridge.public_call import NativeCall, optional_mapping, optional_sequence, optional_str
@ -25,7 +26,7 @@ def client_headers(request: NativeCall) -> bool:
def provider_affinity(request: NativeCall) -> bool:
"""Python validates and fills the affinity header from the session ID."""
return present(request, "provider_affinity_header")
return present(request, "provider_affinity_header") or is_opencode_api_base(api_base(request))
def text_format(request: NativeCall) -> bool:

0
litellm/text_completions/__init__.py Normal file
View file

88
litellm/text_completions/streaming_iterator.py Normal file
View file

@ -0,0 +1,88 @@
from typing import Final
from litellm.types.utils import Choices, ModelResponse, TextChoices, TextCompletionResponse
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response.usage = chunk.get("usage", None)
else:
response.usage = None
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration

2
litellm/types/decisions.py
View file

@ -73,7 +73,7 @@ DecisionQuestion: TypeAlias = Annotated[
DecisionQuestionMap: TypeAlias = Annotated[
Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion],
Field(min_length=1, max_length=MAX_DECISION_QUESTIONS),
Field(min_length=1),
]

30
litellm/types/interactions/README.md
View file

@ -9,16 +9,38 @@ https://ai.google.dev/static/api/interactions.openapi.json
### How to Regenerate
When the API spec changes, regenerate the types with:
The compliance tests in `tests/unit/interactions/test_openapi_compliance.py` read the snapshot at
`tests/unit/interactions/interactions.openapi.json`, so a PR never depends on Google's live spec. The scheduled
`interactions-spec-canary` workflow runs those tests against the live spec once a day (`--interactions-spec-url`),
files the issue "Interactions spec canary is red" on a red run and closes it once a run is green again. A red run
means the spec moved in a way these types no longer cover
When the red run names one new status, the small fix is a hand edit adding that member to the enum the failing
schema's model carries, the way `queued` was added in #34135. When the spec moved further than that, refresh the
snapshot and regenerate the types from it:
```bash
uv tool run --from datamodel-code-generator datamodel-codegen \
--url "https://ai.google.dev/static/api/interactions.openapi.json" \
curl -fsSL https://ai.google.dev/static/api/interactions.openapi.json \
-o tests/unit/interactions/interactions.openapi.json
uv tool run --from datamodel-code-generator==0.66.0 datamodel-codegen \
--input tests/unit/interactions/interactions.openapi.json \
--input-file-type openapi \
--output litellm/types/interactions/generated.py \
--output-model-type pydantic_v2.BaseModel \
--target-python-version 3.9
--target-python-version 3.10 \
--base-class litellm.types.llms.base.LiteLLMBaseModel
```
The pin is the same `datamodel-code-generator` version `scripts/generate_trace_types.py` uses. The generator names
inline enums by order of appearance (`Status`, `Status1`, `Status2`, ...), so a regeneration can rename them. The
compliance tests resolve each status enum through its model's `status` field, but
`tests/unit/interactions/test_background_cost_polling.py` and
`tests/unit/proxy/hooks/test_proxy_track_cost_callback.py` import `Status1` by name and need the new name, and
`litellm/types/interactions/__init__.py` exports the interaction status enum as `InteractionStatus`, which has to
point at the enum the regenerated `Interaction.status` carries (a compliance test checks their members match). The
spec has grown a lot since the 2025-12-16 generation (agents, triggers, credentials, environments), so a
regeneration today is a review-sized change rather than a refresh
Then add the LiteLLM-specific types at the bottom of the generated file:
- `InteractionsAPIResponse`
- `InteractionsAPIStreamingResponse`

103
litellm/utils.py
View file

@ -93,6 +93,7 @@ from litellm.litellm_core_utils.fallback_generalizations import (
)
from litellm.litellm_core_utils.get_model_cost_map import AliasedCostEntry
from litellm.litellm_core_utils.model_param_helper import peek_reasoning_summary_aliases
from litellm.litellm_core_utils.offline_mode import is_offline_mode
from litellm.litellm_core_utils.sensitive_data_masker import redact_credentials_in_payload
from litellm.litellm_core_utils.tokenizer import strip_special_tokens
from litellm.litellm_core_utils.tool_fields import MalformedField, resolve_deployment_tool_fields
@ -218,6 +219,7 @@ def _get_cached_audio_utils():
return _audio_utils_module
from litellm.text_completions.streaming_iterator import TextCompletionStreamWrapper as _TextCompletionStreamWrapper
from litellm.types.llms.openai import (
AllMessageValues,
AllPromptValues,
@ -262,13 +264,17 @@ from litellm.types.utils import (
SearchProviders,
SelectTokenizerResponse,
StreamingChoices,
TextChoices,
TextCompletionResponse,
TranscriptionResponse,
Usage,
credential_api_key_secret_reference,
is_litellm_owned_kwarg,
)
from litellm.types.utils import (
TextChoices as _TextChoices,
)
from litellm.types.utils import (
TextCompletionResponse as _TextCompletionResponse,
)
_CALL_TYPE_ENUM_MAP: Final[dict] = {ct.value: ct for ct in CallTypes}
_BACKFILL_MODES: Final = frozenset({"chat", "responses"})
@ -299,10 +305,14 @@ except (ImportError, AttributeError, TypeError):
claude_json_str = json.dumps(json_data)
import importlib.metadata
from collections.abc import AsyncIterator, Callable, Collection, Iterable, Iterator, Mapping, Sequence
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeVar, cast, runtime_checkable
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, TypeAlias, TypeVar, cast, runtime_checkable
from typing_extensions import assert_never
TextChoices: TypeAlias = _TextChoices
TextCompletionResponse: TypeAlias = _TextCompletionResponse
TextCompletionStreamWrapper: TypeAlias = _TextCompletionStreamWrapper
from litellm import utils as litellm_utils
# These are lazy loaded via __getattr__
@ -340,7 +350,8 @@ if TYPE_CHECKING:
# Type stubs for lazy-loaded functions to help mypy understand their types
# These imports allow mypy to understand the types when these are accessed via __getattr__
from litellm.litellm_core_utils.exception_mapping_utils import exception_type
exception_type: Callable[..., Exception]
from litellm.litellm_core_utils.get_litellm_params import (
get_base_model_from_litellm_call_metadata,
get_litellm_params,
@ -2673,7 +2684,7 @@ def _warn_huggingface_tokenizer_fallback(model: str, reason: str) -> None:
def _return_huggingface_tokenizer(model: str) -> SelectTokenizerResponse | _HuggingFaceTokenizerLoadFailure | None:
kind: Final = huggingface_tokenizer_kind(model)
if kind is None:
if kind is None or (kind not in _BUNDLED_TOKENIZER_LOAD_LOCKS and is_offline_mode()):
return None
backend: Final = _huggingface_tokenizer_backend()
key: Final = _huggingface_tokenizer_failure_key(kind, backend)
@ -7945,88 +7956,6 @@ def register_prompt_template(
return litellm.custom_prompt_dict
class TextCompletionStreamWrapper:
def __init__(
self,
completion_stream,
model,
stream_options: dict | None = None,
custom_llm_provider: str | None = None,
):
self.completion_stream = completion_stream
self.model = model
self.stream_options = stream_options
self.custom_llm_provider = custom_llm_provider
def __iter__(self):
return self
def __aiter__(self):
return self
def convert_to_text_completion_object(self, chunk: ModelResponse):
try:
response: Final = TextCompletionResponse()
response["id"] = chunk.get("id", None)
response["object"] = "text_completion"
response["created"] = chunk.get("created", None)
response["model"] = chunk.get("model", None)
text_choices: Final = TextChoices()
if isinstance(chunk, Choices): # chunk should always be of type StreamingChoices
raise Exception
if chunk["choices"]:
delta: Final = chunk["choices"][0]["delta"]
text_choices["text"] = delta["content"]
text_choices["reasoning_content"] = delta.get("reasoning_content")
text_choices["index"] = chunk["choices"][0]["index"]
text_choices["finish_reason"] = chunk["choices"][0]["finish_reason"]
response["choices"] = [text_choices]
else:
response["choices"] = []
# only pass usage when stream_options["include_usage"] is True
if self.stream_options and self.stream_options.get("include_usage", False) is True:
response["usage"] = chunk.get("usage", None)
return response
except Exception as e:
raise Exception(f"Error occurred converting to text completion object - chunk: {chunk}; Error: {e}")
def __next__(self):
# model_response = ModelResponse(stream=True, model=self.model)
TextCompletionResponse()
try:
for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopIteration
except Exception as e:
exception_type: Final = litellm_utils.exception_type
raise exception_type(
model=self.model,
custom_llm_provider=self.custom_llm_provider or "",
original_exception=e,
completion_kwargs={},
extra_kwargs={},
)
async def __anext__(self):
try:
async for chunk in self.completion_stream:
if chunk == "None" or chunk is None:
raise Exception
processed_chunk = self.convert_to_text_completion_object(chunk=chunk)
return processed_chunk
raise StopIteration
except StopIteration:
raise StopAsyncIteration
def mock_stream_usage_chunk(model_response: ModelResponseStream, model: str, prompt_tokens: int) -> ModelResponseStream:
return ModelResponseStream(
id=model_response.id,

2
migrations/Dockerfile
View file

@ -61,7 +61,7 @@ COPY --from=uvbin /uv /uvx /usr/local/bin/
# instead of nodeenv downloading one whose dynamic deps may not be in Wolfi
# (e.g. Node 26.2.0 needs libatomic). Retry for transient apk.cgr.dev flakes.
RUN for i in 1 2 3; do \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl openssl-dev libsndfile nodejs npm && break; \
apk add --no-cache bash gcc python-3.13 python-3.13-dev openssl libssl3 openssl-dev libsndfile nodejs npm && break; \
[ $i = 3 ] && { echo "apk add failed after 3 retries" >&2; exit 1; }; \
sleep 5; \
done

276
model_prices_and_context_window.json
View file

@ -12827,6 +12827,7 @@
"supports_vision": true
},
"azure_ai/Phi-4": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 16384,
@ -12840,6 +12841,7 @@
"supports_vision": false
},
"azure_ai/Phi-4-mini-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12851,6 +12853,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-multimodal-instruct": {
"deprecation_date": "2027-01-17",
"input_cost_per_audio_token": 4e-06,
"input_cost_per_token": 8e-08,
"litellm_provider": "azure_ai",
@ -12865,6 +12868,7 @@
"supports_vision": true
},
"azure_ai/Phi-4-mini-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 7.5e-08,
"litellm_provider": "azure_ai",
"max_input_tokens": 131072,
@ -12876,6 +12880,7 @@
"supports_function_calling": true
},
"azure_ai/Phi-4-reasoning": {
"deprecation_date": "2027-01-17",
"input_cost_per_token": 1.25e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 32768,
@ -24892,27 +24897,118 @@
]
},
"elevenlabs/eleven_v3": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 - most expressive TTS model with 70+ languages and audio tags support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_multilingual_v2": {
"input_cost_per_character": 0.00018,
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.18/1000 characters (Scale plan pricing, 1 credit per character)",
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Multilingual v2 - default TTS model with 29 languages support"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4": {
"input_cost_per_character": 8e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.08/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 - most emotive TTS model with audio tags and 90+ languages"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v4_turbo": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v4 Turbo - low-latency v4 tuned for real-time and agents"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_v3_conversational": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven v3 Conversational - low-latency v3 tuned for realtime conversation"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2.5 - low-latency multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_flash_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Flash v2 - low-latency English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2_5": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2.5 - multilingual TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
},
"elevenlabs/eleven_turbo_v2": {
"input_cost_per_character": 4e-05,
"litellm_provider": "elevenlabs",
"metadata": {
"calculation": "$0.04/1000 characters (API usage pricing, billed in USD per character)",
"notes": "ElevenLabs Eleven Turbo v2 - English-only TTS model"
},
"mode": "audio_speech",
"source": "https://elevenlabs.io/pricing/api",
"supported_endpoints": [
"/v1/audio/speech"
]
@ -36861,17 +36957,17 @@
},
"gpt-6-luna": {
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_decisions": 1e-07,
"cache_creation_input_token_cost_decisions": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_creation_input_token_cost_above_272k_tokens_decisions": 2.5e-07,
"cache_creation_input_token_cost_above_272k_tokens_flex": 1.25e-07,
"cache_creation_input_token_cost_above_272k_tokens_priority": 5e-07,
"cache_creation_input_token_cost_flex": 6.25e-08,
"cache_creation_input_token_cost_priority": 2.5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_decisions": 1e-07,
"cache_read_input_token_cost_decisions": 1e-08,
"cache_read_input_token_cost_above_272k_tokens": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-07,
"cache_read_input_token_cost_above_272k_tokens_decisions": 2e-08,
"cache_read_input_token_cost_above_272k_tokens_flex": 1e-08,
"cache_read_input_token_cost_above_272k_tokens_priority": 4e-08,
"cache_read_input_token_cost_batches": 5e-09,
@ -42337,6 +42433,134 @@
],
"supports_audio_output": true
},
"mistral/voxtral-mini-transcribe-3": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-3-0": {
"input_cost_per_second": 5.83333e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-small-transcribe-3-0": {
"input_cost_per_second": 6.66667e-05,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-small-transcribe-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-transcribe-realtime-3-0": {
"input_cost_per_second": 0.000133333,
"litellm_provider": "mistral",
"mode": "audio_transcription",
"source": "https://docs.mistral.ai/models/voxtral-mini-transcribe-realtime-3-0",
"supported_endpoints": [
"/v1/audio/transcriptions"
],
"supported_modalities": [
"audio"
],
"supported_output_modalities": [
"text"
],
"supports_audio_input": true
},
"mistral/voxtral-mini-tts-3": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-mini-tts-3-0": {
"input_cost_per_character": 1.3e-05,
"litellm_provider": "mistral",
"mode": "audio_speech",
"source": "https://docs.mistral.ai/models/voxtral-mini-tts-3-0",
"supported_endpoints": [
"/v1/audio/speech"
],
"supported_modalities": [
"text"
],
"supported_output_modalities": [
"audio"
],
"supports_audio_output": true
},
"mistral/voxtral-small-2507": {
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
@ -45623,6 +45847,7 @@
"output_cost_per_token": 0.0
},
"openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -45648,6 +45873,7 @@
"supports_tool_choice": true
},
"openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"supported_endpoints": [
"/v1/chat/completions"
],
@ -48499,6 +48725,17 @@
"supports_tool_choice": true,
"supports_vision": true
},
"together_ai/MiniMaxAI/MiniMax-M3.1-Flash": {
"cache_read_input_token_cost": 1.5e-08,
"input_cost_per_token": 1.5e-07,
"litellm_provider": "together_ai",
"max_input_tokens": 1048576,
"max_tokens": 1048576,
"mode": "chat",
"output_cost_per_token": 6e-07,
"source": "https://www.together.ai/pricing",
"supports_prompt_caching": true
},
"together_ai/Prism-ML/Ternary-Bonsai-27B": {
"input_cost_per_token": 0.0,
"litellm_provider": "together_ai",
@ -49352,6 +49589,7 @@
"supports_vision": false
},
"us-gov.openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -49371,6 +49609,7 @@
"supports_tool_choice": true
},
"us-gov.openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock_converse",
"max_input_tokens": 128000,
@ -52906,12 +53145,14 @@
},
"vertex_ai/meta/llama-4-maverick-17b-16e-instruct-maas": {
"input_cost_per_token": 3.5e-07,
"input_cost_per_token_batches": 1.75e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 1000000,
"max_output_tokens": 1000000,
"max_tokens": 1000000,
"mode": "chat",
"output_cost_per_token": 1.15e-06,
"output_cost_per_token_batches": 5.75e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -52926,12 +53167,14 @@
},
"vertex_ai/meta/llama-4-scout-17b-128e-instruct-maas": {
"input_cost_per_token": 2.5e-07,
"input_cost_per_token_batches": 1.25e-07,
"litellm_provider": "vertex_ai-llama_models",
"max_input_tokens": 10000000,
"max_output_tokens": 10000000,
"max_tokens": 10000000,
"mode": "chat",
"output_cost_per_token": 7e-07,
"output_cost_per_token_batches": 3.5e-07,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing#partner-models",
"supported_modalities": [
"text",
@ -59917,6 +60160,7 @@
}
},
"bedrock_mantle/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7.5e-08,
"input_cost_per_token_flex": 7.5e-08,
@ -59942,6 +60186,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3.5e-08,
"input_cost_per_token_flex": 3.5e-08,
@ -59967,6 +60212,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-120b": {
"supports_vision": false,
"input_cost_per_token": 1.5e-07,
"input_cost_per_token_batches": 7e-08,
"input_cost_per_token_flex": 7e-08,
@ -59990,6 +60236,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/openai.gpt-oss-safeguard-20b": {
"supports_vision": false,
"input_cost_per_token": 7e-08,
"input_cost_per_token_batches": 3e-08,
"input_cost_per_token_flex": 3e-08,
@ -67841,6 +68088,7 @@
"supports_vision": false
},
"bedrock/us-gov-west-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -67860,6 +68108,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-west-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68138,6 +68387,7 @@
"supports_vision": false
},
"bedrock/us-gov-east-1/openai.gpt-oss-20b-1:0": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68157,6 +68407,7 @@
"supports_tool_choice": true
},
"bedrock/us-gov-east-1/openai.gpt-oss-120b-1:0": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"litellm_provider": "bedrock",
"max_input_tokens": 128000,
@ -68545,6 +68796,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68564,6 +68816,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-west-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -68704,6 +68957,7 @@
"supports_vision": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-20b": {
"supports_vision": false,
"input_cost_per_token": 8.4e-08,
"output_cost_per_token": 3.6e-07,
"litellm_provider": "bedrock_mantle",
@ -68723,6 +68977,7 @@
"supports_tool_choice": true
},
"bedrock_mantle/us-gov-east-1/openai.gpt-oss-120b": {
"supports_vision": false,
"input_cost_per_token": 1.8e-07,
"output_cost_per_token": 7.2e-07,
"litellm_provider": "bedrock_mantle",
@ -73330,6 +73585,7 @@
]
},
"vertex_ai/veo-2.0-generate-001": {
"deprecation_date": "2026-06-30",
"litellm_provider": "vertex_ai-video-models",
"max_input_tokens": 1024,
"max_tokens": 1024,

5
osv-scanner.toml
View file

@ -3,6 +3,11 @@ id = "GHSA-w8v5-vhqr-4h9v"
ignoreUntil = 2026-11-01
reason = "diskcache has no fixed release published; remove this entry once one exists"
[[IgnoredVulns]]
id = "GHSA-v4x8-3wr5-3wq8"
ignoreUntil = 2026-11-20
reason = "the only fixed filelock is 4.2.1, a new major that requires Python >= 3.11 while litellm still supports 3.10, and the advisory is in ReadWriteLock, which litellm never uses (only FileLock); bump filelock and remove this entry once 3.10 is dropped or a 3.x fix ships"
[[IgnoredVulns]]
id = "GHSA-vfj7-8cjw-p6xm"
ignoreUntil = 2026-11-03

33
pyproject.toml
View file

@ -35,7 +35,7 @@ dependencies = [
"pydantic>=2.12.0,<3.0.0; python_version >= '3.14'",
"pydantic-settings>=2.14.1,<3.0",
"jsonschema>=4.0.0,<5.0",
"boto3>=1.43.1,<2.0",
"boto3>=1.43.117,<2.0",
"certifi>=2026.10.19",
"typing-extensions>=4.13.0,<5.0",
]
@ -73,7 +73,7 @@ proxy = [
"cryptography>=49.0.0,<51.0",
"pynacl>=1.6.2,<2.0",
"websockets>=15.0.1,<16.0",
"boto3>=1.43.1,<2.0",
"boto3>=1.43.117,<2.0",
"azure-identity>=1.25.2,<2.0",
"azure-storage-blob>=12.28.0,<13.0",
"mcp>=2.3.0,<3",
@ -137,10 +137,10 @@ semantic-router = [
"semantic-router>=0.1.15,<1.0; python_version < '3.14'",
"aurelio-sdk>=0.0.19,<1.0; python_version < '3.14'",
]
mlflow = ["mlflow>=3.11.1,<4.0"]
mlflow = ["mlflow>=3.17.2,<4.0"]
grpc = [
# Newest non-yanked release older than the 30-day cutoff.
"grpcio==1.78.0",
# First release whose bundled roots.pem drops the compromised eMudhra roots, ahead of the 30-day cutoff
"grpcio==1.84.1",
]
stt-vertex-chirp = [
# Google Cloud Speech-to-Text v2 streaming (gRPC) for Chirp models on
@ -174,7 +174,7 @@ proxy-runtime = [
"google-cloud-speech>=2.40.0,<3.0",
"google-genai>=1.37.0,<2.0",
"anthropic[vertex]>=0.84.0,<1.0",
"grpcio==1.78.0",
"grpcio==1.84.1",
"prometheus-client>=0.20.0,<1.0",
"langfuse>=4.7,<5.0",
"opentelemetry-api==1.45.1",
@ -186,7 +186,7 @@ proxy-runtime = [
"mangum>=0.17.0,<1.0",
"azure-ai-contentsafety>=1.0.0,<2.0",
"azure-storage-file-datalake>=12.20.0,<13.0",
"pypdf>=6.21.1,<7.0",
"pypdf>=6.21.2,<7.0",
"llm-sandbox>=0.3.39,<1.0",
"detect-secrets>=1.5.0,<2.0",
]
@ -359,7 +359,7 @@ constraint-dependencies = [
"packaging>=24.0",
"soupsieve>=2.8.4",
"httplib2>=0.32.0",
"setuptools>=83.0.0",
"setuptools>=84.1.0",
]
override-dependencies = [
# a2a-sdk 1.x requires packaging>=24.0; lunary 1.4.x still caps at <24.0.
@ -370,8 +370,23 @@ override-dependencies = [
default-groups = ["dev"]
required-version = ">=0.10.9"
exclude-newer = "3 days"
[tool.uv.exclude-newer-package]
# TODO: ryan, remove after 2026-10-23, once certifi 2026.10.19 (drops the compromised eMudhra root) clears the 3-day window
exclude-newer-package = { certifi = "2026-10-20T03:00:00Z" }
certifi = "2026-10-20T03:00:00Z"
# TODO(shrey): remove after 2026-10-25, once grpcio 1.84.1 (drops the compromised eMudhra roots from roots.pem) clears the 3-day window
grpcio = "2026-10-22T15:00:00Z"
# TODO(shrey): remove after 2026-10-24, once boto3/botocore 1.43.117 (drops the compromised eMudhra roots from cacert.pem) clear the 3-day window
boto3 = "2026-10-21T19:31:00Z"
botocore = "2026-10-21T19:31:00Z"
# TODO: remove after 2026-10-25, once brotli 1.2.1 (CVE-2026-111497) clears the 3-day window
brotli = "2026-10-22T14:01:00Z"
# TODO: ryan, remove after 2026-10-24, once mlflow 3.17.2 (GHSA-wqmh-qccw-pqwf) clears the 3-day window
mlflow = "2026-10-21T03:00:00Z"
mlflow-skinny = "2026-10-21T03:00:00Z"
mlflow-tracing = "2026-10-21T03:00:00Z"
# TODO: ryan, remove after 2026-10-25, once pypdf 6.21.2 (GHSA-6c7f-9265-g8fw) clears the 3-day window
pypdf = "2026-10-21T17:00:00Z"
[tool.uv.sources]
litellm-proxy-extras = { workspace = true }

125
tests/integration/compatibility/test_status_code_followups_usage_chat.py
View file

@ -1,125 +0,0 @@
"""The /usage/ai/chat tools hand the model the reason the daily activity query rejected its dates, the same check
the daily activity routes answer 400 with, instead of a generic fetch error."""
from __future__ import annotations
import json
import uuid
from collections.abc import Iterator
from dataclasses import dataclass
from typing import Final
import pytest
from integration._support.client import JSON_OBJECT, Gateway, object_value
from integration._support.upstream import delete_scenario, register_scenario
from integration.compatibility._status_code_audit import Upstream, json_response, owned_gateway
from pydantic import JsonValue
pytestmark: Final = pytest.mark.timeout(180)
_TOOLS: Final = ("get_usage_data", "get_team_usage_data", "get_tag_usage_data")
@dataclass(frozen=True, slots=True)
class _UsageChat:
gateway: Gateway
identity: str
@pytest.fixture(scope="module")
def usage_chat(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_UsageChat]:
"""A proxy whose usage chat model (`openai/...`, called straight through litellm) is the scripted upstream."""
identity: Final = f"audit-usage-chat-{uuid.uuid4().hex}"
handle: Final = register_scenario(identity, json_response({}))
try:
with owned_gateway(
tmp_path_factory.mktemp("audit-usage-chat"),
{"model_list": [], "router_settings": {"num_retries": 0}},
environment={"OPENAI_API_BASE": handle.api_base(), "OPENAI_API_KEY": identity},
) as owned:
yield _UsageChat(owned, identity)
finally:
delete_scenario(handle)
def _tool_calls(start_date: str, end_date: str) -> dict[str, JsonValue]:
arguments: Final = json.dumps({"start_date": start_date, "end_date": end_date})
return {
"id": "chatcmpl-usage",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": f"call-{tool}", "type": "function", "function": {"name": tool, "arguments": arguments}}
for tool in _TOOLS
],
},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
}
def _tool_results(usage_chat: _UsageChat, start_date: str, end_date: str) -> tuple[dict[str, str], tuple[str, ...]]:
"""Runs one chat turn whose model calls every tool with these dates; returns the tool messages the follow-up
model call carried, keyed by tool, and the tool statuses the SSE stream reported."""
register_scenario(usage_chat.identity, json_response(_tool_calls(start_date, end_date)))
Upstream(usage_chat.gateway.upstream_url).reset()
with usage_chat.gateway.client.stream(
"POST",
"/usage/ai/chat",
json={"model": "openai/gpt-4o-mini", "messages": [{"role": "user", "content": "How much did we spend?"}]},
headers={"Authorization": f"Bearer {usage_chat.gateway.key}"},
timeout=60,
) as response:
assert response.status_code == 200
events: Final = tuple(
object_value(JSON_OBJECT.validate_json(line.removeprefix("data: ")))
for line in response.iter_lines()
if line.startswith("data: ")
)
calls: Final = Upstream(usage_chat.gateway.upstream_url).calls(usage_chat.identity)
assert len(calls) == 2, calls
messages: Final = object_value(calls[1]["body"]).get("messages")
assert isinstance(messages, list), calls[1]
tool_messages: Final = {
str(message["tool_call_id"]).removeprefix("call-"): str(message["content"])
for message in messages
if isinstance(message, dict) and message.get("role") == "tool"
}
statuses: Final = tuple(str(event.get("status")) for event in events if event.get("type") == "tool_call")
return tool_messages, statuses
_REVERSED: Final = "Invalid date range: end_date must be on or after start_date"
_NOT_CANONICAL: Final = "Invalid date range: start_date and end_date must be valid YYYY-MM-DD dates"
_NO_TEAM_DATA: Final = "No Team usage data found for the given date range."
_NO_TAG_DATA: Final = "No Tag usage data found for the given date range."
@pytest.mark.parametrize(
("start_date", "end_date", "team_message", "tag_message"),
(
pytest.param("2026-10-09", "2026-10-01", _REVERSED, _REVERSED, id="reversed"),
pytest.param("2026-13-01", "2026-10-09", _NOT_CANONICAL, _NOT_CANONICAL, id="invalid-month"),
pytest.param("2001/01/01", "2001-01-09", _NOT_CANONICAL, _NOT_CANONICAL, id="slashes"),
pytest.param("2001-01-01", "2001-01-09", _NO_TEAM_DATA, _NO_TAG_DATA, id="valid"),
),
)
def test_usage_chat_tools_answer_with_the_query_layers_date_check(
usage_chat: _UsageChat, start_date: str, end_date: str, team_message: str, tag_message: str
) -> None:
"""The team and tag tools give the model the daily activity routes' 400 reason; the global tool, whose query
takes these dates without that check, still answers with its usage summary."""
tool_messages, statuses = _tool_results(usage_chat, start_date, end_date)
assert tool_messages["get_usage_data"].startswith("Total Spend: $"), tool_messages
assert tool_messages["get_team_usage_data"] == team_message, tool_messages
assert tool_messages["get_tag_usage_data"] == tag_message, tool_messages
assert statuses == ("running", "complete") * len(_TOOLS), statuses

94
tests/integration/management/test_budget_list_sorting.py Normal file
View file

@ -0,0 +1,94 @@
from datetime import datetime, timezone
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.litellm_core_utils.duration_parser import duration_in_seconds_from
from tests.integration._support.client import Gateway, Scenario
@pytest.mark.parametrize("descending", (False, True))
def test_budget_reset_sort_orders_durations_before_pagination(
descending: bool,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
durations: Final = ("30d", "7d", "1h", "60m", "1mo", "weekly", "monthly", "٥d", "۵d", "५d", "５d", "𝟝d", "5d", None)
ids: Final = tuple(
scenario.budget(budget_id=f"{prefix}-{index}", budget_duration=duration, max_budget=10)
for index, duration in enumerate(durations)
)
now: Final = datetime.now(timezone.utc)
expected: Final = tuple(
budget_id
for _, budget_id in sorted(
zip(durations, ids, strict=True),
key=lambda item: (
item[0] is None,
duration_in_seconds_from(now, item[0]) * (-1 if descending else 1) if item[0] else 0,
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "-budget_duration" if descending else "budget_duration",
"page_size": "3",
"page": str(page),
},
)
for page in range(1, -(-len(ids) // 3) + 1)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected
filtered: Final = gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": "budget_duration",
"filter[budget_duration][in]": "7d,30d",
},
)
assert filtered.status_code == 200, filtered.text
assert [row["budget_duration"] for row in filtered.json()["data"]] == ["7d", "30d"]
@pytest.mark.parametrize("field", ("max_budget", "tpm_limit", "rpm_limit", "tpd_limit"))
@pytest.mark.parametrize("descending", (False, True))
def test_budget_limits_sort_unlimited_as_infinity(
field: str, descending: bool, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-unlimited-{uuid4().hex}"
values: Final = (20, None, 0, 3)
ids: Final = tuple(scenario.budget(budget_id=f"{prefix}-{i}", **{field: value}) for i, value in enumerate(values))
expected: Final = tuple(
identity
for value, identity in sorted(
zip(values, ids, strict=True),
key=lambda item: float("inf") if item[0] is None else item[0],
reverse=descending,
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/management/v1/budgets",
params={
"q": prefix,
"sort": ("-" if descending else "") + field,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["budget_id"] for row in chain.from_iterable(page.json()["data"] for page in pages)) == expected

80
tests/integration/management/test_key_list.py
View file

@ -1,11 +1,16 @@
from typing import FrozenSet
from datetime import datetime, timedelta, timezone
from itertools import chain
from typing import Final, FrozenSet
import pytest
from litellm.proxy.utils import hash_token
from tests.integration._support.actors import TEAM_ALPHA, Actor
from tests.integration._support.actors import TEAM_ALPHA, Actor, ManagementDatabase, ScratchResources
from tests.integration._support.actors import create_scratch_key
from tests.integration._support.client import Gateway, Scenario
from litellm.repositories.table_repositories import DeletedVerificationTokenRepository
from litellm.repositories.verification_token_repository import VerificationTokenRepository
pytestmark = pytest.mark.asyncio(loop_scope="session")
@ -202,3 +207,74 @@ async def test_key_list_non_admin_cannot_filter_other_team(gateway_client, actor
},
)
assert resp.status_code == 403, resp.text
@pytest.mark.parametrize(
"field", ("expires", "last_active", "budget_reset_at", "total_spend", "tpm_limit", "rpm_limit")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
@pytest.mark.parametrize("deleted", (False, True), ids=("live", "deleted"))
async def test_key_list_sorts_across_pages_with_nulls_last(
field: str,
direction: str,
deleted: bool,
gateway: Gateway,
scenario: Scenario,
db: ManagementDatabase,
resources: ScratchResources,
) -> None:
team_admin: Final = scenario.user(user_role="internal_user")
owner: Final = scenario.user(user_role="internal_user")
team_id: Final = scenario.team(
members_with_roles=[{"user_id": team_admin, "role": "admin"}, {"user_id": owner, "role": "user"}]
)
team_admin_key: Final = scenario.key(user_id=team_admin)
table: Final = (
DeletedVerificationTokenRepository(db.prisma).table if deleted else VerificationTokenRepository(db.prisma).table
)
values: Final = (20, 3, 3, 100, 0 if field == "total_spend" else None)
tokens: Final = tuple(hash_token(f"{resources.prefix}-{index}") for index in range(len(values)))
rows: Final = [
{
"token": token,
"key_alias": resources.prefix,
"team_id": team_id,
"user_id": owner,
field: (
datetime(2030, 1, 1, tzinfo=timezone.utc) + timedelta(days=value)
if field in ("expires", "last_active", "budget_reset_at") and value is not None
else value
),
}
for token, value in zip(tokens, values, strict=True)
]
await table.create_many(rows)
try:
valued: Final = tuple((value, token) for value, token in zip(values, tokens, strict=True) if value is not None)
expected: Final = [
token
for _, token in sorted(valued, key=lambda item: (item[0] if direction == "asc" else -item[0], item[1]))
] + [token for value, token in zip(values, tokens, strict=True) if value is None]
for caller in (gateway.key, team_admin_key):
pages: Final = [
gateway.client.get(
"/key/list",
headers={"Authorization": f"Bearer {caller}"},
params={
"key_alias": resources.prefix,
"team_id": team_id,
"include_team_keys": "true",
"sort_by": field,
"sort_order": direction,
"size": "2",
"page": str(page),
**({"status": "deleted"} if deleted else {}),
},
)
for page in (1, 2, 3)
]
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total_count"] == len(tokens) for page in pages)
assert list(chain.from_iterable(page.json()["keys"] for page in pages)) == expected
finally:
await table.delete_many(where={"token": {"in": tokens}})

39
tests/integration/management/test_sso_sign_in.py
View file

@ -3,7 +3,9 @@ import contextlib
import json
import uuid
from collections.abc import Iterator, Mapping
from concurrent.futures import ThreadPoolExecutor
from contextlib import contextmanager
from functools import partial
from pathlib import Path
from typing import Final
from urllib.parse import parse_qs, urlencode, urlparse
@ -15,7 +17,7 @@ from pydantic import JsonValue
from tests.integration._support.client import Gateway, gateway_from_environment
from tests.integration._support.database import read_rows
from tests.integration._support.process import owned_proxy
from tests.integration._support.process import OWNED_STATE_VARIABLE, OwnedState, owned_proxy
from tests.integration._support.wire import Reply, Request, wire_server
pytestmark: Final = [pytest.mark.timeout(300), pytest.mark.shares_worker_state]
@ -128,3 +130,38 @@ def test_a_first_sso_sign_in_creates_the_user_with_the_default_role(
"metadata": {"auth_provider": IDP_PROVIDER},
}
]
def test_free_sso_admission_preserves_existing_users_and_serializes_the_last_seat(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.setenv(OWNED_STATE_VARIABLE, OwnedState.PRIVATE.value)
monkeypatch.delenv("LITELLM_LICENSE", raising=False)
with _sso_gateway(tmp_path, {}) as gateway:
for index in range(4):
signed_in: Final = _sign_in(gateway, f"free-sso-{index}")
assert signed_in.status_code == 303, signed_in.text
contenders: Final = tuple(f"free-sso-contender-{index}" for index in range(8))
with ThreadPoolExecutor(max_workers=len(contenders)) as pool:
responses: Final = tuple(pool.map(partial(_sign_in, gateway), contenders))
assert sorted(response.status_code for response in responses) == [303, 403, 403, 403, 403, 403, 403, 403]
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 5}]
for subject, response in zip(contenders, responses):
if response.status_code == 403:
assert "requires a LiteLLM Enterprise license" in response.text
assert not response.cookies
assert gateway.rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (subject,)) == []
existing: Final = _sign_in(gateway, "free-sso-0")
assert existing.status_code == 303, existing.text
admin_created: Final = gateway.request(
"POST", "/user/new", {"user_id": "api-created-user", "auto_create_key": False}
)
assert admin_created.status_code == 200, admin_created.text
above_capacity: Final = _sign_in(gateway, "free-sso-0")
refused: Final = _sign_in(gateway, "free-sso-extra")
assert (above_capacity.status_code, refused.status_code) == (303, 403)
assert gateway.rows('SELECT COUNT(*) AS count FROM "LiteLLM_UserTable"', ()) == [{"count": 6}]

244
tests/integration/management/test_team_list_sorting.py Normal file
View file

@ -0,0 +1,244 @@
import json
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize(
"field", ("spend", "max_budget", "tpm_limit", "rpm_limit", "updated_at", "members", "models", "keys_count")
)
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_across_pages_and_respects_membership(
field: str,
direction: str,
gateway: Gateway,
scenario: Scenario,
) -> None:
prefix: Final = f"sort-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
caller: Final = scenario.key(user_id=owner)
values: Final = (20, 3, 3, 100, 0)
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{index}",
team_alias=f"{prefix}-{index}",
members_with_roles=[{"user_id": owner, "role": "admin"}],
)
for index in range(len(values))
)
outsider: Final = scenario.team(team_alias=f"{prefix}-outside")
for team_id, value in zip(teams, values, strict=True):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET spend = %s::float, max_budget = %s::float, '
"tpm_limit = %s::bigint, rpm_limit = %s::bigint, "
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day', "
"members_with_roles = %s::jsonb, models = ARRAY(SELECT generate_series(1, %s::int)::text) "
"WHERE team_id = %s",
(
str(value),
str(value),
str(value),
str(value),
str(value),
json.dumps(
[{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"} for i in range(value)]
),
str(value),
team_id,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team_id, team_id, str(value)),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET max_budget = NULL, tpm_limit = NULL, rpm_limit = NULL WHERE team_id = %s',
(teams[-1],),
)
try:
expected: Final = tuple(
team_id
for value, team_id in sorted(
zip(values, teams, strict=True),
key=lambda item: (
(item[0] != 0 if direction == "desc" else item[0] == 0)
if field in ("max_budget", "tpm_limit", "rpm_limit")
else False,
item[0] * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"search": prefix,
"user_id": owner,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
returned: Final = tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages))
assert returned == expected
assert outsider not in returned
assert all(page.json()["total"] == len(teams) for page in pages)
finally:
for team_id in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team_id,))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_team_list_sorts_organization_names_and_missing_values(
direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-{uuid4().hex}"
budget: Final = scenario.budget(max_budget=10)
organizations: Final = ((f"{prefix}-z", "alpha"), (f"{prefix}-a", "zulu"), (f"{prefix}-m", ""))
teams: Final = tuple(scenario.team(team_id=f"{prefix}-{i}", team_alias=f"{prefix}-{i}") for i in range(4))
try:
for (organization_id, alias), team_id in zip(organizations, teams):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(organization_id, alias, budget),
)
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s WHERE team_id = %s',
(organization_id, team_id),
)
ascending: Final = (teams[0], teams[2], teams[1])
ordered: Final = ascending if direction == "asc" else tuple(reversed(ascending))
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
params={
"search": prefix,
"sort_by": "organization_alias",
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == (*ordered, teams[-1])
finally:
for team_id in teams:
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team_id,))
for organization_id, _ in organizations:
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (organization_id,))
@pytest.mark.parametrize("field", ("organization_alias", "members", "models", "keys_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_team_sorts_preserve_org_and_membership_union(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-scope-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
teams: Final = tuple(
scenario.team(
team_id=f"{prefix}-{i}",
team_alias=f"{prefix}-{i}",
members_with_roles=[{"user_id": owner, "role": "admin"}] if i == 2 else [],
)
for i in range(4)
)
try:
for org, alias in zip(orgs, ("alpha", "zulu"), strict=True):
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, alias, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, team in enumerate(teams):
write_rows(
'UPDATE "LiteLLM_TeamTable" SET organization_id = %s, members_with_roles = %s::jsonb, '
"models = ARRAY(SELECT generate_series(1, %s::int)::text) WHERE team_id = %s",
(
orgs[0] if index < 2 else orgs[1],
json.dumps(
[
{"user_id": owner if i == 0 else f"{prefix}-member-{i}", "role": "admin"}
for i in range(index + 1)
]
),
str(index + 1),
team,
),
)
write_rows(
'INSERT INTO "LiteLLM_VerificationToken" (token, team_id) '
"SELECT %s || '-key-' || n::text, %s FROM generate_series(1, %s::int) n",
(team, team, str(index + 1)),
)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 0, 1) if field == "organization_alias" else (1, 2, 3)
expected: Final = tuple(
team
for value, team in sorted(
zip(values, teams[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"user_id": owner,
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
assert tuple(row["team_id"] for row in chain.from_iterable(page.json()["teams"] for page in pages)) == expected
denied: Final = gateway.request(
"GET",
"/v2/team/list",
key=caller,
params={
"organization_id": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for team in teams:
write_rows('DELETE FROM "LiteLLM_VerificationToken" WHERE team_id = %s', (team,))
write_rows('UPDATE "LiteLLM_TeamTable" SET organization_id = NULL WHERE team_id = %s', (team,))
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

179
tests/integration/management/test_user_list_sorting.py Normal file
View file

@ -0,0 +1,179 @@
from itertools import chain
from typing import Final
from uuid import uuid4
import pytest
from litellm.constants import UI_SESSION_TOKEN_TEAM_ID
from tests.integration._support.client import Gateway, Scenario
from tests.integration._support.database import write_rows
@pytest.mark.parametrize("field", ("user_alias", "max_budget", "updated_at", "sso_user_id", "status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_user_list_sorts_all_pages(field: str, direction: str, gateway: Gateway, scenario: Scenario) -> None:
prefix: Final = f"sort-{uuid4().hex}"
ids: Final = tuple(
scenario.user(
user_id=f"{prefix}-{index}",
user_role="internal_user",
user_alias=f"Alias {value if index != 2 else 4:03}",
user_email=f"{prefix}-{index}@example.com",
max_budget=value,
metadata={"scim_active": value != 3},
)
for index, value in enumerate((20, 3, 3, 100, 0))
)
for user_id, value in zip(ids, (20, 3, 3, 100, 0), strict=True):
write_rows(
'UPDATE "LiteLLM_UserTable" SET sso_user_id = %s, '
"updated_at = timestamp '2030-01-01' + %s::int * interval '1 day' WHERE user_id = %s",
(f"{prefix}-{value:03}", str(value), user_id),
) if value != 3 or user_id == ids[1] else write_rows(
"UPDATE \"LiteLLM_UserTable\" SET sso_user_id = %s, updated_at = timestamp '2030-01-04' WHERE user_id = %s",
(f"{prefix}-004", user_id),
)
for index in range(value if value < 4 else 1):
scenario.key(user_id=user_id, key_alias=f"{user_id}-key-{index}")
write_rows(
'UPDATE "LiteLLM_UserTable" SET max_budget = NULL, updated_at = NULL, sso_user_id = NULL WHERE user_id = %s',
(ids[-1],),
)
scenario.key(user_id=ids[-1])
write_rows(
'UPDATE "LiteLLM_VerificationToken" SET team_id = %s WHERE user_id = %s', (UI_SESSION_TOKEN_TEAM_ID, ids[-1])
)
values: Final = (
(0, 1, 1, 0, 0)
if field == "status"
else (1, 3, 3, 1, 0)
if field == "key_count"
else (20, 3, 4, 100, None)
if field == "sso_user_id"
else (20, 3, 4, 100, 0)
if field == "user_alias"
else (20, 3, 3, 100, None)
)
expected: Final = tuple(
user_id
for _, user_id in sorted(
((value, user_id) for value, user_id in zip(values, ids, strict=True)),
key=lambda item: (
(item[0] is not None if field == "max_budget" and direction == "desc" else item[0] is None),
(item[0] or 0) * (1 if direction == "asc" else -1),
item[1],
),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2, 3)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == len(ids) for page in pages)
assert tuple(user["user_id"] for user in chain.from_iterable(page.json()["users"] for page in pages)) == expected
@pytest.mark.parametrize("field", ("status", "key_count"))
def test_computed_user_sorts_preserve_filters(field: str, gateway: Gateway, scenario: Scenario) -> None:
team: Final = scenario.team()
member: Final = scenario.member(team)
other: Final = scenario.user(user_role="internal_user")
for filters in ({"team": team}, {"user_ids": member}, {"user_ids": f"{member},{other}"}, {"role": "internal_user"}):
baseline: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": "user_id"})
response: Final = gateway.get("/user/list", {"user_ids": f"{member},{other}", **filters, "sort_by": field})
assert response["total"] == baseline["total"]
assert sorted(user["user_id"] for user in response["users"]) == sorted(
user["user_id"] for user in baseline["users"]
)
ordinary_key: Final = scenario.key(user_id=other)
denied: Final = gateway.request("GET", "/user/list", key=ordinary_key, params={"sort_by": field})
assert denied.status_code == 403, denied.text
@pytest.mark.parametrize("field", ("status", "key_count"))
@pytest.mark.parametrize("direction", ("asc", "desc"))
def test_computed_user_sorts_preserve_org_admin_scope(
field: str, direction: str, gateway: Gateway, scenario: Scenario
) -> None:
prefix: Final = f"sort-org-users-{uuid4().hex}"
owner: Final = scenario.user(user_role="internal_user")
members: Final = tuple(
scenario.user(user_id=f"{prefix}-{index}", user_role="internal_user", metadata={"scim_active": index != 1})
for index in range(4)
)
budget: Final = scenario.budget(max_budget=10)
orgs: Final = (f"{prefix}-allowed", f"{prefix}-outside")
try:
for org in orgs:
write_rows(
'INSERT INTO "LiteLLM_OrganizationTable" '
"(organization_id, organization_alias, budget_id, created_by, updated_by, updated_at) "
"VALUES (%s, %s, %s, 'sorting-test', 'sorting-test', NOW())",
(org, org, budget),
)
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'org_admin')",
(owner, orgs[0]),
)
for index, member in enumerate(members):
write_rows(
'INSERT INTO "LiteLLM_OrganizationMembership" (user_id, organization_id, user_role) '
"VALUES (%s, %s, 'internal_user')",
(member, orgs[0] if index < 3 else orgs[1]),
)
for _ in range(index):
scenario.key(user_id=member)
caller: Final = scenario.key(user_id=owner)
values: Final = (0, 1, 0) if field == "status" else (0, 1, 2)
expected: Final = tuple(
member
for value, member in sorted(
zip(values, members[:3], strict=True),
key=lambda item: (item[0] * (-1 if direction == "desc" else 1), item[1]),
)
)
pages: Final = tuple(
gateway.request(
"GET",
"/user/list",
key=caller,
params={
"search": prefix,
"sort_by": field,
"sort_order": direction,
"page_size": "2",
"page": str(page),
},
)
for page in (1, 2)
)
assert all(page.status_code == 200 for page in pages), [page.text for page in pages]
assert all(page.json()["total"] == 3 for page in pages)
returned: Final = tuple(row["user_id"] for row in chain.from_iterable(page.json()["users"] for page in pages))
assert returned == expected
denied: Final = gateway.request(
"GET",
"/user/list",
key=caller,
params={
"organization_ids": orgs[1],
"sort_by": field,
},
)
assert denied.status_code == 403, denied.text
finally:
for org in orgs:
write_rows('DELETE FROM "LiteLLM_OrganizationMembership" WHERE organization_id = %s', (org,))
write_rows('DELETE FROM "LiteLLM_OrganizationTable" WHERE organization_id = %s', (org,))

180
tests/integration/security/test_offline_mode.py Normal file
View file

@ -0,0 +1,180 @@
from __future__ import annotations
import json
import uuid
from pathlib import Path
from typing import Final
import pytest
from integration._support.client import Gateway, object_value
from integration._support.forward_proxy import refusing_forward_proxy
from integration._support.process import owned_proxy
from pydantic import JsonValue
_PACKAGE: Final = Path(__file__).resolve().parents[3] / "litellm"
_CONTROL_HOST: Final = "provider.offline-mode.example"
_REMOTE_URLS: Final = {
"LITELLM_MODEL_COST_MAP_URL": "https://cost-map.offline-mode.example/model_prices.json",
"LITELLM_ANTHROPIC_BETA_HEADERS_URL": "https://beta-headers.offline-mode.example/config.json",
"LITELLM_AUTOROUTER_PRESETS_URL": "https://autorouter.offline-mode.example/presets.json",
"LITELLM_BLOG_POSTS_URL": "https://blog.offline-mode.example/rss.xml",
}
_SELF_INITIATED_HOSTS: Final = frozenset(
{
"cost-map.offline-mode.example:443",
"beta-headers.offline-mode.example:443",
"autorouter.offline-mode.example:443",
"blog.offline-mode.example:443",
"raw.githubusercontent.com:443",
"docs.litellm.ai:443",
"api.pwnedpasswords.com:443",
"huggingface.co:443",
}
)
_PER_FEATURE_FLAGS: Final = (
"LITELLM_OFFLINE_MODE",
"LITELLM_LOCAL_MODEL_COST_MAP",
"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS",
"LITELLM_LOCAL_AUTOROUTER_PRESETS",
"LITELLM_LOCAL_POLICY_TEMPLATES",
"LITELLM_LOCAL_BLOG_POSTS",
"LITELLM_DISABLE_LATEST_RELEASE_CHECK",
"LITELLM_LICENSE",
)
def _bundled(name: str) -> JsonValue:
return json.loads((_PACKAGE / name).read_text(encoding="utf-8"))
def _write_config(directory: Path) -> Path:
config: Final = directory / f"offline_mode_{uuid.uuid4().hex}.yaml"
config.write_text(
json.dumps(
{
"model_list": [
{
"model_name": "offline-mode-control",
"litellm_params": {
"model": "openai/gpt-4o-mini",
"api_base": f"https://{_CONTROL_HOST}/v1",
"api_key": "synthetic-provider-key",
},
},
{
"model_name": "offline-mode-kimi",
"litellm_params": {
"model": "bedrock/us.moonshotai.kimi-k3",
"aws_region_name": "us-east-1",
"aws_access_key_id": "synthetic-access-key",
"aws_secret_access_key": "synthetic-secret-key",
},
},
],
"router_settings": {"num_retries": 0},
"general_settings": {
"master_key": "os.environ/LITELLM_MASTER_KEY",
"database_url": "os.environ/DATABASE_URL",
},
}
)
)
return config
def _json(proxy: Gateway, method: str, path: str, body: dict[str, JsonValue] | None = None) -> JsonValue:
response: Final = proxy.request(method, path, body)
assert response.status_code == 200, f"{method} {path}: {response.status_code} {response.text}"
return json.loads(response.content)
@pytest.mark.timeout(300)
@pytest.mark.parametrize(
("offline_mode", "expected_hosts", "reload_status"),
[
pytest.param(None, _SELF_INITIATED_HOSTS | {f"{_CONTROL_HOST}:443"}, 502, id="unset"),
pytest.param("true", frozenset({f"{_CONTROL_HOST}:443"}), 200, id="true"),
],
)
def test_offline_mode_stops_every_self_initiated_request(
gateway: Gateway, tmp_path: Path, offline_mode: str | None, expected_hosts: frozenset[str], reload_status: int
) -> None:
with refusing_forward_proxy() as egress:
environment: Final = {
"HTTPS_PROXY": egress.url,
"HTTP_PROXY": egress.url,
"ALL_PROXY": egress.url,
"NO_PROXY": "127.0.0.1,localhost",
"AIOHTTP_TRUST_ENV": "True",
**_REMOTE_URLS,
**({"LITELLM_OFFLINE_MODE": offline_mode} if offline_mode is not None else {}),
}
with owned_proxy(
gateway,
tmp_path,
environment,
config=_write_config(tmp_path),
remove_environment=_PER_FEATURE_FLAGS,
workers=2,
) as isolated:
source: Final = object_value(_json(isolated, "GET", "/model/cost_map/source"))
reloaded: Final = isolated.request("POST", "/reload/model_cost_map")
beta_headers: Final = object_value(_json(isolated, "POST", "/reload/anthropic_beta_headers"))
release: Final = _json(isolated, "GET", "/get/latest_release_info")
presets: Final = object_value(_json(isolated, "GET", "/public/autorouter_presets"))
templates: Final = _json(isolated, "GET", "/policy/templates")
blog: Final = object_value(_json(isolated, "GET", "/public/litellm_blog_posts"))
user: Final = object_value(
_json(isolated, "POST", "/user/new", {"user_email": f"offline-{uuid.uuid4().hex}@example.com"})
)
password_set: Final = object_value(
_json(
isolated,
"POST",
"/user/update",
{"user_id": user["user_id"], "password": f"Offline-{uuid.uuid4()}"},
)
)
tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "meta-llama/llama-3-8b-instruct", "messages": [{"role": "user", "content": "count me"}]},
)
)
kimi_tokens: Final = object_value(
_json(
isolated,
"POST",
"/utils/token_counter",
{"model": "offline-mode-kimi", "messages": [{"role": "user", "content": "count me"}]},
)
)
control: Final = isolated.request(
"POST",
"/v1/chat/completions",
{"model": "offline-mode-control", "messages": [{"role": "user", "content": "control"}]},
)
reached: Final = frozenset(egress.targets())
assert reached == expected_hosts
assert control.status_code != 200, control.text
assert source["source"] == "local", source
assert source["is_env_forced"] is (offline_mode is not None), source
assert reloaded.status_code == reload_status, reloaded.text
assert beta_headers["status"] == "success", beta_headers
assert release is None
assert sorted(presets) == sorted(object_value(_bundled("proxy/public_endpoints/autorouter_presets.json")))
assert templates == _bundled("policy_templates_backup.json")
assert [object_value(post)["url"] for post in _list(blog["posts"])] == [
object_value(post)["url"] for post in _list(object_value(_bundled("blog_posts.json"))["posts"])
]
assert password_set["user_id"] == user["user_id"], password_set
assert tokens["tokenizer_type"] == "openai_tokenizer", tokens
assert isinstance(kimi_tokens["total_tokens"], int) and kimi_tokens["total_tokens"] > 0, kimi_tokens
def _list(value: JsonValue) -> list[JsonValue]:
assert isinstance(value, list), value
return value

104
tests/integration/spend/test_daily_activity_key_metadata_query_timeout.py
View file

@ -99,16 +99,9 @@ class _Activity(BaseModel):
results: tuple[_Day, ...]
class _UpstreamMessage(BaseModel):
model_config = ConfigDict(frozen=True)
role: str | None = None
class _UpstreamRequest(BaseModel):
model_config = ConfigDict(frozen=True)
stream: bool | None = None
tools: tuple[JsonValue, ...] | None = None
messages: tuple[_UpstreamMessage, ...] = ()
@dataclass(frozen=True, slots=True)
@ -171,17 +164,6 @@ def _stream_frames() -> tuple[bytes, ...]:
return (*(f"data: {json.dumps({**envelope, **frame})}\n\n".encode() for frame in frames), b"data: [DONE]\n\n")
def _usage_tool_call() -> Mapping[str, JsonValue]:
arguments: Final = json.dumps({"start_date": _day(-1), "end_date": _day(1)})
return {
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_usage", "type": "function", "function": {"name": "get_usage_data", "arguments": arguments}}
],
}
def _response_object() -> bytes:
return json.dumps(
{
@ -216,9 +198,6 @@ def _respond(request: Request) -> Reply:
body: Final = _UpstreamRequest.model_validate_json(request.body or b"{}")
if body.stream:
return Reply(content_type="text/event-stream", chunks=_stream_frames())
roles: Final = frozenset(message.role for message in body.messages)
if body.tools and "tool" not in roles:
return Reply(body=_completion(_usage_tool_call(), "tool_calls"))
return Reply(body=_completion({"role": "assistant", "content": REPLY_TEXT}, "stop"))
@ -459,12 +438,6 @@ def _restore(database_url: str, digest: str) -> None:
)
def _events(response: httpx.Response) -> tuple[Mapping[str, JsonValue], ...]:
return tuple(
json.loads(line.removeprefix("data: ")) for line in response.text.splitlines() if line.startswith("data: ")
)
@dataclass(frozen=True, slots=True)
class Tenant:
label: str
@ -714,39 +687,19 @@ def _rig(gateway: Gateway, directory: Path) -> Generator[tuple[Gateway, str]]:
@pytest.mark.timeout(360)
def test_usage_ai_chat_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
def test_usage_page_timeout_then_second_timeout_still_recovers_key_alias_after_database_frees(
gateway: Gateway, tmp_path: Path
) -> None:
with _rig(gateway, tmp_path) as (proxy, database_url):
spender: Final = _spender(proxy, database_url, "ai-chat")
spender: Final = _spender(proxy, database_url, "repeated-timeout")
with _pinned(proxy) as pinned:
with _locked_spend_logs(database_url):
with _recording(database_url) as during_chat:
chat: Final = pinned.request(
"POST",
"/usage/ai/chat",
body={
"messages": [{"role": "user", "content": "What did we spend?"}],
"model": "openai/gpt-4o-mini",
},
)
assert chat.status_code == 200, chat.text
assert chat.elapsed >= FAILED_LOOKUP_FLOOR, chat.elapsed
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
}
assert _events(chat) == (
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": REPLY_TEXT},
{"type": "done"},
), chat.text
assert len(_lookups(during_chat)) == 1
with _recording(database_url) as during_timeout:
timed_out: Final = _read(pinned, spender.digest)
assert timed_out.status_code == 200, timed_out.text
assert timed_out.elapsed >= FAILED_LOOKUP_FLOOR, timed_out.elapsed
assert _aliases(timed_out, spender.digest) == (None,), timed_out.text
assert len(_lookups(during_timeout)) == 1
cached_miss: Final = _read(pinned, spender.digest)
assert cached_miss.elapsed < FAILED_LOOKUP_FLOOR, cached_miss.elapsed
assert _aliases(cached_miss, spender.digest) == (None,), cached_miss.text
@ -1087,44 +1040,3 @@ def test_usage_page_rejects_bad_key_filters_and_unrelated_routes_ignore_a_locked
assert gateway_activity == 200, gateway_activity
chat: Final = proxy.chat(MODEL, text=f"locked {uuid.uuid4().hex}")
assert object_value(chat["usage"]) == dict(USAGE), chat
@pytest.mark.timeout(300)
def test_usage_ai_chat_survives_a_dropped_database_connection_during_alias_recovery(
gateway: Gateway, tmp_path: Path
) -> None:
with (
scratch_database() as database_url,
database_relay(database_url, b"AS " + LOOKUP_MARKER.encode()) as (relay, relayed_url),
wire_server(_respond) as wire,
_proxy(gateway, tmp_path, relayed_url, wire.url) as proxy,
):
spender: Final = _spender(proxy, database_url, "dropped-ai-chat")
with _pinned(proxy) as pinned:
relay.arm()
chat: Final = pinned.request(
"POST",
"/usage/ai/chat",
body={
"messages": [{"role": "user", "content": "What did we spend?"}],
"model": "openai/gpt-4o-mini",
},
)
assert chat.status_code == 200, chat.text
assert relay.tripped.is_set(), chat.text
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": _day(-1), "end_date": _day(1)},
}
assert _events(chat) == (
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": REPLY_TEXT},
{"type": "done"},
), chat.text
recovered: Final = _named_once_reconnected(pinned, spender)
assert _metadata(recovered, spender.digest) == _full_metadata(spender), recovered.text

128
tests/integration/spend/test_daily_activity_key_owner_traffic.py
View file

@ -6,7 +6,6 @@ from collections.abc import Iterable
from concurrent.futures import ThreadPoolExecutor
from datetime import UTC, datetime, timedelta
from hashlib import sha256
from pathlib import Path
from queue import SimpleQueue
from typing import Final
@ -31,8 +30,7 @@ from integration._support.daily_activity import (
user_row,
user_with_an_email,
)
from integration._support.database import read_rows, scratch_database
from integration._support.process import owned_proxy
from integration._support.database import read_rows
from integration._support.wire import Reply, Request, wire_server
from pydantic import JsonValue
@ -51,22 +49,6 @@ UNIFIED_ENDPOINTS: Final = ("/v1/chat/completions", "/v1/messages", "/v1/respons
REQUESTS_OF_A_BURST: Final = 21
READS_DURING_A_BURST: Final = 30
TOKEN_LIMIT_DISCOVERY: Final = ("GET", "/v1/models")
TOOL_CALL: Final = "call_integration_usage"
ANSWER: Final = "One request cost $0.25"
SUMMARY_OF_ONE_SEEDED_ROW: Final = "\n".join(
(
"Total Spend: $0.2500",
"Total Requests: 1",
"Successful: 1 | Failed: 0",
"Total Tokens: 15",
"",
"Top Models by Spend:",
" - gpt-4o-mini: $0.2500 (1 reqs, 15 tokens)",
"",
"Top Providers by Spend:",
" - openai: $0.2500 (1 reqs)",
)
)
def _chat_completion() -> dict[str, JsonValue]:
@ -105,55 +87,6 @@ def _provider(request: Request) -> Reply:
return Reply(body=json.dumps(body).encode())
def _usage_tool_call() -> dict[str, JsonValue]:
call: Final[dict[str, JsonValue]] = {
"id": TOOL_CALL,
"type": "function",
"function": {
"name": "get_usage_data",
"arguments": json.dumps({"start_date": DAY, "end_date": DAY}),
},
}
return {
"id": f"chatcmpl-{uuid.uuid4().hex}",
"object": "chat.completion",
"created": 1,
"model": "gpt-4o-mini",
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": None, "tool_calls": [call]},
"finish_reason": "tool_calls",
}
],
"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15},
}
def _streamed_chunk(delta: dict[str, JsonValue], finish_reason: str | None) -> bytes:
chunk: Final = {
"id": "chatcmpl-integration-usage",
"object": "chat.completion.chunk",
"created": 1,
"model": "gpt-4o-mini",
"choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}],
}
return f"data: {json.dumps(chunk)}\n\n".encode()
def _usage_analyst(request: Request) -> Reply:
if json.loads(request.body).get("stream"):
return Reply(
chunks=(
_streamed_chunk({"role": "assistant", "content": ANSWER}, None),
_streamed_chunk({}, "stop"),
b"data: [DONE]\n\n",
),
content_type="text/event-stream",
)
return Reply(body=json.dumps(_usage_tool_call()).encode())
def _sent_for_callers(requests: Iterable[Request]) -> tuple[Request, ...]:
return tuple(request for request in requests if (request.method, request.target) != TOKEN_LIMIT_DISCOVERY)
@ -322,65 +255,6 @@ def test_cli_session_spend_is_reported_with_the_user_and_team_of_the_session(
)
@pytest.mark.timeout(300)
def test_usage_ai_chat_hands_the_model_the_usage_summary_without_any_key_owner(
gateway: Gateway, tmp_path: Path
) -> None:
question: Final = f"what did we spend {uuid.uuid4().hex}"
owner: Final = f"integration-{uuid.uuid4().hex}"
ownerless_key: Final = f"integration-ownerless-{uuid.uuid4().hex}"
with (
scratch_database() as scratch_url,
wire_server(_usage_analyst) as wire,
owned_proxy(
gateway,
tmp_path,
{
"DATABASE_URL": scratch_url,
"OPENAI_API_BASE": f"{wire.url}/v1",
"OPENAI_BASE_URL": f"{wire.url}/v1",
"OPENAI_API_KEY": "integration-provider-key",
},
remove_environment=("DATABASE_URL_READ_REPLICA",),
) as candidate,
):
candidate.post("/user/new", {"user_id": owner, "user_email": f"{owner}@example.com", "auto_create_key": False})
with daily_rows((user_row(owner, ownerless_key, DAY),), database_url=scratch_url):
answer: Final = candidate.request(
"POST",
"/usage/ai/chat",
{"messages": [{"role": "user", "content": question}], "model": "openai/gpt-4o-mini"},
)
assert answer.status_code == 200, answer.text
tool_call: Final = {
"type": "tool_call",
"tool_name": "get_usage_data",
"tool_label": "global usage data",
"arguments": {"start_date": DAY, "end_date": DAY},
}
events: Final = [
json.loads(line.removeprefix("data: ")) for line in answer.text.splitlines() if line.startswith("data: ")
]
assert events == [
{"type": "status", "message": "Thinking..."},
{**tool_call, "status": "running"},
{**tool_call, "status": "complete"},
{"type": "status", "message": "Analyzing results..."},
{"type": "chunk", "content": ANSWER},
{"type": "done"},
], answer.text
asked, analysed = wire.drain()
assert [asked.target, analysed.target] == ["/v1/chat/completions", "/v1/chat/completions"]
assert json.loads(asked.body)["messages"][-1] == {"role": "user", "content": question}
assert json.loads(analysed.body)["messages"][-1] == {
"role": "tool",
"tool_call_id": TOOL_CALL,
"content": SUMMARY_OF_ONE_SEEDED_ROW,
}
assert owner not in analysed.body.decode()
assert ownerless_key not in analysed.body.decode()
@pytest.mark.timeout(300)
def test_owner_is_reported_on_every_route_while_a_burst_of_requests_waits_on_the_provider(gateway: Gateway) -> None:
released: Final = threading.Event()

5
tests/unit/AGENTS.md
View file

@ -1,6 +1,9 @@
# tests/unit
In-process. No network or subprocess
In-process. No network or subprocess. `interactions/test_openapi_compliance.py` reads the spec snapshot beside it,
and only the `interactions-spec-canary` workflow passes `--interactions-spec-url` (registered in
`interactions/conftest.py`, so name that file or directory on the command line) with `ai.google.dev` allowed, to run
the same tests against Google's live spec
Unit tests must not depend on the current time or execution speed. Use fixed timestamps for fixtures
and controlled clocks for time-dependent behavior. Incidental clock reads inside production code are allowed

127
tests/unit/integrations/otel/test_otel_v2_components.py
View file

@ -3,7 +3,6 @@ baggage helpers, metrics, the typed coercion helpers, mapper branches, span-name
builders, and the registry validator's failure paths. Needs the OTel SDK."""
from typing import Final
from requests_mock import Mocker
import contextlib
import json
import threading
@ -15,6 +14,7 @@ from http.server import BaseHTTPRequestHandler, HTTPServer, ThreadingHTTPServer
import pytest
import requests
from requests_mock import Adapter
pytest.importorskip("opentelemetry")
@ -23,14 +23,13 @@ from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ( # noqa:
)
from opentelemetry import baggage # noqa: E402
from opentelemetry.context import attach, detach # noqa: E402
from opentelemetry._logs import LogRecord # noqa: E402
from opentelemetry._logs.severity import SeverityNumber # noqa: E402
from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402
from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402
from opentelemetry._logs import LogRecord, SeverityNumber # noqa: E402 # optional SDK checked above
from opentelemetry.sdk._logs import ReadableLogRecord # noqa: E402 # optional SDK checked above
from opentelemetry.sdk._logs.export import LogRecordExportResult # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.resources import Resource # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.metrics import MeterProvider # noqa: E402
from opentelemetry.sdk.metrics.export import InMemoryMetricReader # noqa: E402
from opentelemetry.sdk.resources import Resource # noqa: E402
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider # noqa: E402 # optional SDK checked above
from opentelemetry.sdk.trace.export import ( # noqa: E402
BatchSpanProcessor,
ConsoleSpanExporter,
@ -41,7 +40,7 @@ from opentelemetry.sdk.trace.export.in_memory_span_exporter import ( # noqa: E4
InMemorySpanExporter,
)
from opentelemetry.sdk.util.instrumentation import InstrumentationScope # noqa: E402
from opentelemetry.trace import SpanContext, SpanKind, TraceFlags, get_current_span # noqa: E402
from opentelemetry.trace import SpanKind, TraceFlags, get_current_span # noqa: E402
from opentelemetry.trace.propagation.tracecontext import ( # noqa: E402
TraceContextTextMapPropagator,
)
@ -50,6 +49,7 @@ import litellm # noqa: E402
from tests.unit.integrations.conftest import TlsSink # noqa: E402
from litellm.integrations.otel.plumbing import context as ctx_mod # noqa: E402
from litellm.integrations.otel.plumbing import providers # noqa: E402
from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter # noqa: E402 # optional SDK checked above
from litellm.integrations.otel.model.config import OpenTelemetryV2Config # noqa: E402
from litellm.integrations.otel.mappers.genai import GenAIMapper # noqa: E402
from litellm.integrations.otel.mappers.legacy import LegacyMapper # noqa: E402
@ -737,7 +737,9 @@ def _export_one_trace_to_local_collector(exporter_kind: str) -> tuple[list[dict]
class Collector(BaseHTTPRequestHandler):
def do_POST(self):
body = self.rfile.read(int(self.headers["Content-Length"]))
received.append({"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body})
received.append(
{"path": self.path, "headers": requests.structures.CaseInsensitiveDict(self.headers), "body": body}
)
self.send_response(200)
self.end_headers()
@ -797,28 +799,93 @@ def test_http_json_exporter_posts_otlp_json_to_traces_endpoint():
assert child["parentSpanId"] == format(parent_id, "016x")
@pytest.mark.parametrize("failure_mode", ("http", "shutdown", "size", "encoding"))
def test_json_export_failures_do_not_deliver_a_success(failure_mode: str, requests_mock: Mocker) -> None:
from litellm.integrations.otel.plumbing.otlp_json import OTLPJsonSpanExporter
@pytest.mark.parametrize(
("statuses", "expected"),
[
((200,), SpanExportResult.SUCCESS),
((400,), SpanExportResult.FAILURE),
((503, 200), SpanExportResult.SUCCESS),
],
)
def test_http_json_exporter_preserves_http_failure_and_retry_results(
statuses: tuple[int, ...], expected: SpanExportResult
) -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri(
"POST",
endpoint,
[{"status_code": status, "headers": {"Retry-After": "0"}} for status in statuses],
)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(
endpoint=endpoint,
headers={
"CONTENT-TYPE": "application/x-protobuf",
"x-collector-token": "secret",
},
session=session,
)
try:
assert exporter.export(()) is expected
finally:
exporter.shutdown()
assert len(adapter.request_history) == len(statuses)
for request in adapter.request_history:
assert request.headers["Content-Type"] == "application/json"
assert request.headers["x-collector-token"] == "secret"
assert request.json() == {"resourceSpans": []}
endpoint: Final = "http://collector.test/v1/traces"
collector: Final = requests_mock.post(endpoint, status_code=400 if failure_mode == "http" else 200)
exporter: Final = OTLPJsonSpanExporter(
endpoint=endpoint, headers={}, max_request_size=1 if failure_mode == "size" else None
)
span: Final = (
ReadableSpan("malformed")
if failure_mode == "encoding"
else ReadableSpan("json-export", context=SpanContext(1, 2, False), resource=Resource({}), start_time=1, end_time=2)
)
try:
if failure_mode == "shutdown":
@pytest.mark.parametrize(
("size_limit", "expected", "request_count"),
[
(0, SpanExportResult.SUCCESS, 1),
(len(b'{"resourceSpans":[]}'), SpanExportResult.SUCCESS, 1),
(len(b'{"resourceSpans":[]}') - 1, SpanExportResult.FAILURE, 0),
],
)
def test_http_json_exporter_checks_json_size_before_sending(
size_limit: int, expected: SpanExportResult, request_count: int
) -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session, max_request_size=size_limit)
try:
assert exporter.export(()) is expected
finally:
exporter.shutdown()
assert exporter.export([span]) is SpanExportResult.FAILURE
assert collector.call_count == int(failure_mode == "http")
finally:
if failure_mode != "shutdown":
assert len(adapter.request_history) == request_count
def test_http_json_exporter_rejects_export_after_shutdown() -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
exporter.shutdown()
assert exporter.export(()) is SpanExportResult.FAILURE
assert adapter.request_history == []
def test_http_json_exporter_returns_failure_for_unencodable_spans() -> None:
adapter: Final = Adapter()
endpoint: Final = "https://collector.test/v1/traces"
adapter.register_uri("POST", endpoint, status_code=200)
with requests.Session() as session:
session.mount("https://", adapter)
exporter: Final = OTLPJsonSpanExporter(endpoint, {}, session=session)
try:
assert exporter.export((ReadableSpan("missing-context"),)) is SpanExportResult.FAILURE
finally:
exporter.shutdown()
assert adapter.request_history == []
def test_http_protobuf_exporter_still_posts_protobuf():
@ -1659,7 +1726,9 @@ def test_v2_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.Monkey
body="v2-tls-test",
)
log_data: Final = ReadableLogRecord(
log_record=record, resource=Resource({}), instrumentation_scope=InstrumentationScope("v2-tls-test")
log_record=record,
resource=Resource({}),
instrumentation_scope=InstrumentationScope("v2-tls-test"),
)
result = exporter.export([log_data])
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"

14
tests/unit/integrations/otel/test_otel_v2_mount.py
View file

@ -7,7 +7,6 @@ surface and the server-span + shared-provider behavior it produces.
import contextvars
from datetime import datetime, timezone
from typing import Final
import pytest
@ -383,7 +382,9 @@ def test_every_traced_response_names_its_own_server_span(monkeypatch):
assert responses["/stream"].text == "data: first\n\ndata: [DONE]\n\n"
for path, response in responses.items():
context = server_spans[path].context
assert response.headers.get_list("traceresponse") == [f"00-{context.trace_id:032x}-{context.span_id:016x}-{int(context.trace_flags):02x}"]
assert response.headers.get_list("traceresponse") == [
f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"
]
@pytest.mark.parametrize("path", ["/relayed", "/relayed/health"])
@ -475,9 +476,10 @@ def test_gate_off_responses_carry_no_traceresponse(monkeypatch):
def test_failed_instrumentation_registers_no_response_propagator(monkeypatch):
monkeypatch.setenv("LITELLM_OTEL_V2", "1")
monkeypatch.setenv("OTEL_PYTHON_FASTAPI_EXCLUDED_URLS", "[")
app: Final = fastapi.FastAPI()
instrument_fastapi_app(app)
app = fastapi.FastAPI()
with TestClient(app):
instrument_fastapi_app(app)
assert get_global_response_propagator() is None
@ -545,4 +547,4 @@ def test_the_unhandled_error_helper_names_the_span_it_is_given_in_any_context(mo
headers = contextvars.Context().run(trace_response_headers, span)
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{int(context.trace_flags):02x}"
assert headers["traceresponse"] == f"00-{context.trace_id:032x}-{context.span_id:016x}-{context.trace_flags:02x}"

11
tests/unit/integrations/otel/test_otel_v2_presets.py
View file

@ -2,6 +2,7 @@
event loop: the preset does no network I/O, and a custom exporter mints the JWT
lazily on its first export (in the BatchSpanProcessor worker thread)."""
import json
from typing import Final
import httpx
@ -101,6 +102,7 @@ def test_agentops_exporter_mints_jwt_lazily(
assert exporter.export([]) is SpanExportResult.SUCCESS
assert exporter.export([]) is SpanExportResult.SUCCESS
assert auth.call_count == 1
assert [json.loads(call.request.content) for call in auth.calls] == [{"api_key": "ak"}]
assert [request.headers.get("Authorization") for request in collector.request_history] == [
"Bearer jwt-xyz",
"Bearer jwt-xyz",
@ -109,17 +111,20 @@ def test_agentops_exporter_mints_jwt_lazily(
exporter.shutdown()
@pytest.mark.parametrize(("status", "payload"), ((500, {}), (200, {}), (200, {"token": 123})))
@pytest.mark.parametrize(("status", "payload"), ((500, {}), (200, {}), (200, {"token": 123}), (None, {})))
def test_agentops_exporter_tolerates_fetch_failure(
requests_mock: Mocker,
respx_mock: respx.MockRouter,
monkeypatch: pytest.MonkeyPatch,
status: int,
status: int | None,
payload: dict[str, object],
) -> None:
monkeypatch.delenv("OTEL_EXPORTER_OTLP_HEADERS", raising=False)
monkeypatch.delenv("OTEL_EXPORTER_OTLP_TRACES_HEADERS", raising=False)
auth: Final = respx_mock.post(agentops_mod._AGENTOPS_AUTH_ENDPOINT).respond(status, json=payload)
route: Final = respx_mock.post(agentops_mod._AGENTOPS_AUTH_ENDPOINT)
auth: Final = (
route.mock(side_effect=httpx.ConnectError("auth down")) if status is None else route.respond(status, json=payload)
)
collector: Final = requests_mock.post(_AGENTOPS_ENDPOINT, status_code=200)
exporter: Final = _build_agentops_exporter(
ExporterSpec(kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, options={"api_key": "ak"})

48
tests/unit/integrations/test_langfuse_otel.py
View file

@ -7,8 +7,6 @@ from unittest.mock import MagicMock, patch
import pytest
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import SpanExportResult
from requests_mock import Mocker
from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
@ -515,6 +513,22 @@ class TestLangfuseOtelIntegration:
# Endpoint assertion removed as side effect is gone
def _exported_auth_headers(exporter):
import requests_mock
from opentelemetry.sdk.trace.export import SpanExportResult
with requests_mock.Mocker() as collector:
collector.post(requests_mock.ANY, status_code=200)
assert exporter.export(()) is SpanExportResult.SUCCESS
return [
(
request.url,
{name: request.headers[name] for name in ("Authorization", "x-langfuse-ingestion-version")},
)
for request in collector.request_history
]
class TestLangfuseOtelKeyDynamicConfig:
"""Key/team-scoped Langfuse credentials must define the full export target
(OTLP endpoint + auth), not just auth headers on the init-time exporter."""
@ -593,7 +607,7 @@ class TestLangfuseOtelKeyDynamicConfig:
assert logger.construct_dynamic_otel_config(self._dynamic_params(langfuse_secret_key=None)) is None
assert logger.construct_dynamic_otel_config(self._dynamic_params(langfuse_public_key=None)) is None
def test_key_dynamic_params_create_otlp_exporter_without_global_env(self, requests_mock: Mocker) -> None:
def test_key_dynamic_params_create_otlp_exporter_without_global_env(self):
"""Without global LANGFUSE_* env vars, a request carrying key-scoped Langfuse
credentials must get a tracer exporting via OTLP HTTP to that key's host."""
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
@ -619,17 +633,15 @@ class TestLangfuseOtelKeyDynamicConfig:
exporter = span_processors[0].span_exporter
assert isinstance(exporter, OTLPSpanExporter)
assert exporter._endpoint == "https://langfuse.example.com/api/public/otel/v1/traces"
import base64
expected_auth = base64.b64encode(b"key_public:key_secret").decode()
collector: Final = requests_mock.post("https://langfuse.example.com/api/public/otel/v1/traces", status_code=200)
assert exporter.export([]) is SpanExportResult.SUCCESS
assert collector.call_count == 1
assert {
key: collector.last_request.headers[key] for key in ("Authorization", "x-langfuse-ingestion-version")
} == {"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"}
assert _exported_auth_headers(exporter) == [
(
"https://langfuse.example.com/api/public/otel/v1/traces",
{"Authorization": f"Basic {expected_auth}", "x-langfuse-ingestion-version": "4"},
)
]
def test_key_dynamic_params_reuse_cached_provider(self):
with self._clean_env():
@ -648,7 +660,7 @@ class TestLangfuseOtelKeyDynamicConfig:
assert tracer is logger.tracer
assert logger._tracer_provider_cache == {}
def test_key_credentials_never_passed_to_debug_logger(self, requests_mock: Mocker) -> None:
def test_key_credentials_never_passed_to_debug_logger(self):
"""The span-processor debug logs must receive a redacted header value, so the
key-scoped Langfuse secret never enters a log record regardless of downstream
handler configuration, while the exporter still gets the real header."""
@ -682,12 +694,12 @@ class TestLangfuseOtelKeyDynamicConfig:
provider = next(iter(logger._tracer_provider_cache.values())).provider
exporter = provider._active_span_processor._span_processors[0].span_exporter
assert isinstance(exporter, OTLPSpanExporter)
collector: Final = requests_mock.post("https://langfuse.example.com/api/public/otel/v1/traces", status_code=200)
assert exporter.export([]) is SpanExportResult.SUCCESS
assert collector.call_count == 1
assert {
key: collector.last_request.headers[key] for key in ("Authorization", "x-langfuse-ingestion-version")
} == {"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"}
assert _exported_auth_headers(exporter) == [
(
"https://langfuse.example.com/api/public/otel/v1/traces",
{"Authorization": f"Basic {secret}", "x-langfuse-ingestion-version": "4"},
)
]
class TestLangfuseOtelResponsesAPI:

18
tests/unit/integrations/test_opentelemetry.py
View file

@ -2324,7 +2324,9 @@ def test_otlp_http_log_export_trusts_ssl_cert_file(monkeypatch: pytest.MonkeyPat
body="tls-export-test",
)
log_data: Final = ReadableLogRecord(
log_record=record, resource=Resource({}), instrumentation_scope=InstrumentationScope("tls-export-test")
log_record=record,
resource=Resource({}),
instrumentation_scope=InstrumentationScope("tls-export-test"),
)
result: Final = exporter.export([log_data])
assert result is LogRecordExportResult.SUCCESS, f"log export failed: {result}"
@ -6717,6 +6719,13 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
must be bounded and must shut down whatever it drops (LIT-5437: threads accumulated
until pods were OOMKilled)."""
def setUp(self) -> None:
processor: Final = BatchSpanProcessor(InMemorySpanExporter())
try:
self.bsp_thread_name = processor._batch_processor._worker_thread.name
finally:
processor.shutdown()
def _logger(self, cap=3, exporter="console"):
logger = OpenTelemetry(
config=OpenTelemetryConfig(exporter=exporter, skip_set_global=True),
@ -6732,12 +6741,7 @@ class TestDynamicTracerProviderCache(unittest.TestCase):
logger._tracer_provider_cache.clear()
def _live_exporter_threads(self):
probe: Final = BatchSpanProcessor(InMemorySpanExporter())
try:
worker_name: Final = probe._batch_processor._worker_thread.name
finally:
probe.shutdown()
return [t for t in threading.enumerate() if t.name == worker_name]
return [t for t in threading.enumerate() if t.name == self.bsp_thread_name]
def _wait_for_exporter_threads(self, expected, timeout=10.0):
"""Dropped providers are shut down off-thread, so poll instead of sleeping."""

9
tests/unit/interactions/conftest.py Normal file
View file

@ -0,0 +1,9 @@
import pytest
def pytest_addoption(parser: pytest.Parser) -> None:
parser.addoption(
"--interactions-spec-url",
default=None,
help="Run the Interactions OpenAPI compliance tests against this live spec instead of the vendored snapshot",
)

10024
tests/unit/interactions/interactions.openapi.json Normal file
View file

File diff suppressed because it is too large Load diff

102
tests/unit/interactions/test_openapi_compliance.py
View file

@ -1,43 +1,56 @@
"""
OpenAPI compliance tests for Google Interactions API.
Validates that our SDK requests/responses match the OpenAPI spec at:
https://ai.google.dev/static/api/interactions.openapi.json
Validates that our SDK requests/responses match the OpenAPI spec snapshot in
interactions.openapi.json, taken from https://ai.google.dev/static/api/interactions.openapi.json
(litellm/types/interactions/README.md says how to refresh it). The scheduled
interactions-spec-canary workflow runs the same tests against the live spec by passing
--interactions-spec-url, so a change Google makes shows up there, not in every PR.
Run with: pytest tests/unit/interactions/test_openapi_compliance.py -v
"""
import json
import os
import re
from typing import Any, Dict, Final
from dataclasses import dataclass
from enum import Enum
from pathlib import Path
from typing import Any, Dict, Final, get_args
from unittest.mock import MagicMock, patch
import httpx
import pytest
from openapi_core import OpenAPI
from litellm.types.interactions import InteractionStatus
from pydantic import BaseModel, TypeAdapter
OPENAPI_SPEC_URL = "https://ai.google.dev/static/api/interactions.openapi.json"
OPENAPI_SPEC_SNAPSHOT: Final = Path(__file__).with_name("interactions.openapi.json")
SPEC_URL_OPTION: Final = TypeAdapter(str | None)
def _load_openapi_spec_dict() -> Dict[str, Any]:
"""
Load the OpenAPI spec JSON.
@dataclass(frozen=True, slots=True)
class LoadedSpec:
source: str
spec: Dict[str, Any]
In CI or offline environments, network access may not be available.
In that case, gracefully skip these tests instead of erroring.
"""
try:
response = httpx.get(OPENAPI_SPEC_URL, timeout=5.0)
response.raise_for_status()
return response.json()
except Exception as e: # pragma: no cover - defensive, env-dependent
pytest.skip(
f"Skipping Google Interactions OpenAPI compliance tests - "
f"unable to load spec from {OPENAPI_SPEC_URL}: {e}"
)
def _status_enum_of(model: type[BaseModel]) -> type[Enum]:
annotation: Final = model.model_fields["status"].annotation
candidates: Final = (annotation, *get_args(annotation))
return next(candidate for candidate in candidates if isinstance(candidate, type) and issubclass(candidate, Enum))
def _requested_spec_url(request: pytest.FixtureRequest) -> str | None:
return SPEC_URL_OPTION.validate_python(request.config.getoption("interactions_spec_url"))
def _load_openapi_spec(live_url: str | None) -> LoadedSpec:
if live_url is None:
return LoadedSpec(source=str(OPENAPI_SPEC_SNAPSHOT), spec=json.loads(OPENAPI_SPEC_SNAPSHOT.read_text()))
with httpx.Client(transport=httpx.HTTPTransport(retries=3), timeout=30.0) as client:
response: Final = client.get(live_url)
response.raise_for_status()
return LoadedSpec(source=str(response.url), spec=response.json())
def _model_create_request_schema(spec_dict: Dict[str, Any]) -> Dict[str, Any]:
@ -67,9 +80,13 @@ def _declared_type_value(variant_schema: Dict[str, Any]) -> Any:
@pytest.fixture(scope="module")
def spec_dict() -> Dict[str, Any]:
"""Load raw spec dict for manual validation."""
return _load_openapi_spec_dict()
def loaded_spec(request: pytest.FixtureRequest) -> LoadedSpec:
return _load_openapi_spec(_requested_spec_url(request))
@pytest.fixture(scope="module")
def spec_dict(loaded_spec: LoadedSpec) -> Dict[str, Any]:
return loaded_spec.spec
@pytest.fixture(scope="module")
@ -78,6 +95,18 @@ def openapi_spec(spec_dict: Dict[str, Any]) -> OpenAPI:
return OpenAPI.from_dict(spec_dict)
def test_spec_comes_from_the_requested_source(loaded_spec: LoadedSpec, request: pytest.FixtureRequest) -> None:
assert loaded_spec.source == (_requested_spec_url(request) or str(OPENAPI_SPEC_SNAPSHOT))
def test_interaction_status_export_carries_the_interaction_models_statuses() -> None:
from litellm.types.interactions import InteractionStatus, generated
exported: Final = frozenset(member.value for member in InteractionStatus)
modelled: Final = frozenset(member.value for member in _status_enum_of(generated.Interaction))
assert exported == modelled, sorted(exported ^ modelled)
class TestRequestCompliance:
"""Tests that our request bodies match the OpenAPI spec."""
@ -251,15 +280,24 @@ class TestResponseCompliance:
assert field in schema["properties"], f"Output field '{field}' not in spec"
print(f"✓ Output field '{field}' exists in spec")
def test_response_status_enum_supports_advertised_values(self, spec_dict):
schema: Final = spec_dict["components"]["schemas"]["Interaction"]
advertised_statuses: Final = frozenset(schema["properties"]["status"]["enum"])
supported_statuses: Final = frozenset(status.value for status in InteractionStatus)
@pytest.mark.parametrize(
"schema_name",
[
"Interaction",
"InteractionStatusUpdate",
"CreateModelInteractionParams",
"CreateAgentInteractionParams",
"UrlContextResult",
],
)
def test_status_enum_values(self, spec_dict, schema_name):
from litellm.types.interactions import generated
assert advertised_statuses
assert advertised_statuses <= supported_statuses, (
f"Unsupported interaction statuses: {sorted(advertised_statuses - supported_statuses)}"
)
schema = spec_dict["components"]["schemas"][schema_name]
spec_statuses: Final = frozenset(schema["properties"]["status"]["enum"])
parsed_statuses: Final = frozenset(member.value for member in _status_enum_of(getattr(generated, schema_name)))
assert spec_statuses
assert spec_statuses <= parsed_statuses, sorted(spec_statuses - parsed_statuses)
def test_usage_schema(self, spec_dict):
"""Verify Usage schema fields."""

6
tests/unit/litellm_core_utils/test_exception_mapping_utils.py
View file

@ -160,6 +160,12 @@ def test_api_base_line_keeps_an_at_sign_in_the_path(api_base: str) -> None:
# Tuple format: (error_message, expected_result)
context_window_test_cases = [
# Positive cases (should return True)
(
'"{\n "error": {\n "message": "Your input exceeds the context window of this model. '
'Please adjust your input and try again.",\n "type": "invalid_request_error",\n'
' "param": "input",\n "code": "context_length_exceeded"\n }\n}"',
True,
),
(
"An error occurred: The input exceeds the model's maximum context limit of 8192 tokens.",
True,

9
tests/unit/litellm_core_utils/test_get_llm_provider_logic.py
View file

@ -8,6 +8,7 @@ from litellm import CustomLLM
from litellm.litellm_core_utils import get_llm_provider_logic
from litellm.litellm_core_utils.get_llm_provider_logic import (
get_llm_provider,
get_provider_affinity_header,
inferred_provider,
is_registered_custom_provider,
)
@ -706,3 +707,11 @@ class TestClaudeModelPatternMatching:
set_fallback_generalizations([])
with pytest.raises(litellm.BadRequestError):
litellm.get_llm_provider(model="claude-opus-4-9")
@pytest.mark.parametrize("prefix", ["openai", "anthropic", "custom_openai", "openai_like"])
def test_opencode_affinity_resolution_preserves_existing_transport(prefix: str) -> None:
api_base: Final = "https://opencode.ai/zen/go/v1"
model, provider, _, destination = get_llm_provider(model=f"{prefix}/test-model", api_base=api_base)
assert (model, provider, destination) == ("test-model", prefix, api_base)
assert get_provider_affinity_header(destination) == "x-opencode-session"

14
tests/unit/litellm_core_utils/test_get_model_cost_map.py
View file

@ -399,9 +399,10 @@ async def test_refetch_shrunk_map_fails_integrity_not_swapped_in():
@pytest.mark.asyncio
async def test_refetch_respects_local_env_override(monkeypatch):
"""LITELLM_LOCAL_MODEL_COST_MAP=True short-circuits to the bundled backup, zero HTTP."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
async def test_refetch_respects_local_env_override(monkeypatch, flag):
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True short-circuits to the bundled backup, zero HTTP."""
monkeypatch.setenv(flag, "True")
def _fail(request):
raise AssertionError("no HTTP request should be made when local map is forced")
@ -708,9 +709,10 @@ def test_boot_load_falls_back_when_urlsplit_rejects_the_url():
assert "invalid URL" in (source["fallback_reason"] or "")
def test_boot_load_respects_local_env_override(monkeypatch):
"""LITELLM_LOCAL_MODEL_COST_MAP=True still short-circuits to the backup with zero HTTP."""
monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
@pytest.mark.parametrize("flag", ["LITELLM_LOCAL_MODEL_COST_MAP", "LITELLM_OFFLINE_MODE"])
def test_boot_load_respects_local_env_override(monkeypatch, flag):
"""LITELLM_LOCAL_MODEL_COST_MAP=True or LITELLM_OFFLINE_MODE=True still short-circuits to the backup with zero HTTP."""
monkeypatch.setenv(flag, "True")
def _fail(request):
raise AssertionError("no HTTP request should be made when local map is forced")

16
tests/unit/litellm_core_utils/test_litellm_logging.py
View file

@ -21,7 +21,6 @@ from unittest.mock import AsyncMock, MagicMock, patch
import httpx
import pytest
import pytest_asyncio
from requests_mock import Mocker
from mcp.types import AudioContent, CallToolResult, ImageContent, TextContent
from openai import AsyncOpenAI
from openai._legacy_response import HttpxBinaryResponseContent
@ -2045,10 +2044,11 @@ async def test_logfire_logger_accepts_env_vars_for_base_url(monkeypatch):
],
)
def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
monkeypatch: pytest.MonkeyPatch, api_host: str | None, expected_endpoint: str, requests_mock: Mocker
monkeypatch: pytest.MonkeyPatch, api_host: str | None, expected_endpoint: str
) -> None:
"""The exporter must post to Langtrace's complete /api/trace path with the key in x-api-key,
without leaking it into the process-wide OTEL_EXPORTER_OTLP_TRACES_HEADERS."""
import requests_mock
from opentelemetry.exporter.otlp.proto.http.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace.export import SpanExportResult
@ -2072,10 +2072,12 @@ def test_langtrace_callback_exports_to_api_trace_with_x_api_key(
assert type(logger) is OpenTelemetry and logger.callback_name == "langtrace"
exporter: Final = logger._get_span_processor().span_exporter
assert isinstance(exporter, OTLPSpanExporter)
collector: Final = requests_mock.post(expected_endpoint, status_code=200)
assert exporter.export([]) is SpanExportResult.SUCCESS
assert collector.call_count == 1
assert collector.last_request.headers["x-api-key"] == api_key
with requests_mock.Mocker() as collector:
collector.post(expected_endpoint, status_code=200)
assert exporter.export(()) is SpanExportResult.SUCCESS
assert [(request.url, request.headers.get("x-api-key")) for request in collector.request_history] == [
(expected_endpoint, api_key)
]
assert "OTEL_EXPORTER_OTLP_TRACES_HEADERS" not in os.environ
finally:
logging_module._in_memory_loggers.clear()
@ -2228,7 +2230,7 @@ async def test_aresponses_websocket_turn_reaches_a_custom_logger_through_the_asy
async def test_aresponses_websocket_hands_back_the_provider_failure_without_a_success_log(
monkeypatch: pytest.MonkeyPatch,
):
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER, LoggingWorker
from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER
from litellm.responses.main import base_llm_http_handler
success_events = []

17
tests/unit/litellm_core_utils/test_offline_mode.py Normal file
View file

@ -0,0 +1,17 @@
import pytest
from litellm.litellm_core_utils.offline_mode import is_offline_mode
@pytest.mark.parametrize(
("value", "expected"),
[("true", True), ("True", True), (" TRUE ", True), ("false", False), ("", False), ("1", False), ("yes", False)],
)
def test_is_offline_mode_reads_true_case_insensitively(monkeypatch, value, expected):
monkeypatch.setenv("LITELLM_OFFLINE_MODE", value)
assert is_offline_mode() is expected
def test_is_offline_mode_is_off_when_unset(monkeypatch):
monkeypatch.delenv("LITELLM_OFFLINE_MODE", raising=False)
assert is_offline_mode() is False

21
tests/unit/litellm_core_utils/test_token_counter.py
View file

@ -1173,6 +1173,27 @@ def test_disable_hf_tokenizer_download_keeps_deepseek_v4_on_cl100k(monkeypatch:
assert counted == litellm.token_counter(model="", text=_CODE_SAMPLE)
@pytest.mark.parametrize("model", ["llama-2-7b", "meta-llama/llama-3-8b-instruct"])
def test_offline_mode_counts_downloaded_tokenizer_models_with_tiktoken(monkeypatch: pytest.MonkeyPatch, model: str):
_load_huggingface_tokenizer.cache_clear()
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
with patch("litellm.utils.tokenizer_dispatch.from_pretrained") as from_pretrained:
result: Final = _select_tokenizer_helper(model)
from_pretrained.assert_not_called()
assert result["type"] == "openai_tokenizer"
def test_offline_mode_keeps_deepseek_v4_on_its_bundled_tokenizer(monkeypatch: pytest.MonkeyPatch):
monkeypatch.setenv("LITELLM_OFFLINE_MODE", "true")
assert litellm.token_counter(model="deepseek/deepseek-v4-flash", text=_CODE_SAMPLE) == _deepseek_token_count(
_CODE_SAMPLE
)
def test_concurrent_first_deepseek_counts_share_one_loaded_tokenizer():
_load_huggingface_tokenizer.cache_clear()
_HUGGINGFACE_TOKENIZER_LOAD_FAILURES.flush_cache()

3
tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_handler.py
View file

@ -2033,6 +2033,7 @@ async def test_anthropic_messages_with_extra_headers():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2096,6 +2097,7 @@ async def test_anthropic_messages_with_thinking():
],
"model": "claude-haiku-4-5-20251001",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}
@ -2210,6 +2212,7 @@ async def test_anthropic_messages_bedrock_dynamic_region():
"content": [{"type": "text", "text": "This is a mock response"}],
"model": "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0",
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 10, "output_tokens": 20},
}

52
tests/unit/llms/anthropic/pass_through/messages/test_anthropic_experimental_pass_through_messages_transformation.py
View file

@ -1,17 +1,23 @@
from collections.abc import Mapping
from datetime import datetime, timezone
from pathlib import Path
from typing import Final
from unittest.mock import patch
import httpx
import pytest
from pydantic import JsonValue
import litellm
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.anthropic.pass_through.messages.transformation import (
AnthropicMessagesConfig,
)
from litellm.llms.anthropic.wif import get_anthropic_wif_token
from litellm.llms.minimax.messages.transformation import MinimaxMessagesConfig
from litellm.llms.openai_like.messages.transformation import OpenAILikeAnthropicMessagesConfig
from litellm.llms.tencent.messages.transformation import TencentAnthropicMessagesConfig
from litellm.types.llms.anthropic import AnthropicFinishReason
from litellm.types.router import GenericLiteLLMParams
from tests.unit.llms.anthropic.test_anthropic_wif import (
ScriptedPoster,
@ -27,6 +33,52 @@ _WIF_PARAMS: Final[dict] = {
}
@pytest.mark.parametrize("config", (AnthropicMessagesConfig(), OpenAILikeAnthropicMessagesConfig()), ids=type)
@pytest.mark.parametrize(
("stop_reason", "stop_fields", "expected_stop_sequence"),
(
pytest.param("end_turn", {}, None, id="end_turn_missing"),
pytest.param("max_tokens", {}, None, id="max_tokens_missing"),
pytest.param("tool_use", {}, None, id="tool_use_missing"),
pytest.param("end_turn", {"stop_sequence": None}, None, id="explicit_null"),
pytest.param("stop_sequence", {"stop_sequence": ", 7"}, ", 7", id="matched_stop"),
),
)
def test_native_response_defaults_stop_sequence_without_changing_other_fields(
config: AnthropicMessagesConfig,
stop_reason: AnthropicFinishReason,
stop_fields: Mapping[str, str | None],
expected_stop_sequence: str | None,
) -> None:
upstream: Final[dict[str, JsonValue]] = {
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "test-model",
"content": [{"type": "text", "text": "ok"}],
"stop_reason": stop_reason,
"usage": {"input_tokens": 19, "output_tokens": 42},
**stop_fields,
}
logging_obj: Final = Logging(
model="test-model",
messages=[],
stream=False,
call_type="anthropic_messages",
start_time=datetime(2026, 1, 1, tzinfo=timezone.utc),
litellm_call_id="test-call",
function_id="test-function",
)
response: Final = config.transform_anthropic_messages_response(
model="test-model", raw_response=httpx.Response(200, json=upstream), logging_obj=logging_obj
)
assert response == {
**upstream,
"stop_sequence": expected_stop_sequence,
}
def test_workload_identity_allowed_for_anthropic() -> None:
assert AnthropicMessagesConfig()._allows_workload_identity is True

Some files were not shown because too many files have changed in this diff Show more