Merge remote-tracking branch 'origin/main' into litellm_messages_drop_unsigned_thinking

# Conflicts:
#	litellm/rust/crates/inference-messages/src/handler.rs
This commit is contained in:
mateo-berri 2026-11-15 19:53:13 -08:00
commit e130ab8ca8
451 changed files with 44348 additions and 12031 deletions

View file

@ -2,7 +2,7 @@ extend = "../../pyproject.toml"
[lint]
preview = true
select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"]
select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR0916", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"]
extend-select = []
# Overrides the inherited list: rules this gate enforces itself must NOT be external here,
# so this config's RUF100 flags their stale `# noqa` directives. What remains external is
@ -44,6 +44,7 @@ convention = "google"
[lint.pylint]
max-args = 5
max-bool-expr = 3
[lint.flake8-tidy-imports.banned-api]
"typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads."

View file

@ -178,6 +178,7 @@ jobs:
windows-sdk:
name: windows-sdk
if: github.event_name != 'pull_request'
runs-on: windows-2025
timeout-minutes: 30
env:
@ -213,6 +214,7 @@ jobs:
proxy-build-from-pip:
name: proxy-build-from-pip
if: github.event_name != 'pull_request'
runs-on: ubuntu-24.04
timeout-minutes: 10
services:
@ -352,7 +354,7 @@ jobs:
smoke-passed:
name: smoke passed
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
needs: [dashboard-build, core-checks, all-extras]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 2

View file

@ -91,6 +91,8 @@ jobs:
- name: Install Python dependencies
working-directory: .
env:
SQLX_OFFLINE: "true"
run: uv sync --frozen --extra extra-proxy
- name: Deploy Prisma migrations

View file

@ -126,7 +126,7 @@ jobs:
name: ${{ matrix.shard }}
needs: [rust-bridge, assert-shard-coverage]
if: ${{ !cancelled() }}
runs-on: ubuntu-latest
runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
timeout-minutes: ${{ matrix.job-timeout-minutes }}
permissions:
contents: read
@ -220,6 +220,7 @@ jobs:
dist: loadscope
- shard: llms-providers
runner: ubuntu-latest-16-cores
test-path: |-
tests/unit/llms/test_api_base_trailing_slash_routes.py
tests/unit/llms/test_cache_control_and_reasoning.py
@ -380,7 +381,7 @@ jobs:
tests/unit/llms/zai
--ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
python-version: "3.12"
workers: 4
workers: 8
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
@ -398,10 +399,11 @@ jobs:
dist: loadscope
- shard: root
runner: ubuntu-latest-16-cores
test-path: >-
tests/unit/test_*.py
python-version: "3.12"
workers: 4
workers: 8
reruns: 2
timeout-minutes: 20
job-timeout-minutes: 60
@ -522,6 +524,7 @@ jobs:
dist: loadscope
- shard: proxy-endpoints
runner: ubuntu-latest-16-cores
test-path: |-
tests/unit/proxy/management_endpoints
tests/unit/proxy/management_helpers
@ -539,13 +542,14 @@ jobs:
--ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
--ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
python-version: "3.12"
workers: 4
workers: 8
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60
dist: loadscope
- shard: proxy-feature-endpoints
runner: ubuntu-latest-16-cores
test-path: |-
tests/unit/proxy/guardrails
--ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
@ -578,7 +582,7 @@ jobs:
tests/unit/proxy/config_resolvers
tests/unit/proxy/utils
python-version: "3.12"
workers: 4
workers: 8
reruns: 0
timeout-minutes: 20
job-timeout-minutes: 60

View file

@ -24,7 +24,7 @@ from litellm.types.utils import (
)
if TYPE_CHECKING:
from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.s3_v2 import S3Logger
BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"]
@ -242,7 +242,7 @@ async def _deliver(
def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None":
from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.s3_v2 import S3Logger
match callback:
case CustomLogger() | S3Logger():

View file

@ -169,7 +169,7 @@ class DualCache(BaseCache):
key,
parent_otel_span: Span | None = None,
local_only: bool = False,
**kwargs,
**kwargs: object,
):
# Try to fetch from in-memory cache first
try:
@ -190,7 +190,7 @@ class DualCache(BaseCache):
result = redis_result
print_verbose(f"get cache: cache result: {result}")
print_verbose("get cache: cache result: %s", result)
return result
except Exception as e:
log_redis_failure(
@ -245,16 +245,16 @@ class DualCache(BaseCache):
key,
parent_otel_span: Span | None = None,
local_only: bool = False,
**kwargs,
**kwargs: object,
):
# Try to fetch from in-memory cache first
try:
print_verbose(f"async get cache: cache key: {key}; local_only: {local_only}")
print_verbose("async get cache: cache key: %s; local_only: %s", key, local_only)
result = None
if self.in_memory_cache is not None:
in_memory_result: Final = await self.in_memory_cache.async_get_cache(key, **kwargs)
print_verbose(f"in_memory_result: {in_memory_result}")
print_verbose("in_memory_result: %s", in_memory_result)
if in_memory_result is not None:
result = in_memory_result
@ -268,7 +268,7 @@ class DualCache(BaseCache):
result = redis_result
print_verbose(f"get cache: cache result: {result}")
print_verbose("get cache: cache result: %s", result)
return result
except Exception as e:
log_redis_failure(
@ -451,8 +451,8 @@ class DualCache(BaseCache):
with_traceback=True,
)
async def async_set_cache(self, key, value, local_only: bool = False, **kwargs):
print_verbose(f"async set cache: cache key: {key}; local_only: {local_only}; value: {value}")
async def async_set_cache(self, key, value, local_only: bool = False, **kwargs: object):
print_verbose("async set cache: cache key: %s; local_only: %s; value: %s", key, local_only, value)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
@ -473,7 +473,7 @@ class DualCache(BaseCache):
"""
Batch write values to the cache
"""
print_verbose(f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}")
print_verbose("async batch set cache: cache keys: %s; local_only: %s", cache_list, local_only)
try:
if self.in_memory_cache is not None:
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:

View file

@ -2362,6 +2362,7 @@ MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "model_batch_enqueued_tok
MAX_BATCH_FILE_RECORDS_KEY: Final = "max_batch_file_records"
MAX_BATCH_FILE_UPLOADS_PER_DAY_KEY: Final = "max_batch_file_uploads_per_day"
MAX_FILE_DOWNLOADS_PER_MINUTE_KEY: Final = "max_file_downloads_per_minute"
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: Final = "model_limit_max_borrow_multiple"
ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS: Final = (
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
MAX_BATCH_FILE_RECORDS_KEY,

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-enterprise"
version = "0.1.85"
version = "0.1.86"
description = "Package for LiteLLM Enterprise features"
readme = "README.md"
requires-python = ">=3.9"
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.1.85"
version = "0.1.86"
version_files = [
"pyproject.toml:^version",
"../../pyproject.toml:litellm-enterprise==",

View file

@ -35,7 +35,6 @@ FileDeleteProvider = Literal[
]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"]
import litellm
from litellm import get_secret_str
from litellm.files.streaming import FileContentStreamingResponse
from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
@ -47,7 +46,6 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
from litellm.llms.xai.batches.handler import XAIBatchesHandler
from litellm.llms.xai.batches.transformation import is_xai_batch_results_id
from litellm.types.llms.openai import (
@ -102,7 +100,6 @@ def _file_content_logging_obj(kwargs: dict[str, object], _is_async: bool) -> Lit
openai_files_instance: Final = OpenAIFilesAPI()
azure_files_instance: Final = AzureOpenAIFilesAPI()
vertex_ai_files_instance: Final = VertexAIFilesHandler()
bedrock_files_instance: Final = BedrockFilesHandler()
xai_batch_results_instance: Final = XAIBatchesHandler()
#################################################
@ -986,27 +983,6 @@ def file_content(
client=client,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "vertex_ai":
api_base: Final = optional_params.api_base or ""
vertex_ai_project: Final = (
optional_params.vertex_project or litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT")
)
vertex_ai_location: Final = (
optional_params.vertex_location or litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION")
)
vertex_credentials: Final = optional_params.vertex_credentials or get_secret_str("VERTEXAI_CREDENTIALS")
response = vertex_ai_files_instance.file_content(
_is_async=_is_async,
file_content_request=_file_content_request,
api_base=api_base,
vertex_credentials=vertex_credentials,
vertex_project=vertex_ai_project,
vertex_location=vertex_ai_location,
timeout=timeout,
max_retries=optional_params.max_retries,
litellm_params=litellm_params_dict,
)
elif custom_llm_provider == "bedrock":
response = bedrock_files_instance.file_content(
_is_async=_is_async,

View file

@ -2,7 +2,7 @@
from collections.abc import Mapping
from enum import Enum
from functools import lru_cache
from functools import cache, lru_cache
from typing import Annotated, Final
from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator
@ -218,7 +218,8 @@ class OpenTelemetryV2Config(BaseSettings):
description=(
"``llm_only`` keeps just the model-call spans on the operator's own Langfuse "
"exporter (the spec whose owner is ``langfuse_otel``). Other exporters and "
"key/team destinations are not affected."
"key/team destinations are not affected. Destinations default to "
"``langfuse_destination_span_scope`` instead."
),
)
excluded_services: Annotated[frozenset[str], NoDecode] = Field(
@ -317,9 +318,7 @@ class OpenTelemetryV2Config(BaseSettings):
@field_validator("langfuse_span_scope", mode="before")
@classmethod
def _normalize_langfuse_span_scope(cls, value: object) -> object:
if isinstance(value, str):
return value.strip().lower()
return value
return _normalized_span_scope(value)
@field_validator(
"baggage_promoted_keys",
@ -417,6 +416,44 @@ def capture_message_content_from_env() -> str:
return _CaptureMessageContentSettings().capture_message_content
def _normalized_span_scope(value: object) -> object:
return value.strip().lower() if isinstance(value, str) else value
class _LangfuseDestinationSpanScopeSettings(BaseSettings):
model_config = SettingsConfigDict(extra="ignore", frozen=True)
langfuse_destination_span_scope: OtelSpanScope = Field(
default="full",
validation_alias=AliasChoices("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE"),
)
@field_validator("langfuse_destination_span_scope", mode="before")
@classmethod
def _normalize_scope(cls, value: object) -> object:
return _normalized_span_scope(value)
def langfuse_destination_span_scope(configured: object = None) -> OtelSpanScope:
return _cached_langfuse_destination_span_scope(None if configured is None else str(configured))
@cache
def _cached_langfuse_destination_span_scope(configured: str | None) -> OtelSpanScope:
try:
return _LangfuseDestinationSpanScopeSettings.model_validate(
{"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": configured} if configured is not None else {}
).langfuse_destination_span_scope
except ValidationError as error:
verbose_logger.warning(
"Invalid LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE or "
"callback_settings.otel.langfuse_destination_span_scope. "
"Allowed values are 'full' and 'llm_only'. Falling back to 'full'. %s",
error,
)
return "full"
_EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...])

View file

@ -161,10 +161,12 @@ def _resource_defaults(callback_name: str) -> Mapping[str, str]:
return backend() if backend is not None else _NO_ATTRS
def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> OtelSpanScope:
def _span_scope(
callback_name: str, params: StandardCallbackDynamicParams, langfuse_destination_span_scope: OtelSpanScope
) -> OtelSpanScope:
if callback_name != "langfuse_otel":
return "full"
return params.get("langfuse_span_scope") or "full"
return params.get("langfuse_span_scope") or langfuse_destination_span_scope
def destination_capable_backends() -> frozenset[str]:
@ -199,6 +201,7 @@ def destination_for(
capture_message_content: str | None = None,
excluded_db_systems: frozenset[str] | None = None,
metadata_keys: tuple[str, ...] | None = None,
langfuse_destination_span_scope: OtelSpanScope = "full",
) -> OtelDestination | None:
"""The destination ``params`` names for ``callback_name``, or ``None``.
@ -228,7 +231,7 @@ def destination_for(
resource_defaults=_resource_defaults(callback_name),
callback_name=callback_name,
protocol=protocol,
span_scope=_span_scope(callback_name, params),
span_scope=_span_scope(callback_name, params, langfuse_destination_span_scope),
capture_message_content=capture_message_content,
excluded_db_systems=excluded_db_systems,
metadata_keys=metadata_keys or (),

View file

@ -181,6 +181,18 @@ class _CustomerBudgetRow(Protocol):
def litellm_budget_table(self) -> _JoinedBudgetRow | None: ...
def _dynamic_limiter_holder_labels(
kind: Literal["team", "key", "customer_org"], holder_id: str, holder_alias: str | None
) -> Mapping[str, str | None]:
match kind:
case "team":
return {"team": holder_id, "team_alias": holder_alias}
case "key":
return {"hashed_api_key": holder_id, "api_key_alias": holder_alias}
case "customer_org":
return {"customer_org_id": holder_id, "customer_org_alias": holder_alias}
def _customer_budget_metrics_enabled() -> bool:
return litellm.enable_end_user_cost_tracking_prometheus_only is True and not litellm.disable_end_user_cost_tracking
@ -4201,14 +4213,17 @@ class PrometheusLogger(CustomLogger):
def set_dynamic_limiter_share(
self,
model_group: str,
team: str,
team_alias: str | None,
holder_kind: Literal["team", "key", "customer_org"],
holder_id: str,
holder_alias: str | None,
priority: str,
share_tokens: int,
borrowed_tokens: int,
) -> None:
enum_values: Final = UserAPIKeyLabelValues(
model_group=model_group, team=team, team_alias=team_alias, priority=priority
model_group=model_group,
priority=priority,
**_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias),
)
self._set_labeled_gauge(
self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens
@ -4239,8 +4254,9 @@ class PrometheusLogger(CustomLogger):
def increment_dynamic_limiter_throttled(
self,
model_group: str,
team: str,
team_alias: str | None,
holder_kind: Literal["team", "key", "customer_org"],
holder_id: str,
holder_alias: str | None,
priority: str,
rate_limit_type: str,
reason: str,
@ -4250,11 +4266,10 @@ class PrometheusLogger(CustomLogger):
"litellm_dynamic_limiter_throttled_total",
UserAPIKeyLabelValues(
model_group=model_group,
team=team,
team_alias=team_alias,
priority=priority,
rate_limit_type=rate_limit_type,
reason=reason,
**_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias),
),
)

View file

@ -1,3 +1,481 @@
from litellm.integrations.s3_v3 import S3BatchUploadError, S3Logger
import asyncio
import copy
import json
import os
from collections.abc import Callable, Collection, Mapping
from typing import TYPE_CHECKING, Final
__all__ = ["S3BatchUploadError", "S3Logger"]
from typing_extensions import Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
COMPLETION_HTTP_FALLBACK_SECONDS,
DEFAULT_S3_BATCH_SIZE,
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
)
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.call import CallContext
from litellm.integrations.custom_logger_v2.derivations import (
cold_storage_object_key,
log_id,
logged_key_identity,
mapping,
payload_copy,
)
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
from litellm.integrations.custom_logger_v2.rust import RustLogger
from litellm.integrations.custom_logger_v2.selection import Selection
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
if TYPE_CHECKING:
from litellm.rust_bridge._native import NativeS3Log
CALLBACK_NAME: Final = "s3_v2"
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
class S3BatchUploadError(Exception):
def __init__(self, failed: int, total: int) -> None:
self.failed: Final = failed
self.total: Final = total
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
def _raise_batch_upload_error(failed: int, total: int) -> None:
raise S3BatchUploadError(failed=failed, total=total)
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
if failure is None:
return
queued, failed, total = failure
try:
_raise_batch_upload_error(failed, total)
except S3BatchUploadError:
verbose_logger.exception(
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
)
def _cold_storage_is_s3_v2() -> bool:
return litellm.cold_storage_custom_logger == "s3_v2"
def _region_from_environment() -> str | None:
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
region: Final = get_secret_str("AWS_REGION", None)
return region if region is not None else region_name
class _Host:
def __init__(self) -> None:
self.failures: Final = CustomLogger()
def callback_failed(self) -> None:
self.failures.handle_callback_failure(callback_name="S3Logger")
def web_identity_token(self, reference: str) -> str:
if reference.startswith("os.environ/") or reference in os.environ:
raise ValueError("Invalid web identity token reference.")
match _read_secret(get_secret, reference):
case None:
raise ValueError("OIDC token could not be retrieved from secret manager.")
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
raise ValueError(
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
)
case str() as token:
return token
case other:
raise ValueError(
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
f"value: {other}, type: {type(other)}, valid types: {str}"
)
def _read_secret(read: Callable[[str], object], reference: str) -> object:
return read(reference)
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
source: Final[Mapping[str, object] | Mapping[object, object]] = (
params_source if params_source is not None else (configured or {})
)
return {
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
for key, value in source.items()
if isinstance(key, str)
}
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
return {
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
}
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
_CONFIGURED_FIRST: Final = frozenset(
{
"s3_bucket_name",
"s3_path",
"s3_region_name",
"s3_api_version",
"s3_endpoint_url",
"s3_aws_access_key_id",
"s3_aws_secret_access_key",
"s3_aws_session_token",
"s3_aws_session_name",
"s3_aws_profile_name",
"s3_aws_role_name",
"s3_aws_web_identity_token",
"s3_aws_sts_endpoint",
"s3_config",
"s3_server_side_encryption",
"s3_sse_kms_key_id",
}
)
_CONSTRUCTOR_ENABLES: Final = frozenset(
{
"s3_use_team_prefix",
"s3_strip_base64_files",
"s3_use_key_prefix",
"s3_use_virtual_hosted_style",
"s3_adaptive_concurrency",
"s3_batch_file_upload",
}
)
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
"s3_use_ssl": True,
"s3_drop_on_terminal_error": True,
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
}
def _blank(value: object) -> bool:
return value is None or value == ""
def _constructor_wins(key: str, configured: object, value: object) -> bool:
return (
(key in _CONFIGURED_FIRST and not configured)
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
)
_KEEP_CONFIGURED: Final = object()
def _constructor_value(key: str, configured: object, value: object) -> object:
if _constructor_wins(key, configured, value):
return value
if key in _CONSTRUCTOR_ENABLES and value:
return True
if key == "s3_max_queue_size" and value is not None:
return resolve_s3_max_queue_size(
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
)
if key == "s3_max_retry_age_seconds":
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
return 0 if resolved is None else resolved
return _KEEP_CONFIGURED
def _constructed_params(
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
) -> dict[str, object]: # mutable-ok: handed to Rust
params: Final = _resolved_params(params_source)
overrides: Final = {
key: replaced
for key, value in settings.items()
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
}
return {**params, **overrides}
class _Base64Stripper(CustomLogger):
def strip(self, payload: StandardLoggingPayload) -> None:
_ = self._strip_base64_from_messages_sync(payload)
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
def _shallow_copy(message: object) -> object:
entries: Final = mapping(message)
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
_STRIPPER: Final = _Base64Stripper()
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
from litellm.litellm_core_utils import litellm_logging
if litellm.global_disable_no_log_param:
return True
litellm_params: Final = kwargs.get("litellm_params", {})
lookup: Final = getattr(litellm_params, "get", None)
if callable(lookup) and lookup("no-log", False) is True:
return False
return not litellm_logging.callback_disabled_dynamically(
callback=CALLBACK_NAME,
litellm_params=litellm_params,
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
)
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
raw: Final = payload.get("messages")
return [*raw] if isinstance(raw, list) else []
class S3StripBase64Files(CustomLoggerV2):
selection = Selection(name=STRIP_CALLBACK_NAME)
def __init__(self, s3_logger_key: str) -> None:
self.s3_logger_key: Final = s3_logger_key
@on(
Hook.log_prepare,
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
writes={Field.standard_logging_payload},
)
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
return None
messages: Final = _payload_messages(payload)
if not messages:
return None
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
while True:
await asyncio.sleep(interval)
try:
batch_upload_failed(await sink.flush())
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
raise
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
details: Final = mapping(kwargs)
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
return None
if logged.get("status") == "failure":
return None
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
source: Final = mapping(kwargs) if details is None else details
return None if source is None else source.get("standard_logging_object")
def _legacy_kwargs(view: object) -> object:
return getattr(view, Field.deprecated_kwargs.value, None)
_read_attribute: Final[Callable[[object, str], object]] = getattr
class _ExcludedView:
def __init__(self, view: object, details: Mapping[str, object]) -> None:
args: Final = HookArgs(request=details)
self.view: Final = view
self.fields: Final[Mapping[str, object]] = {
Field.deprecated_kwargs.value: details,
Field.standard_logging_payload_ref.value: payload_copy(args),
Field.log_id.value: log_id(args),
Field.cold_storage_object_key.value: cold_storage_object_key(args),
Field.logged_key_identity.value: logged_key_identity(args),
}
def __getattr__(self, name: str) -> object:
if name not in self.fields:
return _read_attribute(self.view, name)
value: Final = self.fields[name]
if value is None:
raise AttributeError(name)
return value
_ATTEMPT_ENDED_READS: Final = frozenset(
{
Field.deprecated_kwargs,
Field.attempt_timing,
Field.log_id,
Field.cold_storage_object_key,
Field.standard_logging_payload_ref,
Field.logged_key_identity,
}
)
class S3Logger(RustLogger):
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
def __init__(
self,
s3_callback_params_override: Mapping[str, object] | None = None,
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
) -> None:
from litellm.rust_bridge._native import NativeS3Log
host: Final = _Host()
params: Final = _constructed_params(s3_callback_params_override, settings)
sink: Final = NativeS3Log(
params,
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
host,
_cold_storage_is_s3_v2(),
None if params.get("s3_region_name") else _region_from_environment(),
)
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
self.sink: Final = sink
self.s3_host: Final = host
self.periodic_flush_task: Final = asyncio.create_task(
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
)
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
self.v2_companion: Final = (
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
if sink.strip_base64_files
else None
)
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
async def run_rust(
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
) -> Patch | Abort | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
return None
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
if payload is not None and not isinstance(payload, Mapping):
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
try:
raw: Final = _legacy_kwargs(view)
details: Final = _excluding_fields(raw)
logged: Final = _logged_payload(raw, details)
if self.sink.strip_base64_files and isinstance(logged, Mapping):
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
return self._failed(e)
def _failed(self, error: Exception) -> None:
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
self.s3_host.callback_failed()
@property
def s3_path(self) -> str | None:
return self.sink.s3_path
@property
def s3_bucket_name(self) -> str | None:
return self.sink.s3_bucket_name
def resolve_partition_granularity(self) -> S3PartitionGranularity:
return "hour" if self.sink.partition_granularity() == "hour" else "day"
async def flush_queue(self) -> None:
batch_upload_failed(await self.sink.flush())
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
try:
audit_log_id: Final = audit_log.get("id", "unknown")
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
verbose_logger.exception("S3 audit log error: %s", e)
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
object_name: Final = batch_line_items_object_name(payload)
s3_path: Final = self.s3_path
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
delivered: Final = await self.sink.upload_line_items(
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
)
if not delivered:
verbose_logger.error(
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
len(payload["lines"]),
payload["part"],
payload["batch_id"],
s3_object_key,
)
async def _download_object_from_s3(self, s3_object_key: str) -> object:
try:
downloaded: Final = await self.sink.download(s3_object_key)
return None if downloaded is None else json.loads(downloaded)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error downloading from S3: %s", e)
return None
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
try:
return await self._download_object_from_s3(object_key)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
return None
__all__ = [
"S3BatchUploadError",
"S3Logger",
"S3StripBase64Files",
"legacy_should_log",
]

View file

@ -1,481 +0,0 @@
import asyncio
import copy
import json
import os
from collections.abc import Callable, Collection, Mapping
from typing import TYPE_CHECKING, Final
from typing_extensions import Unpack
import litellm
from litellm._logging import verbose_logger
from litellm.constants import (
COMPLETION_HTTP_FALLBACK_SECONDS,
DEFAULT_S3_BATCH_SIZE,
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
)
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.custom_logger import CustomLogger
from litellm.integrations.custom_logger_v2.call import CallContext
from litellm.integrations.custom_logger_v2.derivations import (
cold_storage_object_key,
log_id,
logged_key_identity,
mapping,
payload_copy,
)
from litellm.integrations.custom_logger_v2.enums import Field, Hook
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
from litellm.integrations.custom_logger_v2.rust import RustLogger
from litellm.integrations.custom_logger_v2.selection import Selection
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
from litellm.secret_managers.main import get_secret, get_secret_str
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
if TYPE_CHECKING:
from litellm.rust_bridge._native import NativeS3Log
CALLBACK_NAME: Final = "s3_v2"
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
class S3BatchUploadError(Exception):
def __init__(self, failed: int, total: int) -> None:
self.failed: Final = failed
self.total: Final = total
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
def _raise_batch_upload_error(failed: int, total: int) -> None:
raise S3BatchUploadError(failed=failed, total=total)
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
if failure is None:
return
queued, failed, total = failure
try:
_raise_batch_upload_error(failed, total)
except S3BatchUploadError:
verbose_logger.exception(
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
)
def _cold_storage_is_s3_v2() -> bool:
return litellm.cold_storage_custom_logger == "s3_v2"
def _region_from_environment() -> str | None:
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
region: Final = get_secret_str("AWS_REGION", None)
return region if region is not None else region_name
class _Host:
def __init__(self) -> None:
self.failures: Final = CustomLogger()
def callback_failed(self) -> None:
self.failures.handle_callback_failure(callback_name="S3Logger")
def web_identity_token(self, reference: str) -> str:
if reference.startswith("os.environ/") or reference in os.environ:
raise ValueError("Invalid web identity token reference.")
match _read_secret(get_secret, reference):
case None:
raise ValueError("OIDC token could not be retrieved from secret manager.")
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
raise ValueError(
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
)
case str() as token:
return token
case other:
raise ValueError(
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
f"value: {other}, type: {type(other)}, valid types: {str}"
)
def _read_secret(read: Callable[[str], object], reference: str) -> object:
return read(reference)
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
source: Final[Mapping[str, object] | Mapping[object, object]] = (
params_source if params_source is not None else (configured or {})
)
return {
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
for key, value in source.items()
if isinstance(key, str)
}
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
return {
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
}
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
_CONFIGURED_FIRST: Final = frozenset(
{
"s3_bucket_name",
"s3_path",
"s3_region_name",
"s3_api_version",
"s3_endpoint_url",
"s3_aws_access_key_id",
"s3_aws_secret_access_key",
"s3_aws_session_token",
"s3_aws_session_name",
"s3_aws_profile_name",
"s3_aws_role_name",
"s3_aws_web_identity_token",
"s3_aws_sts_endpoint",
"s3_config",
"s3_server_side_encryption",
"s3_sse_kms_key_id",
}
)
_CONSTRUCTOR_ENABLES: Final = frozenset(
{
"s3_use_team_prefix",
"s3_strip_base64_files",
"s3_use_key_prefix",
"s3_use_virtual_hosted_style",
"s3_adaptive_concurrency",
"s3_batch_file_upload",
}
)
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
"s3_use_ssl": True,
"s3_drop_on_terminal_error": True,
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
}
def _blank(value: object) -> bool:
return value is None or value == ""
def _constructor_wins(key: str, configured: object, value: object) -> bool:
return (
(key in _CONFIGURED_FIRST and not configured)
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
)
_KEEP_CONFIGURED: Final = object()
def _constructor_value(key: str, configured: object, value: object) -> object:
if _constructor_wins(key, configured, value):
return value
if key in _CONSTRUCTOR_ENABLES and value:
return True
if key == "s3_max_queue_size" and value is not None:
return resolve_s3_max_queue_size(
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
)
if key == "s3_max_retry_age_seconds":
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
return 0 if resolved is None else resolved
return _KEEP_CONFIGURED
def _constructed_params(
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
) -> dict[str, object]: # mutable-ok: handed to Rust
params: Final = _resolved_params(params_source)
overrides: Final = {
key: replaced
for key, value in settings.items()
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
}
return {**params, **overrides}
class _Base64Stripper(CustomLogger):
def strip(self, payload: StandardLoggingPayload) -> None:
_ = self._strip_base64_from_messages_sync(payload)
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
def _shallow_copy(message: object) -> object:
entries: Final = mapping(message)
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
_STRIPPER: Final = _Base64Stripper()
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
from litellm.litellm_core_utils import litellm_logging
if litellm.global_disable_no_log_param:
return True
litellm_params: Final = kwargs.get("litellm_params", {})
lookup: Final = getattr(litellm_params, "get", None)
if callable(lookup) and lookup("no-log", False) is True:
return False
return not litellm_logging.callback_disabled_dynamically(
callback=CALLBACK_NAME,
litellm_params=litellm_params,
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
)
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
raw: Final = payload.get("messages")
return [*raw] if isinstance(raw, list) else []
class S3StripBase64Files(CustomLoggerV2):
selection = Selection(name=STRIP_CALLBACK_NAME)
def __init__(self, s3_logger_key: str) -> None:
self.s3_logger_key: Final = s3_logger_key
@on(
Hook.log_prepare,
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
writes={Field.standard_logging_payload},
)
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
return None
messages: Final = _payload_messages(payload)
if not messages:
return None
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
while True:
await asyncio.sleep(interval)
try:
batch_upload_failed(await sink.flush())
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
raise
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
details: Final = mapping(kwargs)
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
return None
if logged.get("status") == "failure":
return None
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
source: Final = mapping(kwargs) if details is None else details
return None if source is None else source.get("standard_logging_object")
def _legacy_kwargs(view: object) -> object:
return getattr(view, Field.deprecated_kwargs.value, None)
_read_attribute: Final[Callable[[object, str], object]] = getattr
class _ExcludedView:
def __init__(self, view: object, details: Mapping[str, object]) -> None:
args: Final = HookArgs(request=details)
self.view: Final = view
self.fields: Final[Mapping[str, object]] = {
Field.deprecated_kwargs.value: details,
Field.standard_logging_payload_ref.value: payload_copy(args),
Field.log_id.value: log_id(args),
Field.cold_storage_object_key.value: cold_storage_object_key(args),
Field.logged_key_identity.value: logged_key_identity(args),
}
def __getattr__(self, name: str) -> object:
if name not in self.fields:
return _read_attribute(self.view, name)
value: Final = self.fields[name]
if value is None:
raise AttributeError(name)
return value
_ATTEMPT_ENDED_READS: Final = frozenset(
{
Field.deprecated_kwargs,
Field.attempt_timing,
Field.log_id,
Field.cold_storage_object_key,
Field.standard_logging_payload_ref,
Field.logged_key_identity,
}
)
class S3Logger(RustLogger):
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
def __init__(
self,
s3_callback_params_override: Mapping[str, object] | None = None,
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
) -> None:
from litellm.rust_bridge._native import NativeS3Log
host: Final = _Host()
params: Final = _constructed_params(s3_callback_params_override, settings)
sink: Final = NativeS3Log(
params,
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
host,
_cold_storage_is_s3_v2(),
None if params.get("s3_region_name") else _region_from_environment(),
)
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
self.sink: Final = sink
self.s3_host: Final = host
self.periodic_flush_task: Final = asyncio.create_task(
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
)
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
self.v2_companion: Final = (
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
if sink.strip_base64_files
else None
)
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
async def run_rust(
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
) -> Patch | Abort | None:
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
return None
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
if payload is not None and not isinstance(payload, Mapping):
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
try:
raw: Final = _legacy_kwargs(view)
details: Final = _excluding_fields(raw)
logged: Final = _logged_payload(raw, details)
if self.sink.strip_base64_files and isinstance(logged, Mapping):
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
return self._failed(e)
def _failed(self, error: Exception) -> None:
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
self.s3_host.callback_failed()
@property
def s3_path(self) -> str | None:
return self.sink.s3_path
@property
def s3_bucket_name(self) -> str | None:
return self.sink.s3_bucket_name
def resolve_partition_granularity(self) -> S3PartitionGranularity:
return "hour" if self.sink.partition_granularity() == "hour" else "day"
async def flush_queue(self) -> None:
batch_upload_failed(await self.sink.flush())
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
try:
audit_log_id: Final = audit_log.get("id", "unknown")
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
verbose_logger.exception("S3 audit log error: %s", e)
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
object_name: Final = batch_line_items_object_name(payload)
s3_path: Final = self.s3_path
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
delivered: Final = await self.sink.upload_line_items(
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
)
if not delivered:
verbose_logger.error(
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
len(payload["lines"]),
payload["part"],
payload["batch_id"],
s3_object_key,
)
async def _download_object_from_s3(self, s3_object_key: str) -> object:
try:
downloaded: Final = await self.sink.download(s3_object_key)
return None if downloaded is None else json.loads(downloaded)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error downloading from S3: %s", e)
return None
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
try:
return await self._download_object_from_s3(object_key)
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
return None
__all__ = [
"S3BatchUploadError",
"S3Logger",
"S3StripBase64Files",
"legacy_should_log",
]

View file

@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger
from litellm.integrations.pointfive import PointFiveLogger
from litellm.integrations.posthog import PostHogLogger
from litellm.integrations.prometheus import PrometheusLogger
from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.s3_v2 import S3Logger
from litellm.integrations.sqs import SQSLogger
from litellm.integrations.ternary.ternary_logger import TernaryLogger
from litellm.integrations.vantage.vantage_logger import VantageLogger

View file

@ -171,8 +171,12 @@ def is_annual_budget_duration(duration: str) -> bool:
return _parse_duration(normalize_duration(duration)) == (12, "mo")
def normalize_budget_window_start(moment: datetime) -> datetime:
return moment.replace(microsecond=moment.microsecond // 1000 * 1000)
def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime:
boundary: Final = _setup_timezone(reset_at, timezone_str)[0]
boundary: Final = _setup_timezone(normalize_budget_window_start(reset_at), timezone_str)[0]
return _shift_months(boundary, -12).astimezone(timezone.utc)
@ -180,24 +184,25 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
"""Start of the budget period that ends at `reset_at`, under the same rules
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
months and a duration it does not recognize resets at the next midnight."""
boundary: Final = normalize_budget_window_start(reset_at)
value, unit = _parse_duration(normalize_duration(duration))
if value is None:
return reset_at - timedelta(days=1)
return boundary - timedelta(days=1)
match unit:
case "mo":
return _shift_months(reset_at, -value)
return _shift_months(boundary, -value)
case "d":
return _shift_months(reset_at, -1) if value == 30 else reset_at - timedelta(days=value)
return _shift_months(boundary, -1) if value == 30 else boundary - timedelta(days=value)
case "w":
return reset_at - timedelta(weeks=value)
return boundary - timedelta(weeks=value)
case "h":
return reset_at - timedelta(hours=value)
return boundary - timedelta(hours=value)
case "m":
return reset_at - timedelta(minutes=value)
return boundary - timedelta(minutes=value)
case "s":
return reset_at - timedelta(seconds=value)
return boundary - timedelta(seconds=value)
case _:
return reset_at - timedelta(days=1)
return boundary - timedelta(days=1)
def _setup_timezone(current_time: datetime, timezone_str: str = "UTC") -> tuple[datetime, tzinfo]:

View file

@ -6536,7 +6536,7 @@ def build_custom_logger_compatible_class(
_in_memory_loggers.append(_gcs_bucket_logger)
return _gcs_bucket_logger
elif logging_integration == "s3_v2":
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
@ -7360,7 +7360,7 @@ def get_custom_logger_compatible_class(
if isinstance(callback, GCSBucketLogger):
return callback
elif logging_integration == "s3_v2":
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
for callback in _in_memory_loggers:
if isinstance(callback, S3V2Logger):
@ -8067,7 +8067,7 @@ class StandardLoggingPayloadSetup:
)
if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"):
s3_path = getattr(custom_logger, "s3_path")
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
if isinstance(custom_logger, S3V2Logger):
partition_granularity = custom_logger.resolve_partition_granularity()

View file

@ -1008,9 +1008,8 @@ class ChunkProcessor:
cache_creation_token_details: CacheCreationTokenDetails | None = None
cost: float | None = None
for chunk in chunks:
usage_chunk = self.extract_usage_chunk(chunk)
usage_chunks: Final = tuple(self.extract_usage_chunk(chunk) for chunk in chunks)
for usage_chunk in usage_chunks:
if usage_chunk is not None:
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
if usage_chunk_dict["prompt_tokens"] is not None and (
@ -1091,15 +1090,12 @@ class ChunkProcessor:
completion_tokens_details=None if cursor_was_reset else completion_tokens_details,
prompt_tokens_details=prompt_tokens_details,
cost=cost,
inference_geo=self._last_provider_pricing_field(chunks, "inference_geo"),
speed=self._last_provider_pricing_field(chunks, "speed"),
inference_geo=self._last_provider_pricing_field(usage_chunks, "inference_geo"),
speed=self._last_provider_pricing_field(usage_chunks, "speed"),
)
def _last_provider_pricing_field(
self,
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
field: str,
) -> str | None:
@staticmethod
def _last_provider_pricing_field(usage_chunks: Sequence[Usage | None], field: str) -> str | None:
"""
Last value of a provider-specific usage field that changes pricing but is not a
declared ``Usage`` field, e.g. Anthropic's ``speed`` (fast mode multiplies
@ -1107,9 +1103,8 @@ class ChunkProcessor:
"""
values: Final = [
value
for chunk in chunks
if (usage_chunk := self.extract_usage_chunk(chunk)) is not None
and isinstance(value := getattr(usage_chunk, field, None), str)
for usage_chunk in usage_chunks
if usage_chunk is not None and isinstance(value := getattr(usage_chunk, field, None), str)
]
return values[-1] if values else None

View file

@ -1884,9 +1884,11 @@ def _count_anthropic_content(
except Exception as e:
if default_token_count is not None:
return default_token_count
raise (UnsupportedContentTypeError if isinstance(e, UnsupportedContentTypeError) else ValueError)(
f"Error counting field '{field_name}': {e}"
)
raise (
UnsupportedContentTypeError
if server_fields is not None or isinstance(e, UnsupportedContentTypeError)
else ValueError
)(f"Error counting field '{field_name}': {e}")
return tokens

View file

@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str:
return model if region is None else model_id
_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"})
def _split_async_invoke_extra_body(
extra_body: Mapping[str, object],
) -> tuple[Mapping[str, object], Mapping[str, object]]:
return (
{key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS},
{key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS},
)
class BedrockEmbedding(BaseAWSLLM):
_web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY
@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM):
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in ({**extra_body, **item} for item in batch_data):
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
for data in (
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
if "modelInput" in item
else {**extra_body, **item}
for item in batch_data
):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}
@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM):
extra_body: Mapping[str, object] = MappingProxyType({}),
):
responses: Final[list[dict]] = []
for data in ({**extra_body, **item} for item in batch_data):
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
for data in (
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
if "modelInput" in item
else {**extra_body, **item}
for item in batch_data
):
headers = {"Content-Type": "application/json"}
if extra_headers is not None:
headers = {"Content-Type": "application/json", **extra_headers}

View file

@ -92,6 +92,7 @@ class FalAIBytedanceSeedreamV3Config(FalAIBytedanceBaseConfig):
"""
IMAGE_GENERATION_ENDPOINT: str = "fal-ai/bytedance/seedream/v3/text-to-image"
MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/bytedance/seedream/", "bytedance/seedream/")
class FalAIBytedanceDreaminaV31Config(FalAIBytedanceBaseConfig):

View file

@ -34,6 +34,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig):
"""
IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux-pro/v1.1-ultra"
MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/flux-pro/",)
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
"""

View file

@ -5,7 +5,7 @@ import httpx
from pydantic import ConfigDict, TypeAdapter
from typing_extensions import ReadOnly, TypedDict
from litellm.litellm_core_utils.url_utils import join_api_base
from litellm.litellm_core_utils.url_utils import encode_url_path_segments, join_api_base
from litellm.llms.base_llm.image_generation.transformation import (
BaseImageGenerationConfig,
)
@ -71,6 +71,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
DEFAULT_BASE_URL: str = "https://fal.run"
IMAGE_GENERATION_ENDPOINT: str = ""
MODEL_FAMILY_ROOTS: tuple[str, ...] = ()
def get_complete_url(
self,
@ -89,7 +90,16 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
base_url: Final = api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL
if not self.IMAGE_GENERATION_ENDPOINT:
return base_url.rstrip("/")
return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT)
return join_api_base(base_url, self._endpoint_for_model(model))
def _endpoint_for_model(self, model: str) -> str:
path: Final = encode_url_path_segments(model.strip("/"), field_name="model")
names_default_endpoint: Final = path.lower().removeprefix(
"fal-ai/"
) == self.IMAGE_GENERATION_ENDPOINT.removeprefix("fal-ai/")
if names_default_endpoint or not path.lower().startswith(self.MODEL_FAMILY_ROOTS):
return self.IMAGE_GENERATION_ENDPOINT
return path
def validate_environment(
self,

View file

@ -55,7 +55,7 @@ class _ModelProfile:
_SEEDANCE_PROFILE: Final[_ModelProfile] = _ModelProfile(
resolutions=frozenset({"480p", "720p", "1080p", "4k"}),
resolution_tiers=((480, "480p"), (720, "720p"), (1080, "1080p"), (sys.maxsize, "4k")),
default_resolution="720p",
default_resolution=None,
integer_duration=False,
reference_key="image_url",
reference_as_list=False,

View file

@ -1,7 +1,7 @@
"""
Supports writing files to Google AI Studio Files API.
For vertex ai, check out the vertex_ai/files/handler.py file.
For vertex ai, check out the vertex_ai/files/transformation.py file.
"""
import re

View file

@ -3,6 +3,7 @@ Helper util for handling perplexity-specific cost calculation
- e.g.: citation tokens, search queries
"""
import math
from datetime import datetime
from typing import Final
@ -52,13 +53,14 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
except (ValueError, TypeError):
return default
cache_read_cost_value: Final = model_info.get("cache_read_input_token_cost")
rates: Final = apply_off_peak_pricing(
model_info,
current_time,
TokenRates(
input_rate=_safe_float_cast(model_info.get("input_cost_per_token")),
output_rate=_safe_float_cast(model_info.get("output_cost_per_token")),
cache_read_rate=0.0,
cache_read_rate=math.nan if cache_read_cost_value is None else _safe_float_cast(cache_read_cost_value),
cache_creation_rate=0.0,
reasoning_rate=None,
),
@ -67,7 +69,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
output_cost_per_token: Final = rates.output_rate
## CALCULATE INPUT COST
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
prompt_tokens: Final = usage.prompt_tokens or 0
reported_cached_tokens: Final = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
cached_tokens: Final = min(prompt_tokens, reported_cached_tokens)
cache_read_cost_per_token: Final = (
input_cost_per_token if math.isnan(rates.cache_read_rate) else rates.cache_read_rate
)
prompt_cost: float = (
prompt_tokens - cached_tokens
) * input_cost_per_token + cached_tokens * cache_read_cost_per_token
## ADD CITATION TOKENS COST (if present)
citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0

View file

@ -77,13 +77,19 @@ def _profile_for_model(model: str) -> _ModelProfile:
return _MINIMAX_PROFILE if replicate_model_owner(model) == "minimax" else _GENERIC_PROFILE
def _invalid_seconds() -> ReplicateError:
return ReplicateError(status_code=400, message="Replicate seconds must be a finite number", headers={})
def _duration_value(seconds: object) -> int | float:
if isinstance(seconds, bool) or not isinstance(seconds, (int, float, str)):
raise TypeError("Replicate seconds must be a numeric value")
raise _invalid_seconds()
try:
duration: Final = float(seconds)
except ValueError as error:
raise ValueError("Replicate seconds must be a numeric value") from error
except (ValueError, OverflowError) as error:
raise _invalid_seconds() from error
if not math.isfinite(duration):
raise _invalid_seconds()
return int(duration) if duration.is_integer() else duration
@ -99,7 +105,7 @@ def _size_params(size: object, profile: _ModelProfile) -> Mapping[str, object]:
if normalized.count("x") != 1:
return MappingProxyType({})
width_text, height_text = normalized.split("x")
if not (width_text.isdigit() and height_text.isdigit()) or int(width_text) <= 0 or int(height_text) <= 0:
if not (width_text.isdecimal() and height_text.isdecimal()) or int(width_text) <= 0 or int(height_text) <= 0:
return MappingProxyType({})
width: Final = int(width_text)
height: Final = int(height_text)

View file

@ -1,239 +0,0 @@
import asyncio
import json
import os
import time
from collections.abc import Coroutine, Mapping
from typing import Final
from urllib.parse import unquote
import httpx
from litellm import LlmProviders
from litellm.integrations.gcs_bucket.gcs_bucket_base import (
GCSBucketBase,
GCSLoggingConfig,
)
from litellm.litellm_core_utils.cloud_storage_security import (
VERTEX_AI_MANAGED_GCS_PREFIX,
should_allow_legacy_cloud_file_ids,
validate_managed_cloud_file_id,
)
from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
from litellm.types.llms.openai import (
FileContentRequest,
HttpxBinaryResponseContent,
)
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
from litellm.types.utils import StandardCallbackDynamicParams
from .transformation import VertexAIFilesConfig
class VertexAIFilesHandler(GCSBucketBase):
"""
Handles Calling VertexAI in OpenAI Files API format v1/files/*
This implementation uploads files on GCS Buckets
"""
def __init__(self):
super().__init__()
self.async_httpx_client = get_async_httpx_client(
llm_provider=LlmProviders.VERTEX_AI,
)
def _resolve_read_gcs_config(
self,
litellm_params: Mapping[str, object] | None,
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
) -> tuple[str | None, str | None]:
"""
Resolve the GCS bucket and service-account credentials for the read/content path.
Sources them from the deployment's ``litellm_params`` (``gcs_bucket_name`` /
``bucket_name`` and ``vertex_credentials``), mirroring the write path in
``VertexAIFilesConfig._get_configured_bucket_name``, and falls back to the
``GCS_BATCH_BUCKET_NAME`` then ``GCS_BUCKET_NAME`` / ``GCS_PATH_SERVICE_ACCOUNT``
env vars. This lets Vertex batch run entirely at the model-group level, so output
written to a per-model bucket is readable without setting the global env vars.
"""
params: Final[Mapping[str, object]] = litellm_params or {}
bucket_candidate: Final = params.get("gcs_bucket_name") or params.get("bucket_name")
configured_bucket_name = (
bucket_candidate
if isinstance(bucket_candidate, str)
else os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME")
)
credentials: Final = params.get("vertex_credentials") or vertex_credentials
if isinstance(credentials, dict):
path_service_account: str | None = json.dumps(credentials)
elif isinstance(credentials, str):
path_service_account = credentials
else:
path_service_account = os.getenv("GCS_PATH_SERVICE_ACCOUNT")
return configured_bucket_name, path_service_account
def _extract_bucket_and_object_from_file_id(
self,
file_id: str,
configured_bucket_name: str,
litellm_params: dict | None = None,
) -> tuple[str, str]:
"""
Validate and extract bucket name and object path from file_id.
Expected format: gs://bucket-name/litellm-vertex-files/path/to/file
Returns:
tuple: (bucket_name, object_path)
- bucket_name: "bucket-name"
- object_path: "litellm-vertex-files/path/to/file"
"""
return validate_managed_cloud_file_id(
file_id=file_id,
scheme="gs://",
configured_bucket_name=configured_bucket_name,
allowed_object_prefixes=(VERTEX_AI_MANAGED_GCS_PREFIX,),
allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params),
)
async def afile_content(
self,
file_content_request: FileContentRequest,
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
vertex_project: str | None,
vertex_location: str | None,
timeout: float | httpx.Timeout,
max_retries: int | None,
litellm_params: dict | None = None,
) -> HttpxBinaryResponseContent:
"""
Download file content from GCS bucket for VertexAI files.
Args:
file_content_request: Contains file_id (URL-encoded GCS path)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
"""
file_id: Final = file_content_request.get("file_id")
if not file_id:
raise ValueError("file_id is required in file_content_request")
configured_bucket_name, path_service_account = self._resolve_read_gcs_config(
litellm_params=litellm_params,
vertex_credentials=vertex_credentials,
)
dynamic_params: Final = StandardCallbackDynamicParams(
gcs_bucket_name=configured_bucket_name,
gcs_path_service_account=path_service_account,
)
gcs_logging_config: Final[GCSLoggingConfig] = await self.get_gcs_logging_config(
kwargs={"standard_callback_dynamic_params": dynamic_params}
)
bucket_name, object_path = self._extract_bucket_and_object_from_file_id(
file_id=file_id,
configured_bucket_name=gcs_logging_config["bucket_name"],
litellm_params=litellm_params,
)
download_kwargs: Final = {
"standard_callback_dynamic_params": {
"gcs_bucket_name": bucket_name,
"gcs_path_service_account": gcs_logging_config["path_service_account"],
}
}
file_content: Final = await self.download_gcs_object(object_name=object_path, **download_kwargs)
decoded_file_id: Final = unquote(file_id)
if file_content is None:
raise ValueError(f"Failed to download file from GCS: {decoded_file_id}")
mock_response: Final = httpx.Response(
status_code=200,
content=file_content,
headers={
"content-type": "application/octet-stream",
"content-length": str(len(file_content)),
},
request=httpx.Request(method="GET", url=decoded_file_id),
)
# Apply transformation to convert Vertex AI batch outputs to OpenAI format
config: Final = VertexAIFilesConfig()
# Create a logging object for transformation
logging_obj: Final = Logging(
model="",
messages=[],
stream=False,
call_type="afile_content",
start_time=time.time(),
litellm_call_id="",
function_id="",
)
return config.transform_file_content_response(
raw_response=mock_response, logging_obj=logging_obj, litellm_params={}
)
def file_content(
self,
_is_async: bool,
file_content_request: FileContentRequest,
api_base: str | None,
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
vertex_project: str | None,
vertex_location: str | None,
timeout: float | httpx.Timeout,
max_retries: int | None,
litellm_params: dict | None = None,
) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]:
"""
Download file content from GCS bucket for VertexAI files.
Supports both sync and async operations.
Args:
_is_async: Whether to run asynchronously
file_content_request: Contains file_id (URL-encoded GCS path)
api_base: API base (unused for GCS operations)
vertex_credentials: VertexAI credentials
vertex_project: VertexAI project ID
vertex_location: VertexAI location
timeout: Request timeout
max_retries: Max retry attempts
Returns:
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
"""
if _is_async:
return self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
litellm_params=litellm_params,
)
else:
return asyncio.run(
self.afile_content(
file_content_request=file_content_request,
vertex_credentials=vertex_credentials,
vertex_project=vertex_project,
vertex_location=vertex_location,
timeout=timeout,
max_retries=max_retries,
litellm_params=litellm_params,
)
)

View file

@ -10,8 +10,9 @@ Model name handling:
- This module focuses on request/response transformation only
"""
from typing import Final
from typing import Final, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks
from litellm.llms.base_llm.embedding.transformation import split_extra_body
from litellm.types.utils import EmbeddingResponse, Usage
from .types import (
@ -84,7 +85,12 @@ class VertexBGEConfig:
vertex_text_embedding_input_list.append(embedding_input)
vertex_request["instances"] = vertex_text_embedding_input_list
vertex_request["parameters"] = EmbeddingParameters(**optional_params)
body_params, extra_body = split_extra_body(optional_params)
vertex_request["parameters"] = (
cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list
EmbeddingParameters, {**extra_body, **body_params}
)
)
return vertex_request

View file

@ -6,6 +6,9 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele
from litellm.exceptions import BadRequestError, UnsupportedParamsError
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
from litellm.llms.anthropic.messages.native import handler as main
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
AmazonAnthropicClaudeMessagesConfig,
)
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.rust_bridge import catalog, detectors
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
@ -105,8 +108,11 @@ def _route_target(
api_base: str | None,
compacting: bool,
) -> Target:
if main._provider_messages_config(model, provider, api_base) is not None: # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
return Target.MESSAGES
provider_config: Final = main._provider_messages_config(model, provider, api_base) # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
if provider_config is not None:
return (
Target.BEDROCK_INVOKE if type(provider_config) is AmazonAnthropicClaudeMessagesConfig else Target.MESSAGES
)
if main._deployment_passes_through_anthropic_messages(model_info): # pyright: ignore[reportPrivateUsage] # the handler's own passthrough opt-in
return Target.MESSAGES_PASSTHROUGH
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice

View file

@ -101932,5 +101932,31 @@
"supports_parallel_function_calling": true,
"supports_reasoning": true,
"supports_vision": true
},
"fal_ai/bytedance/seedream/v5/pro/us/edit": {
"litellm_provider": "fal_ai",
"metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K with one input image; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135; the first input image is free and each additional input image adds $0.0045 per output, which is not modeled; fal calls this pricing tentative"},
"mode": "image_generation",
"output_cost_per_image": 0.135,
"output_cost_per_image_1K": 0.0675,
"output_cost_per_image_2K": 0.135,
"source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/edit",
"supported_endpoints": [
"/v1/images/edits",
"/v1/images/generations"
],
"supports_vision": true
},
"fal_ai/bytedance/seedream/v5/pro/us/text-to-image": {
"litellm_provider": "fal_ai",
"metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135, so 1.5K has no catalog field"},
"mode": "image_generation",
"output_cost_per_image": 0.135,
"output_cost_per_image_1K": 0.0675,
"output_cost_per_image_2K": 0.135,
"source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/text-to-image",
"supported_endpoints": [
"/v1/images/generations"
]
}
}

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy"
version = "1.109.0"
version = "1.111.0"
description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy"
requires-python = ">=3.10, <3.16"
license = "MIT"

View file

@ -1,6 +1,6 @@
SET LOCAL lock_timeout = '10s';
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" (
"delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
"delivery_id" TEXT PRIMARY KEY,
"request_id" TEXT NOT NULL,
"state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
"attempts" INTEGER NOT NULL DEFAULT 0,

View file

@ -0,0 +1,2 @@
SET LOCAL lock_timeout = '10s';
ALTER TABLE "LiteLLM_SpendLogDelivery" ALTER COLUMN "delivery_id" DROP DEFAULT;

View file

@ -0,0 +1,7 @@
SET LOCAL lock_timeout = '10s';
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_parent_team_ids_idx" ON "LiteLLM_CustomerMemberTable" USING GIN ("parent_team_ids");
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_override_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_override_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_id");
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_override_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_override_id");

View file

@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable {
updated_at DateTime @default(now()) @updatedAt
updated_by String
@@index([parent_org_id])
@@index([budget_id])
@@index([budget_override_id])
@@index([spend])
}
@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable {
updated_at DateTime @default(now()) @updatedAt
updated_by String
@@index([parent_org_id])
@@index([parent_team_ids], type: Gin)
@@index([budget_id])
@@index([budget_override_id])
@@index([spend])
}
@ -2308,7 +2313,7 @@ enum AccountingDeliveryState {
}
model LiteLLM_SpendLogDelivery {
delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text"))
delivery_id String @id @default(uuid())
request_id String
state AccountingDeliveryState @default(pending)
accounting_payload Json

View file

@ -1,6 +1,6 @@
[project]
name = "litellm-proxy-extras"
version = "0.4.116"
version = "0.4.117"
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
readme = "README.md"
requires-python = ">=3.9"
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
module-root = ""
[tool.commitizen]
version = "0.4.116"
version = "0.4.117"
version_files = [
"pyproject.toml:^version",
"../../pyproject.toml:litellm-proxy-extras==",

View file

@ -206,6 +206,8 @@ from .auth_utils import (
if TYPE_CHECKING:
from opentelemetry.trace import Span as _Span
from litellm.proxy.spend_tracking.native_budgets import BudgetDecision
Span = _Span
else:
Span = Any
@ -948,12 +950,11 @@ def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_che
and RouteChecks.runs_inference_checks(route=route)
and route not in MODEL_LIST_ROUTES
):
if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget:
raise litellm.BudgetExceededError(
current_cost=global_proxy_spend,
max_budget=litellm.max_budget,
entity_type=Litellm_EntityType.PROXY.value,
)
native: Final = _native_budget_decision(
{"check": "proxy", "spend": global_proxy_spend, "max_budget": litellm.max_budget}
)
if native.refused:
raise native.error()
_GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = (
@ -1488,14 +1489,16 @@ async def common_checks(
user_info=call_info,
)
)
if math.isfinite(user_budget) and user_spend >= user_budget:
raise litellm.BudgetExceededError(
current_cost=user_spend,
max_budget=user_budget,
message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
entity_type=Litellm_EntityType.USER.value,
entity_id=user_object.user_id,
)
native: Final = _native_budget_decision(
{
"check": "user",
"spend": user_spend,
"max_budget": user_budget,
"user_id": user_object.user_id,
}
)
if native.refused:
raise native.error()
# Each scope reads a distinct counter key with no cross-scope ordering
# dependency, so the per-scope Redis-first reads run concurrently instead
@ -2032,14 +2035,16 @@ async def check_end_user_budget(
max_budget=end_user_budget,
fallback_authoritative=True,
)
if end_user_spend > end_user_budget:
raise litellm.BudgetExceededError(
current_cost=end_user_spend,
max_budget=end_user_budget,
message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}",
entity_type=Litellm_EntityType.END_USER.value,
entity_id=end_user_obj.user_id,
)
native: Final = _native_budget_decision(
{
"check": "end_user",
"spend": end_user_spend,
"max_budget": end_user_budget,
"end_user_id": end_user_obj.user_id,
}
)
if native.refused:
raise native.error()
_check_end_user_budget: Final = check_end_user_budget
@ -6957,13 +6962,6 @@ async def is_valid_fallback_model(
return True
# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The
# last four characters are only barred from being whitespace or a control code,
# because a custom key's can be anything else, punctuation and non-ASCII included;
# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches.
_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?")
def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
"""
Throttle an over-budget key instead of blocking it, when the key opted in
@ -6982,6 +6980,17 @@ def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
return True
def _native_budget_decision(check: Mapping[str, object]) -> "BudgetDecision":
from litellm.proxy.spend_tracking.native_budgets import native_budget_decision
return native_budget_decision(check)
def _budget_exceeded_throttle_pct(valid_token: UserAPIKeyAuth) -> float | None:
pct: Final = budget_throttle_percentage()
return pct if pct is not None and should_throttle_budget_exceeded(valid_token) else None
async def virtual_key_max_budget_check(
valid_token: UserAPIKeyAuth,
proxy_logging_obj: ProxyLogging,
@ -7041,25 +7050,21 @@ async def virtual_key_max_budget_check(
# Defense-in-depth (GHSA-2rv4-xv66-fpjg): spend >= NaN is always False,
# so a NaN max_budget would silently disable enforcement. Treat a
# non-finite max_budget as "no configured limit" rather than as a bypass.
if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget:
if _apply_budget_exceeded_throttle(valid_token):
return
# This message is returned to the caller, and key_name has no enforced
# shape (a direct DB write bypasses abbreviate_api_key), so echo it only
# when it still looks masked and fall back to the alias otherwise.
key_label: Final = valid_token.key_alias or "key"
key_descriptor: Final = (
f"{key_label} ({valid_token.key_name})"
if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name)
else key_label
)
raise litellm.BudgetExceededError(
current_cost=spend,
max_budget=valid_token.max_budget,
message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
entity_type=Litellm_EntityType.KEY.value,
entity_id=valid_token.token,
)
native: Final = _native_budget_decision(
{
"check": "key_max",
"spend": spend,
"max_budget": valid_token.max_budget,
"token": valid_token.token,
"key_alias": valid_token.key_alias,
"key_name": valid_token.key_name,
"throttle_pct": _budget_exceeded_throttle_pct(valid_token),
}
)
if native.decision == "throttle" and _apply_budget_exceeded_throttle(valid_token):
return
if native.refused:
raise native.error()
_virtual_key_max_budget_check: Final = virtual_key_max_budget_check
@ -7096,17 +7101,17 @@ async def _virtual_key_multi_budget_check(
window_duration=str(w["budget_duration"]),
window_start=get_budget_window_start(w),
)
if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
raise litellm.BudgetExceededError(
current_cost=window_spend,
max_budget=w["max_budget"],
message=(
f"ExceededBudget: Key over {w['budget_duration']} budget. "
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
),
entity_type=Litellm_EntityType.KEY.value,
entity_id=valid_token.token,
)
native = _native_budget_decision(
{
"check": "key_window",
"spend": window_spend,
"max_budget": w["max_budget"],
"budget_duration": str(w["budget_duration"]),
"token": valid_token.token,
}
)
if native.refused:
raise native.error()
async def virtual_key_soft_budget_check(
@ -7119,7 +7124,10 @@ async def virtual_key_soft_budget_check(
"""
if valid_token.soft_budget and valid_token.spend >= valid_token.soft_budget:
native: Final = _native_budget_decision(
{"check": "key_soft", "spend": valid_token.spend, "soft_budget": valid_token.soft_budget}
)
if native.alerts:
verbose_proxy_logger.debug(
"Crossed Soft Budget for token %s, spend %s, soft_budget %s",
valid_token.token,
@ -7422,7 +7430,16 @@ async def _check_team_member_budget(
max_budget=team_member_budget,
)
if not math.isfinite(team_member_budget):
native: Final = _native_budget_decision(
{
"check": "team_member",
"spend": team_member_spend,
"max_budget": team_member_budget,
"user_id": valid_token.user_id,
"team_id": team_object.team_id,
}
)
if native.skipped:
return
team_member_max_budget_alert_check(
@ -7437,14 +7454,8 @@ async def _check_team_member_budget(
max_budget=team_member_budget,
)
if team_member_spend >= team_member_budget:
raise litellm.BudgetExceededError(
current_cost=team_member_spend,
max_budget=team_member_budget,
message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}",
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
entity_id=f"{valid_token.user_id}:{team_object.team_id}",
)
if native.refused:
raise native.error()
async def check_team_member_model_access(
@ -7534,7 +7545,15 @@ async def team_max_budget_check(
max_budget=team_object.max_budget,
)
if math.isfinite(team_object.max_budget) and spend > team_object.max_budget:
native: Final = _native_budget_decision(
{
"check": "team_max",
"spend": spend,
"max_budget": team_object.max_budget,
"team_id": team_object.team_id,
}
)
if native.refused:
if valid_token:
call_info: Final = CallInfo(
token=valid_token.token,
@ -7553,13 +7572,7 @@ async def team_max_budget_check(
)
)
raise litellm.BudgetExceededError(
current_cost=spend,
max_budget=team_object.max_budget,
message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}",
entity_type=Litellm_EntityType.TEAM.value,
entity_id=team_object.team_id,
)
raise native.error()
_team_max_budget_check: Final = team_max_budget_check
@ -7571,29 +7584,26 @@ async def _user_multi_budget_check(user_object: LiteLLM_UserTable | None) -> Non
from litellm.proxy.proxy_server import get_current_spend
for window in user_object.budget_limits:
if (
spend := await get_current_spend(
counter_key=user_window_counter(user_object.user_id, window.budget_duration),
fallback_spend=0.0,
max_budget=window.max_budget,
window_entity_type="User",
window_entity_id=user_object.user_id,
window_duration=window.budget_duration,
window_start=get_budget_window_start(window),
)
) < window.max_budget:
continue
if math.isfinite(window.max_budget) and spend >= window.max_budget:
raise litellm.BudgetExceededError(
current_cost=spend,
max_budget=window.max_budget,
message=(
f"ExceededBudget: User={user_object.user_id} over {window.budget_duration} budget. "
f"Spend=${spend:.4f}, Limit=${window.max_budget:.2f}"
),
entity_type=Litellm_EntityType.USER.value,
entity_id=user_object.user_id,
)
spend = await get_current_spend(
counter_key=user_window_counter(user_object.user_id, window.budget_duration),
fallback_spend=0.0,
max_budget=window.max_budget,
window_entity_type="User",
window_entity_id=user_object.user_id,
window_duration=window.budget_duration,
window_start=get_budget_window_start(window),
)
native = _native_budget_decision(
{
"check": "user_window",
"spend": spend,
"max_budget": window.max_budget,
"budget_duration": window.budget_duration,
"user_id": user_object.user_id,
}
)
if native.refused:
raise native.error()
async def _team_multi_budget_check(
@ -7623,17 +7633,17 @@ async def _team_multi_budget_check(
window_duration=str(w["budget_duration"]),
window_start=get_budget_window_start(w),
)
if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
raise litellm.BudgetExceededError(
current_cost=window_spend,
max_budget=w["max_budget"],
message=(
f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. "
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
),
entity_type=Litellm_EntityType.TEAM.value,
entity_id=team_object.team_id,
)
native = _native_budget_decision(
{
"check": "team_window",
"spend": window_spend,
"max_budget": w["max_budget"],
"budget_duration": str(w["budget_duration"]),
"team_id": team_object.team_id,
}
)
if native.refused:
raise native.error()
async def _team_soft_budget_check(
@ -7644,12 +7654,12 @@ async def _team_soft_budget_check(
"""
Triggers a budget alert if the team is over it's soft budget.
"""
if (
team_object is not None
and team_object.soft_budget is not None
and team_object.spend is not None
and team_object.spend >= team_object.soft_budget
):
if team_object is None:
return
native: Final = _native_budget_decision(
{"check": "team_soft", "spend": team_object.spend, "soft_budget": team_object.soft_budget}
)
if native.alerts and team_object.spend is not None:
verbose_proxy_logger.debug(
"Crossed Soft Budget for team %s, spend %s, soft_budget %s",
team_object.team_id,
@ -7737,7 +7747,15 @@ async def _project_max_budget_check(
max_budget=max_budget,
)
if project_spend >= max_budget:
native: Final = _native_budget_decision(
{
"check": "project_max",
"spend": project_spend,
"max_budget": max_budget,
"project_id": project_object.project_id,
}
)
if native.refused:
if valid_token:
call_info: Final = CallInfo(
token=valid_token.token,
@ -7756,13 +7774,7 @@ async def _project_max_budget_check(
)
)
raise litellm.BudgetExceededError(
current_cost=project_spend,
max_budget=max_budget,
message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_spend}, Max budget: {max_budget}",
entity_type=Litellm_EntityType.PROJECT.value,
entity_id=project_object.project_id,
)
raise native.error()
async def _project_soft_budget_check(
@ -7782,7 +7794,10 @@ async def _project_soft_budget_check(
if project_object.litellm_budget_table is not None:
soft_budget = project_object.litellm_budget_table.soft_budget
if soft_budget is not None and project_object.spend is not None and project_object.spend >= soft_budget:
native: Final = _native_budget_decision(
{"check": "project_soft", "spend": project_object.spend, "soft_budget": soft_budget}
)
if native.alerts:
verbose_proxy_logger.debug(
"Crossed Soft Budget for project %s, spend %s, soft_budget %s",
project_object.project_id,
@ -7945,7 +7960,10 @@ async def organization_max_budget_check(
)
# Check if organization spend exceeds max budget
if math.isfinite(org_max_budget) and org_spend >= org_max_budget:
native: Final = _native_budget_decision(
{"check": "organization", "spend": org_spend, "max_budget": org_max_budget, "org_id": org_id}
)
if native.refused:
# Trigger budget alert
call_info: Final = CallInfo(
token=valid_token.token,
@ -7964,13 +7982,7 @@ async def organization_max_budget_check(
)
)
raise litellm.BudgetExceededError(
current_cost=org_spend,
max_budget=org_max_budget,
message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
entity_type=Litellm_EntityType.ORGANIZATION.value,
entity_id=org_id,
)
raise native.error()
_organization_max_budget_check: Final = organization_max_budget_check
@ -8034,15 +8046,16 @@ async def tag_max_budget_check_for_tags(
max_budget=tag_object.litellm_budget_table.max_budget,
fallback_authoritative=True,
)
if tag_spend <= tag_object.litellm_budget_table.max_budget:
continue
raise litellm.BudgetExceededError(
current_cost=tag_spend,
max_budget=tag_object.litellm_budget_table.max_budget,
message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}",
entity_type=Litellm_EntityType.TAG.value,
entity_id=tag_name,
native = _native_budget_decision(
{
"check": "tag",
"spend": tag_spend,
"max_budget": tag_object.litellm_budget_table.max_budget,
"tag": tag_name,
}
)
if native.refused:
raise native.error()
async def _model_access_group_max_budget_check(
@ -8089,15 +8102,11 @@ async def _model_access_group_max_budget_check(
max_budget=budget.max_budget,
fallback_authoritative=True,
)
if group_spend < budget.max_budget:
continue
raise litellm.BudgetExceededError(
current_cost=group_spend,
max_budget=budget.max_budget,
message=f"Budget has been exceeded! Model access group={group} Current cost: {group_spend}, Max budget: {budget.max_budget}",
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP.value,
entity_id=group,
native = _native_budget_decision(
{"check": "model_access_group", "spend": group_spend, "max_budget": budget.max_budget, "group": group}
)
if native.refused:
raise native.error()
def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool:

View file

@ -22,6 +22,7 @@ from litellm.constants import (
INVALID_VIRTUAL_KEY_ERROR_MARKER,
MINIMUM_CUSTOM_KEY_LENGTH,
MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
STANDARD_CUSTOMER_ID_HEADERS,
)
from litellm.litellm_core_utils.clientside_credential_handler import (
@ -1922,7 +1923,21 @@ class BatchLimitRequest(Protocol):
def model_fields_set(self) -> Collection[str]: ...
_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY)
_ADMIN_ONLY_METADATA_KEYS: Final = (
*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS,
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY,
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
)
_ADMIN_ONLY_METADATA_REASONS: Final[Mapping[str, str]] = MappingProxyType(
{
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: (
"It lets the holder read and write those buckets with the proxy's AWS credentials."
),
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: (
"It caps how far the holder borrows idle model capacity, so the holder cannot raise or remove it."
),
}
)
def enforce_batch_limits_are_admin_only(
@ -1955,10 +1970,8 @@ def enforce_batch_limits_are_admin_only(
)
if changed is None:
return
reason: Final = (
"It lets the holder read and write those buckets with the proxy's AWS credentials."
if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY
else "It limits what the holder can do with batches, so the holder cannot raise it."
reason: Final = _ADMIN_ONLY_METADATA_REASONS.get(
changed, "It limits what the holder can do with batches, so the holder cannot raise it."
)
raise HTTPException(
status_code=403,

View file

@ -553,7 +553,7 @@ async def _register_organization(
if isinstance(registered, Failure):
if registered.code is ErrorCode.FREE_TIER_LIMIT:
verbose_proxy_logger.warning(
"CUSTOMER TRACKING OFF for organization %s because the free customer portal limit was reached. "
"CUSTOMER TRACKING OFF for organization %s because the Customer Management free tier limit was reached. "
"Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise",
customer_org_alias,
)
@ -620,7 +620,7 @@ async def _register_member(
if isinstance(registered, Failure):
if registered.code is ErrorCode.FREE_TIER_LIMIT:
verbose_proxy_logger.warning(
"CUSTOMER TRACKING OFF for member %s because the free customer portal limit was reached. "
"CUSTOMER TRACKING OFF for member %s because the Customer Management free tier limit was reached. "
"Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise",
customer_member_id,
)

View file

@ -198,6 +198,7 @@ from litellm.proxy.spend_tracking.committed_spend import (
committed_spend_tracked,
read_admission_committed_spend,
)
from litellm.proxy.spend_tracking.native_budgets import native_budget_decision
from litellm.proxy.spend_tracking.spend_counter_batch import (
bind_admission_counter_keys,
release_spend_counter_batch,
@ -2322,12 +2323,22 @@ async def _small_preflight_failure_json(request: Request) -> Mapping[str, object
}
def _is_bodyless_get(request: Request) -> bool:
return (
request.scope.get("method") == "GET"
and request.headers.get("content-length", "0") == "0"
and request.headers.get("transfer-encoding") is None
)
async def _reject_from_headers_before_reading_body(
request: Request, api_key: str | None, custom_litellm_key_header: str | None, route: str
) -> None:
"""Refuse a request its headers already condemn, a blocked key or an exhausted key limit, before reading its body.
Only a key already in the cache is considered, so a miss changes nothing and the full auth path still runs.
The limit is only checked where a body is declared and the route is metered, because model listing, token
counting and other GET discovery routes never reach the limiter and must keep answering an exhausted key.
A retry flood of large payloads against a limited key used to buffer every payload just to answer 429.
"""
if not RouteChecks.is_llm_api_route(route=route):
@ -2377,6 +2388,8 @@ async def _reject_from_headers_before_reading_body(
dispatched_call_type=str(request_data["call_type"]) if request_data["call_type"] is not None else None,
)
return
if RouteChecks.is_unbilled_route(route) or _is_bodyless_get(request):
return
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
return
@ -3119,7 +3132,7 @@ async def user_api_key_auth_builder(
for k, v in team_obj_dict.items():
field_name = f"team_{k}"
if field_name in valid_token.__fields__:
if field_name in type(valid_token).model_fields:
setattr(valid_token, field_name, v)
except Exception as e:
verbose_logger.debug(e) # moving from .warning to .debug as it spams logs when team missing from cache.
@ -3420,7 +3433,16 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
fallback_spend=team_member_spend,
max_budget=team_member_budget,
)
if team_member_spend >= team_member_budget:
native: Final = native_budget_decision(
{
"check": "team_member_positive",
"spend": team_member_spend,
"max_budget": team_member_budget,
"user_id": valid_token.user_id,
"team_id": valid_token.team_id,
}
)
if native.refused:
# common_checks sends this alert on requests that get past here, so only the
# request rejected here sends it from the builder.
team_member_max_budget_alert_check(
@ -3434,17 +3456,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
spend=team_member_spend,
max_budget=team_member_budget,
)
_entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}"
raise litellm.BudgetExceededError(
current_cost=team_member_spend,
max_budget=team_member_budget,
message=(
f"Budget has been exceeded! TeamMember={_entity_id} "
f"Current cost: {team_member_spend}, Max budget: {team_member_budget}"
),
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
entity_id=_entity_id,
)
raise native.error()
# Check 3. If token is expired
if valid_token.expires is not None:

View file

@ -5,6 +5,7 @@ import json
import logging
import math
import time
import weakref
from collections.abc import (
AsyncGenerator,
AsyncIterator,
@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing:
),
)
def _request_fields(self) -> Mapping[str, object]:
return self.data
def _arm_detached_stream_failure_hook(
self,
logging_obj: LiteLLMLoggingObj,
@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing:
so the failed spend is still written and the budget reservation released;
a replacement error the hook raises has no client left to reach.
"""
request_data: Final = self.data
# The closure lives on the logging object until the logging object dies, so it must not
# hold the logging object, or a request stays alive until a full GC pass (#51277)
logging_obj_ref: Final = weakref.ref(logging_obj)
request_data: Final = {
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
}
route_type: Final = self.route_type
async def _on_detached_stream_failure(exc: Exception) -> None:
if not logging_obj.claim_stream_failure_report():
live_logging_obj: Final = logging_obj_ref()
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
return
try:
await proxy_logging_obj.post_call_failure_hook(
user_api_key_dict=user_api_key_dict,
original_exception=exc,
request_data=request_data,
dispatched_call_type=self.route_type,
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
dispatched_call_type=route_type,
)
except HTTPException:
return

View file

@ -3,7 +3,7 @@ import json
import math
import time
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from datetime import datetime, timedelta, timezone
from enum import Enum
from itertools import chain
@ -36,6 +36,7 @@ from litellm.litellm_core_utils.duration_parser import (
duration_in_seconds,
get_annual_budget_window_start,
is_annual_budget_duration,
normalize_budget_window_start,
)
from litellm.proxy._types import (
DB_RETRY_SAFE_ERROR_TYPES,
@ -378,9 +379,15 @@ def _customer_reset_key(
def _customer_ids_by_reset(
rows: Sequence[_CustomerScopedRow], ids: Sequence[str], now: datetime, windows: Mapping[str, str]
) -> Mapping[tuple[float | None, str | None], tuple[str, ...]]:
keys: Final = tuple(_customer_reset_key(row, now, windows) for row in rows)
capped: Final = tuple(
(row_id, row) for row_id, row in zip(ids, rows) if _customer_effective_cap(row, now) is not None
)
keys: Final = tuple(_customer_reset_key(row, now, windows) for _, row in capped)
return MappingProxyType(
{key: tuple(row_id for row_id, row_key in zip(ids, keys) if row_key == key) for key in dict.fromkeys(keys)}
{
key: tuple(row_id for (row_id, _), row_key in zip(capped, keys) if row_key == key)
for key in dict.fromkeys(keys)
}
)
@ -506,6 +513,10 @@ class _BudgetCascade:
customer_ids: Mapping[CustomerSpendEntity, tuple[str, ...]] = field(default_factory=lambda: MappingProxyType({}))
customer_org_alias_keys: tuple[str, ...] = ()
@property
def customer_budget_ids(self) -> tuple[str, ...]:
return tuple(b.budget_id for b in self.budgets if b.max_budget is not None and b.budget_id in self.budget_ids)
@dataclass(frozen=True, slots=True)
class _EndUserWalk:
@ -1280,7 +1291,11 @@ class ResetBudgetJob:
_queue_budget_linked_resets(uow.projects, cascade, extra=_SPENT_ROWS_WHERE)
_queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE)
_queue_budget_linked_resets(uow.model_access_groups, cascade, extra=_SPENT_ROWS_WHERE)
_queue_budget_linked_resets(uow.customer_organizations, cascade, extra=_SPENT_ROWS_WHERE)
_queue_budget_linked_resets(
uow.customer_organizations,
replace(cascade, budget_ids=cascade.customer_budget_ids),
extra=_SPENT_ROWS_WHERE,
)
for budget_id, budget_reset_at in cascade.budget_resets:
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
@ -1932,7 +1947,7 @@ class ResetBudgetJob:
entity_type=entity_type.value,
entity_id=entity_id,
window_duration=budget_duration,
new_window_start=window_start,
new_window_start=normalize_budget_window_start(window_start),
)
except Exception as e: # noqa: BLE001 # the row is best effort; counter resets must still land
verbose_proxy_logger.warning(

View file

@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import re
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload
from weakref import WeakValueDictionary
@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None:
return _TTL.validate_python(kwargs.get("ttl"))
@dataclass(frozen=True, slots=True)
class _DecodedModel:
payload: object
model_type: type[BaseModel]
model: BaseModel
class CacheLease:
__slots__ = ("__weakref__", "key", "shared")
@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache):
self._redis_connection: RedisCache | None = None
self._shared: AuthCacheStore | None = None
self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary()
self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads
self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
@property
@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache):
def set_local(self, key: str, value: object, ttl: float | None) -> None:
"""A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None."""
self._drop_decoded((key,))
if self._shared is not None:
self._shared.set_local(key, value, ttl)
return
@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache):
"""Caches ``value`` unless the lease's key was evicted since it was taken; False when refused."""
if self._leases.get(lease.key) is not lease:
return False
self._drop_decoded((lease.key,))
shared: Final = self._shared
if shared is None:
await self.async_set_cache(key=lease.key, value=value, ttl=ttl)
@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache):
return True
def _void_leases(self, keys: Sequence[str]) -> None:
self._drop_decoded(keys)
for key in keys:
_ = self._leases.pop(key, None)
def _drop_decoded(self, keys: Sequence[str]) -> None:
for key in keys:
_ = self._decoded_models.pop(key, None)
def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
"""In Python memory, a read of the same payload object a previous read validated returns a shallow copy of
that model instead of validating again, so a caller that sets a field cannot change what the next reader
gets. The shared store hands back a new payload on every read, so it is validated each time."""
if model_type is None or cached is None:
return cached
memo: Final = self._decoded_models.get(key)
if memo is not None and memo.payload is cached and memo.model_type is model_type:
return memo.model.model_copy()
decoded: Final = _validated(cached, key, model_type, method)
if decoded is None or decoded is cached or self._shared is not None:
return decoded
_ = self._decoded_models.pop(key, None)
self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded)
if len(self._decoded_models) > self._decoded_capacity():
_ = self._decoded_models.pop(next(iter(self._decoded_models)))
return decoded.model_copy()
def _decoded_capacity(self) -> int:
return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory
def delete_local(self, key: str) -> None:
"""Drops this worker's memory copy alone, as a cross-worker eviction broadcast does."""
self._void_leases((key,))
@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache):
if is_user_key_cache_key(key)
else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
)
return _decoded(cached, key, model_type, "get_cache")
return self._decoded(cached, key, model_type, "get_cache")
@overload
async def async_get_cache(
@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache):
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
)
)
return _decoded(cached, key, model_type, "async_get_cache")
return self._decoded(cached, key, model_type, "async_get_cache")
@staticmethod
def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object:
@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache):
def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object):
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
if key is not None:
self._drop_decoded((key,))
shared: Final = self._shared
if shared is not None:
if key is None:
@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache):
else, or with options the pipeline does not carry, it goes to Redis directly as before."""
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
if key is not None:
self._drop_decoded((key,))
ttl: Final = kwargs.get("ttl")
pipelined: Final = (
key is not None
@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache):
def flush_cache(self) -> None:
self._leases.clear()
self._decoded_models.clear()
self.budget_windows_absent_cache.flush_cache()
if self._shared is not None:
self._shared.flush_local()
@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache):
``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged.
"""
normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list)
self._drop_decoded(tuple(key for key, _ in normalized))
shared: Final = self._shared
if shared is not None:
await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs))
@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache):
)
def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
if model_type is None or cached is None:
return cached
def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None:
decoded: Final = CacheCodec.deserialize(cached, model_type=model_type)
if decoded is None:
verbose_proxy_logger.error(

View file

@ -0,0 +1,56 @@
"""Split each Postgres pool's connection budget between Prisma and the Rust core's SQLx pool.
Migration scaffolding. Prisma and SQLx share a database only while the Python gateway still runs Prisma
queries next to ported ones. These invariants hold until Prisma is gone:
- Operators set one `connection_limit` per pool (`database_connection_pool_limit`, a URL pin or
`database_extra_connection_params`). Nothing an operator sets names Prisma, SQLx or Rust. The Rust share
is carved out of that limit, never added on top, so a Rust pool does not grow the connection budget.
- The Rust share is the hard-coded `RUST_RESERVED_CONNECTIONS`. It is 0 until a ported query runs on the
native pool, so Prisma keeps its whole limit, untouched, and no native pool opens.
- Only a positive integer limit is split. Any other value goes to Prisma as it is, which judges it the same
way it did before the split existed.
- The split is Python's. It is about Prisma and Prisma lives only in the Python gateway, so this module is
deleted with Prisma.
- The Rust core takes a plain `max_connections` and never learns about the split. That is already its final
interface, since the Rust gateway will pass `database_connection_pool_limit` straight to SQLx.
"""
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
RUST_RESERVED_CONNECTIONS: Final = 0
@dataclass(frozen=True, slots=True)
class ConnectionBudget:
total: int
python: int
rust: int
@classmethod
def split(cls, total: int, rust_reserved: int = RUST_RESERVED_CONNECTIONS) -> "ConnectionBudget":
"""Prisma keeps at least one connection, so a total at or below the reserve ends up over it."""
return cls(total=total, python=max(1, total - rust_reserved), rust=rust_reserved)
@classmethod
def of(
cls, connection_limit: str | float, rust_reserved: int = RUST_RESERVED_CONNECTIONS
) -> "ConnectionBudget | None":
"""The split of a positive integer limit, or None for any other value Prisma is left to judge."""
digits: Final = str(connection_limit)
if not (digits.isascii() and digits.isdigit()) or int(digits) == 0:
return None
return cls.split(int(digits), rust_reserved)
@property
def prisma_limit(self) -> int | None:
"""Prisma's share once the Rust pool takes some of the total, or None to leave Prisma's limit untouched."""
return None if self.rust == 0 else self.python
def prisma_params(self, params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]:
"""``params`` with Prisma's share as its ``connection_limit``, untouched while the Rust pool takes none."""
limit: Final = self.prisma_limit
return params if limit is None else MappingProxyType({**params, "connection_limit": limit})

View file

@ -32,7 +32,7 @@ from litellm.proxy.db.log_db_metrics import ( # noqa: F401 # legacy module exp
_T = TypeVar("_T")
def _service_logging() -> ServiceLogging | None:
def proxy_service_logging() -> ServiceLogging | None:
try:
from litellm.proxy.proxy_server import proxy_logging_obj
except ImportError:
@ -40,7 +40,9 @@ def _service_logging() -> ServiceLogging | None:
return proxy_logging_obj.service_logging_obj
def _event_metadata(table: str | None, operation: str | None) -> dict[str, str]:
def db_event_metadata(
table: str | None, operation: str | None
) -> dict[str, str]: # mutable-ok: ServiceLogging hooks take event_metadata as a dict
pairs: Final = (("table_name", table), ("db_operation", operation))
return {key: value for key, value in pairs if value is not None}
@ -73,9 +75,9 @@ async def db_span(call_type: str, table: str | None, operation: str | None = Non
if db_io_claimed():
yield
return
service_logging: Final = _service_logging()
service_logging: Final = proxy_service_logging()
start_time: Final = datetime.now()
event_metadata: Final = _event_metadata(table, operation)
event_metadata: Final = db_event_metadata(table, operation)
with claim_db_io() as witness:
try:
yield

View file

@ -254,12 +254,28 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p
return urllib.parse.urlunsplit(parsed._replace(query=query))
def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str:
PRISMA_APPLICATION_NAME: Final = "litellm-prisma"
NATIVE_APPLICATION_NAME: Final = "litellm-native"
def prisma_default_params(
lifetime_params: Mapping[str, str | int | float], native_connections: int
) -> Mapping[str, str | int | float]:
"""Prisma's URL defaults, tagged ``PRISMA_APPLICATION_NAME`` only once a native pool shares the database."""
if native_connections == 0:
return lifetime_params
return MappingProxyType({**lifetime_params, "application_name": PRISMA_APPLICATION_NAME})
def postgres_connection_budget_message(
writer_limit: str, rust_writer_limit: int, reader_limit: str | None, rust_reader_limit: int, num_workers: str
) -> str:
"""The startup line that states this pod's worst-case Postgres connection demand.
Prisma's ``connection_limit`` is per query engine, and every uvicorn worker
owns one engine per configured database (writer, plus the reader when
``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs,
``DATABASE_URL_READ_REPLICA`` is set) plus a Rust pool beside each whose
share (``connection_budget``) is above 0. The limits are read off the final URLs,
so a reader that pins its own ``connection_limit`` or an operator override in
``database_extra_connection_params`` is counted at its real value. The
server-side cap is shared by every pod, so the number an operator has to keep
@ -272,16 +288,21 @@ def postgres_connection_budget_message(writer_limit: str, reader_limit: str | No
reader: Final = int(reader_limit) if reader_limit is not None else 0
except ValueError:
return (
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader "
f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + "
f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit"
f"{' + rust reader pool' if rust_reader_limit else ''}) "
f"(workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
"keep pods x that figure under max_connections minus superuser_reserved_connections"
)
engines: Final = (
f"(writer connection_limit {writer} + reader connection_limit {reader})"
if reader_limit is not None
else f"writer connection_limit {writer}"
rust_reader: Final = rust_reader_limit if reader_limit is not None else 0
terms: Final = (
f"writer connection_limit {writer}",
*((f"rust writer pool {rust_writer_limit}",) if rust_writer_limit else ()),
*((f"reader connection_limit {reader}",) if reader_limit is not None else ()),
*((f"rust reader pool {rust_reader}",) if rust_reader else ()),
)
per_pod: Final = workers * (writer + reader)
engines: Final = terms[0] if len(terms) == 1 else f"({' + '.join(terms)})"
per_pod: Final = workers * (writer + rust_writer_limit + reader + rust_reader)
return (
f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} "
"connections; keep pods x that figure under max_connections minus superuser_reserved_connections"

View file

@ -14,6 +14,7 @@ from litellm.proxy._types import (
ProxyException,
)
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
from litellm.rust_bridge._native import NativeDatabaseFailure
from litellm.secret_managers.main import str_to_bool
# Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain.
@ -175,6 +176,8 @@ class PrismaDBExceptionHandler:
"""
import prisma.engine.errors
if isinstance(e, NativeDatabaseFailure):
return e.unavailable
if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)):
return True
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
@ -769,6 +772,8 @@ _DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"})
def db_write_failure_is_requeue_safe(e: Exception) -> bool:
"""Whether a failed write provably never applied, so its rows can be sent again without double counting."""
if isinstance(e, NativeDatabaseFailure):
return e.requeue_safe
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)

View file

@ -0,0 +1,369 @@
"""Settings for the native (SQLx) pool, resolved from the same URL Prisma connects with.
``DATABASE_URL`` is final once ``db_url_settings`` and the CLI have assembled it: token
minted, libpq TLS translated to Prisma's dialect, timeouts in ``options``, pool params
added. Reading that URL keeps one resolver for both pools instead of a second pass over
the ``DATABASE_*`` env vars.
Credentials follow the host. Each process runs one token refresh loop, owned by whatever
hosts it, and feeds every client in that process: here the worker's loop re-mints Prisma's
URL and pushes the same kind of token into the Rust pool through ``set_writer_password``.
The Rust core only accepts a password and never mints one, so a Rust gateway process
brings its own loop. Two processes minting at once is fine, since IAM and Entra tokens
are independent. What must not exist twice is the logic that resolves region, endpoint
and user, so when the Rust gateway needs it, it moves into the Rust core and this
loop calls it.
"""
import asyncio
import shlex
import struct
import urllib.parse
from collections.abc import Awaitable, Callable, Mapping, Sequence
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from pathlib import Path
from types import MappingProxyType
from typing import Final, Protocol
from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12
from litellm._logging import verbose_proxy_logger
from litellm._service_logger import ServiceTypes
from litellm.proxy.db.db_span import db_event_metadata
from litellm.proxy.db.db_url_settings import NATIVE_APPLICATION_NAME, PRISMA_APPLICATION_NAME
from litellm.proxy.db.pgbouncer import database_url_is_pooled, fold_libpq_option_tokens, libpq_option_setting
from litellm.proxy.db.token_auth import (
DEFAULT_POSTGRES_PORT,
DatabaseTokenAuth,
IAMEndpoint,
mint_database_token,
parse_database_token_expiration,
parse_iam_endpoint_from_url,
resolve_database_token_auth,
)
from litellm.rust_bridge.database import (
NativeClientIdentity,
NativeDatabaseEndpoint,
NativePoolLimits,
SslMode,
)
TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0
TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0
TOKEN_RETRY_SECONDS: Final = 30.0
PRISMA_DEFAULT_SCHEMA: Final = "public"
PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS: Final = 10.0
PRISMA_DEFAULT_STATEMENT_CACHE_SIZE: Final = 100
TLS_PROBE_TIMEOUT_SECONDS: Final = 5.0
_POSTGRES_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103)
@dataclass(frozen=True, slots=True)
class NativeEndpointSettings:
endpoint: NativeDatabaseEndpoint
ignored_options: tuple[str, ...]
def _params(url: str) -> Mapping[str, str]:
return MappingProxyType(dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)))
def needs_tls_probe(url: str) -> bool:
"""Prisma's ``sslmode=prefer`` with ``sslaccept=strict`` verifies TLS when the server offers it
and falls back to plaintext when it does not; SQLx has no verified-or-plaintext mode, so the
caller asks the server once and pins the answer."""
params: Final = _params(url)
return params.get("sslmode", "prefer") == "prefer" and params.get("sslaccept") == "strict" and "host" not in params
async def server_offers_tls(url: str, timeout: float = TLS_PROBE_TIMEOUT_SECONDS) -> bool:
"""Whether the server answers a Postgres ``SSLRequest`` with ``S``; an unreachable server counts as offering it."""
parsed: Final = urllib.parse.urlsplit(url)
try:
reader, writer = await asyncio.wait_for(
asyncio.open_connection(parsed.hostname, parsed.port or int(DEFAULT_POSTGRES_PORT)), timeout
)
except (OSError, asyncio.TimeoutError):
return True
try:
writer.write(_POSTGRES_SSL_REQUEST)
await writer.drain()
answer: Final = await asyncio.wait_for(reader.read(1), timeout)
except (OSError, asyncio.TimeoutError):
return True
finally:
writer.close()
return answer != b"N"
def _tls(params: Mapping[str, str], tls_offered: bool) -> tuple[SslMode, str | None]:
sslmode: Final = params.get("sslmode", "prefer")
strict: Final = params.get("sslaccept") == "strict"
if sslmode == "disable" or (sslmode == "prefer" and strict and not tls_offered):
return "disable", None
if strict:
return "verify-full", params.get("sslcert")
return ("require" if sslmode == "require" else "prefer"), None
def _client_identity(
params: Mapping[str, str], ssl_mode: SslMode, read_file: Callable[[Path], bytes]
) -> NativeClientIdentity | None:
identity_path: Final = params.get("sslidentity")
if identity_path is None or ssl_mode == "disable":
return None
password: Final = params.get("sslpassword")
key, cert, chain = pkcs12.load_key_and_certificates(
read_file(Path(identity_path)), None if password is None else password.encode()
)
if key is None or cert is None:
raise ValueError(f"sslidentity {identity_path!r} must hold a client certificate and its private key")
return NativeClientIdentity(
cert_pem=b"".join(c.public_bytes(Encoding.PEM) for c in (cert, *chain)).decode(),
key_pem=key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode(),
)
def _session_options(params: Mapping[str, str]) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]:
raw: Final = params.get("options", "")
try:
tokens: Final[Sequence[str]] = fold_libpq_option_tokens(tuple(shlex.split(raw)))
except ValueError:
return (), (raw,)
readable: Final = tuple((token, libpq_option_setting(token)) for token in tokens)
schema: Final = params.get("schema", PRISMA_DEFAULT_SCHEMA)
search_path: Final = () if schema == PRISMA_DEFAULT_SCHEMA else (("search_path", f'"{schema}"'),)
settings: Final = tuple(
(name, value) for name, _, value in (setting.partition("=") for _, setting in readable if setting is not None)
)
return (*search_path, *settings), tuple(token for token, setting in readable if setting is None)
def _seconds(params: Mapping[str, str], key: str, default: float | None = None) -> float | None:
value: Final = params.get(key)
if value is None:
return default
return None if float(value) == 0 else float(value)
def _statement_cache_capacity(params: Mapping[str, str]) -> int:
if params.get("pgbouncer") == "true":
return 0
return int(params.get("statement_cache_size", PRISMA_DEFAULT_STATEMENT_CACHE_SIZE))
def _application_name(params: Mapping[str, str]) -> str:
configured: Final = params.get("application_name", PRISMA_APPLICATION_NAME)
return NATIVE_APPLICATION_NAME if configured == PRISMA_APPLICATION_NAME else configured
def native_endpoint(
url: str,
*,
tls_offered: bool,
max_connections: int,
read_file: Callable[[Path], bytes] = Path.read_bytes,
) -> NativeEndpointSettings:
"""The native pool's endpoint for the Prisma URL ``url``, read the way Prisma reads it.
``host`` is Prisma's Unix socket directory. The pool takes ``max_connections``, the
Rust share of the budget (``connection_budget``), and tags its connections
``NATIVE_APPLICATION_NAME``. An unset ``pool_timeout`` takes Prisma's default and
0 waits forever.
``sslidentity`` / ``sslpassword`` are a PKCS#12 client certificate, handed over as PEM.
``options`` tokens that are not ``-c name=value`` settings cannot be sent by SQLx and
come back in ``ignored_options`` instead of failing boot.
"""
parsed: Final = urllib.parse.urlsplit(url)
params: Final = _params(url)
socket: Final = params.get("host")
if not (parsed.hostname or socket) or not parsed.username:
raise ValueError("DATABASE_URL must name a host and a user for the native pool")
ssl_mode, ssl_root_cert = _tls(params, tls_offered)
options, ignored = _session_options(params)
return NativeEndpointSettings(
endpoint=NativeDatabaseEndpoint(
host=parsed.hostname or "localhost",
port=parsed.port or int(DEFAULT_POSTGRES_PORT),
socket=socket,
username=urllib.parse.unquote(parsed.username),
password=urllib.parse.unquote(parsed.password) if parsed.password else None,
database=urllib.parse.unquote(parsed.path.lstrip("/")) or "postgres",
ssl_mode=ssl_mode,
ssl_root_cert=ssl_root_cert,
ssl_client=_client_identity(params, ssl_mode, read_file),
application_name=_application_name(params),
options=options,
statement_cache_capacity=_statement_cache_capacity(params),
pool=NativePoolLimits(
max_connections=max_connections,
acquire_timeout_seconds=_seconds(params, "pool_timeout", PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS),
idle_timeout_seconds=_seconds(params, "max_idle_connection_lifetime"),
socket_timeout_seconds=_seconds(params, "socket_timeout"),
),
),
ignored_options=ignored,
)
async def resolve_native_endpoint(url: str, max_connections: int) -> NativeEndpointSettings:
tls_offered: Final = await server_offers_tls(url) if needs_tls_probe(url) else True
return native_endpoint(url, tls_offered=tls_offered, max_connections=max_connections)
@dataclass(frozen=True, slots=True)
class TokenRotation:
auth: DatabaseTokenAuth
endpoint: IAMEndpoint
mint_token: Callable[[DatabaseTokenAuth, IAMEndpoint], str] = mint_database_token
def mint(self) -> str:
return urllib.parse.unquote(self.mint_token(self.auth, self.endpoint))
def seconds_until_refresh(self, token: str, now: datetime) -> float:
expires_at: Final = parse_database_token_expiration(self.auth, token)
if expires_at is None:
return TOKEN_FALLBACK_REFRESH_SECONDS
return max(TOKEN_RETRY_SECONDS, (expires_at - now).total_seconds() - TOKEN_REFRESH_BUFFER_SECONDS)
def writer_token_rotation(url: str) -> TokenRotation | None:
"""The token rotation the writer needs, or None for a static password.
A URL pointed at the in-container PgBouncer carries PgBouncer's own credential,
and PgBouncer renews the upstream token itself.
"""
if database_url_is_pooled():
return None
auth: Final = resolve_database_token_auth()
if auth is None:
return None
return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url))
def reader_token_rotation(url: str) -> TokenRotation | None:
"""The token rotation the read replica needs, or None for a static password.
The reader never goes through the in-container PgBouncer, so only the token toggles
decide, with the reader's own region, the same as its Prisma client.
"""
auth: Final = resolve_database_token_auth(read_replica=True)
if auth is None:
return None
return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url))
def _utcnow() -> datetime:
return datetime.now(timezone.utc).replace(tzinfo=None)
async def _refresh(install: Callable[[str], None], rotation: TokenRotation, now: Callable[[], datetime]) -> float:
try:
token: Final = await asyncio.to_thread(rotation.mint)
except Exception as error: # noqa: BLE001 # boto3 and azure-identity raise many types; any failure means retry
verbose_proxy_logger.error(
"Native database pool keeps its current %s (%r); retrying in %.0fs.",
rotation.auth.label,
error,
TOKEN_RETRY_SECONDS,
)
return TOKEN_RETRY_SECONDS
install(token)
return rotation.seconds_until_refresh(token, now())
async def keep_token_fresh(
install: Callable[[str], None],
rotation: TokenRotation,
current_token: str,
now: Callable[[], datetime] = _utcnow,
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
) -> None:
"""Hand the native pool a fresh token before the current one expires.
Existing connections stay authenticated; only connections opened after a refresh
use the new token, the same as a re-minted Prisma URL.
"""
delay = rotation.seconds_until_refresh(current_token, now()) # rebind-ok: each refresh schedules the next
while True:
await sleep(delay)
delay = await _refresh(install, rotation, now)
class DatabaseServiceHooks(Protocol):
async def async_service_success_hook(
self,
service: ServiceTypes,
call_type: str,
duration: float,
parent_otel_span: None = None,
start_time: datetime | None = None,
end_time: datetime | None = None,
event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature
) -> None: ...
async def async_service_failure_hook(
self,
service: ServiceTypes,
duration: float,
error: Exception,
call_type: str,
parent_otel_span: None = None,
start_time: datetime | None = None,
end_time: datetime | None = None,
event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature
) -> None: ...
class NativeDatabaseError(Exception):
"""A native (SQLx) query failure, reported to service logging like a Prisma one."""
@dataclass(frozen=True, slots=True)
class ServiceLoggingEvents:
"""Turns native round trips into the ``ServiceTypes.DB`` events ``db_span`` emits.
Rust calls ``record`` from its own threads once per round trip, so the hook is
handed to the proxy's event loop rather than awaited here.
"""
loop: asyncio.AbstractEventLoop
service_logging: Callable[[], DatabaseServiceHooks | None]
def record(
self,
call_type: str,
table: str | None,
operation: str | None,
started_at: float,
duration_seconds: float,
failure: str | None,
) -> None:
service_logging: Final = self.service_logging()
if service_logging is None:
return
start_time: Final = datetime.fromtimestamp(started_at, tz=timezone.utc).astimezone().replace(tzinfo=None)
hook: Final = (
service_logging.async_service_success_hook(
service=ServiceTypes.DB,
call_type=call_type,
parent_otel_span=None,
duration=duration_seconds,
start_time=start_time,
end_time=start_time + timedelta(seconds=duration_seconds),
event_metadata=db_event_metadata(table, operation),
)
if failure is None
else service_logging.async_service_failure_hook(
error=NativeDatabaseError(failure),
service=ServiceTypes.DB,
call_type=call_type,
parent_otel_span=None,
duration=duration_seconds,
start_time=start_time,
end_time=start_time + timedelta(seconds=duration_seconds),
event_metadata=db_event_metadata(table, operation),
)
)
_ = self.loop.call_soon_threadsafe(self.loop.create_task, hook)

View file

@ -140,19 +140,27 @@ def _userlist_quote(value: str) -> str:
return '"' + value.replace('"', '""') + '"'
def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None:
"""The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else.
Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a
detached ``-c`` is folded into the token that follows it first.
"""
folded: Final = tuple(
def fold_libpq_option_tokens(tokens: Sequence[str]) -> tuple[str, ...]:
"""The tokens of a libpq ``options`` string with each detached ``-c`` folded into the token after it."""
return tuple(
f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token
for index, token in enumerate(tokens)
if index == 0 or tokens[index - 1] != "-c"
)
settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:])
return settings if len(settings) == len(folded) else None
def libpq_option_setting(token: str) -> str | None:
"""The ``name=value`` setting a folded ``-cname=value`` or ``--name=value`` token carries, or None."""
return token[2:] if token.startswith(("-c", "--")) and "=" in token[2:] else None
def libpq_option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None:
"""The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else.
Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``.
"""
settings: Final = tuple(libpq_option_setting(token) for token in fold_libpq_option_tokens(tokens))
return None if None in settings else tuple(setting for setting in settings if setting is not None)
def _connect_query(options: str) -> str | PgBouncerError:
@ -163,7 +171,7 @@ def _connect_query(options: str) -> str | PgBouncerError:
the settings are applied to each new server connection instead. Every
client shares them, which is what the single ``DATABASE_URL`` gave anyway.
"""
settings: Final = _option_settings(tuple(shlex.split(options)))
settings: Final = libpq_option_settings(tuple(shlex.split(options)))
if settings is None:
return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings")
return "; ".join(

File diff suppressed because it is too large Load diff

View file

@ -41,6 +41,7 @@ from litellm.caching.redis_cache import log_redis_failure
from litellm.constants import (
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
INTERNAL_CALL_ORIGIN_METADATA_KEY,
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
)
@ -118,6 +119,7 @@ from litellm.types.extraction import ExtractionResponse
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
from litellm.types.mcp_server.mcp_server_manager import MCPServer
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
from litellm.types.proxy.auth.customer_hierarchy import CustomerScopeAuth
from litellm.types.utils import (
CallTypes,
EmbeddingResponse,
@ -200,7 +202,7 @@ def inherited_team_model_limit(
return team_limit
TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "team_model_limit_max_borrow_multiple"
MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY
MAX_BORROW_MULTIPLE_CEILING: Final = 1_000_000
_MAX_BORROW_MULTIPLE: Final[TypeAdapter[float]] = TypeAdapter(
Annotated[float, Field(ge=1, le=MAX_BORROW_MULTIPLE_CEILING, allow_inf_nan=False)]
@ -231,30 +233,48 @@ class BorrowingTeamModelLimits:
NO_BORROWING: Final = BorrowingTeamModelLimits(tpm=None, rpm=None)
def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None:
raw: Final[object] = (user_api_key_dict.team_metadata or {}).get(TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE)
BorrowerKind: TypeAlias = Literal["team", "key", "customer_org"]
BORROWER_KINDS: Final[tuple[BorrowerKind, ...]] = ("team", "key", "customer_org")
_METADATA: Final = TypeAdapter(Mapping[str, object])
@dataclass(frozen=True, slots=True)
class ModelLimitBorrower:
"""Who holds a per-model limit the dynamic rate limiter enforces at saturation, and that limit."""
kind: BorrowerKind
entity_id: str
alias: str | None
limits: BorrowingTeamModelLimits
def _max_borrow_multiple(raw: object, setting: str, owner: str) -> float | None:
if raw is None:
return None
try:
return _MAX_BORROW_MULTIPLE.validate_python(raw)
except ValidationError:
verbose_proxy_logger.warning(
"Ignoring %s=%r on team %s: expected a number from 1 to %d, so the team borrows up to the model limit",
TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE,
"Ignoring %s=%r on %s: expected a number from 1 to %d, so the value is not used",
setting,
raw,
user_api_key_dict.team_id,
owner,
MAX_BORROW_MULTIPLE_CEILING,
)
return None
def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None:
return _max_borrow_multiple(
_METADATA.validate_python(user_api_key_dict.team_metadata or {}).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE),
MODEL_LIMIT_MAX_BORROW_MULTIPLE,
f"team {user_api_key_dict.team_id}",
)
def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits:
settings: Final = litellm.priority_reservation_settings
if (
settings is None
or not settings.team_model_limit_borrows_idle_capacity_from_other_teams
or user_api_key_dict.team_id is None
):
if settings is None or "team" not in settings.model_limit_borrowing_entities or user_api_key_dict.team_id is None:
return NO_BORROWING
limits: Final = BorrowingTeamModelLimits(
tpm=inherited_team_model_limit(user_api_key_dict, model, "model_tpm_limit"),
@ -263,10 +283,90 @@ def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLi
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
return NO_BORROWING
return BorrowingTeamModelLimits(
tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict)
tpm=limits.tpm,
rpm=limits.rpm,
max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict)
or settings.model_limit_borrowing_config.max_borrow_multiple,
)
def borrowing_key_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits:
"""The key's own per-model limits when keys are listed to borrow idle capacity."""
settings: Final = litellm.priority_reservation_settings
if settings is None or "key" not in settings.model_limit_borrowing_entities or user_api_key_dict.api_key is None:
return NO_BORROWING
limits: Final = BorrowingTeamModelLimits(
tpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit")),
rpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit")),
)
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
return NO_BORROWING
return BorrowingTeamModelLimits(
tpm=limits.tpm,
rpm=limits.rpm,
max_borrow_multiple=_max_borrow_multiple(
_METADATA.validate_python(
user_api_key_dict.metadata or {} # pyright: ignore[reportUnknownMemberType] # validates the untyped key metadata
).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE),
MODEL_LIMIT_MAX_BORROW_MULTIPLE,
"a key",
)
or settings.model_limit_borrowing_config.max_borrow_multiple,
)
def _customer_org_scope(user_api_key_dict: UserAPIKeyAuth) -> CustomerScopeAuth | None:
hierarchy: Final = user_api_key_dict.customer_hierarchy
if hierarchy is None:
return None
return next((scope for scope in hierarchy.scopes if scope.key == "customer_org"), None)
def borrowing_customer_org_model_limits(
user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel
) -> BorrowingTeamModelLimits:
"""The customer organization's per-model limits when customer organizations are listed to borrow idle
capacity. The share is one counter per organization and model, across every key the customer calls."""
settings: Final = litellm.priority_reservation_settings
scope: Final = _customer_org_scope(user_api_key_dict)
if settings is None or "customer_organization" not in settings.model_limit_borrowing_entities or scope is None:
return NO_BORROWING
limits: Final = BorrowingTeamModelLimits(
tpm=model.limit_in(scope.model_tpm_limit), rpm=model.limit_in(scope.model_rpm_limit)
)
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
return NO_BORROWING
return BorrowingTeamModelLimits(
tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=settings.model_limit_borrowing_config.max_borrow_multiple
)
def model_limit_borrowers(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> tuple[ModelLimitBorrower, ...]:
"""Every level of the request whose per-model limit on ``model`` is a share that borrows idle capacity."""
org_scope: Final = _customer_org_scope(user_api_key_dict)
candidates: Final = (
ModelLimitBorrower(
kind="team",
entity_id=str(user_api_key_dict.team_id),
alias=user_api_key_dict.team_alias,
limits=borrowing_team_model_limits(user_api_key_dict, model),
),
ModelLimitBorrower(
kind="key",
entity_id=str(user_api_key_dict.api_key),
alias=user_api_key_dict.key_alias,
limits=borrowing_key_model_limits(user_api_key_dict, model),
),
ModelLimitBorrower(
kind="customer_org",
entity_id="" if org_scope is None else org_scope.entity_id,
alias=None if org_scope is None else org_scope.alias,
limits=borrowing_customer_org_model_limits(user_api_key_dict, model),
),
)
return tuple(borrower for borrower in candidates if borrower.limits != NO_BORROWING)
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
if router is None:
return (model,)
@ -587,6 +687,34 @@ class BorrowingTeamModelHold:
share: RateLimitDescriptor | None = None
@dataclass(slots=True)
class BorrowingShareState:
"""One borrower's share on the admitting model for this request: what was enforced, what was held on it, and the
holds taken on router fallback targets."""
limit: RateLimitDescriptor | None = None
reserved_tokens: int = 0
reserved_window: ReservationWindowIdentity | None = None
share: RateLimitDescriptor | None = None
cap: RateLimitDescriptor | None = None
cap_reserved_tokens: int = 0
cap_reserved_window: ReservationWindowIdentity | None = None
fallback_limits: Mapping[str, BorrowingTeamModelHold] = field(default_factory=lambda: MappingProxyType({}))
def clear_reservation(self) -> None:
self.reserved_tokens = 0
self.cap_reserved_tokens = 0
self.reserved_window = None
self.cap_reserved_window = None
def hold_fallback(self, names: tuple[str, ...], hold: BorrowingTeamModelHold) -> None:
self.fallback_limits = MappingProxyType({**self.fallback_limits, **dict.fromkeys(names, hold)})
def drop_fallback_hold(self, hold: BorrowingTeamModelHold | None) -> None:
if hold is not None:
self.fallback_limits = MappingProxyType({k: h for k, h in self.fallback_limits.items() if h is not hold})
@dataclass(slots=True)
class RequestRateLimiterStash:
"""
@ -631,17 +759,11 @@ class RequestRateLimiterStash:
otpm_reserved_window_identities: frozenset[ReservationWindowIdentity] = field(default_factory=frozenset)
batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
batch_tpd_refund_ops: tuple[ReservationOperation, ...] = ()
borrowing_team_model_limit: RateLimitDescriptor | None = None
borrowing_team_model_reserved_tokens: int = 0
borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None
borrowing_team_model_share: RateLimitDescriptor | None = None
borrowing_team_model_cap: RateLimitDescriptor | None = None
borrowing_team_model_cap_reserved_tokens: int = 0
borrowing_team_model_cap_reserved_window: ReservationWindowIdentity | None = None
borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field(
default_factory=lambda: MappingProxyType({})
borrowing_shares: Mapping[BorrowerKind, BorrowingShareState] = field(
default_factory=lambda: MappingProxyType({kind: BorrowingShareState() for kind in BORROWER_KINDS})
)
priority_class_share: RateLimitDescriptor | None = None
priority_class_share_holder: str | None = None
dynamic_limiter_priority: str | None = None
reservation_released: bool = False
io_reservation_released: bool = False
@ -2718,8 +2840,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
model: Final = self._rate_limited_model(requested_model)
if model is None or user_api_key_dict.customer_hierarchy is None:
return ()
org_borrowing: Final = borrowing_customer_org_model_limits(user_api_key_dict, model)
limits: Final = tuple(
(scope, model.limit_in(scope.model_tpm_limit), model.limit_in(scope.model_rpm_limit))
(
scope,
None
if scope.key == "customer_org" and org_borrowing.tpm is not None
else model.limit_in(scope.model_tpm_limit),
None
if scope.key == "customer_org" and org_borrowing.rpm is not None
else model.limit_in(scope.model_rpm_limit),
)
for scope in user_api_key_dict.customer_hierarchy.scopes
)
return tuple(
@ -2779,14 +2910,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
if model is None:
return
borrowing: Final = borrowing_team_model_limits(user_api_key_dict, model)
key_borrowing: Final = borrowing_key_model_limits(user_api_key_dict, model)
model_specific_tpm_limit: Final = (
None
if borrowing.tpm is not None
if borrowing.tpm is not None or key_borrowing.tpm is not None
else model.limit_in(get_key_model_tpm_limit(user_api_key_dict, model_name=model.group))
)
model_specific_rpm_limit: Final = (
None
if borrowing.rpm is not None
if borrowing.rpm is not None or key_borrowing.rpm is not None
else model.limit_in(get_key_model_rpm_limit(user_api_key_dict, model_name=model.group))
)
if model_specific_tpm_limit is None and model_specific_rpm_limit is None:

View file

@ -1,6 +1,6 @@
"""Splits a priority class's share of a model among the teams active on it.
"""Splits a priority class's share of a model among the callers active on it.
Enabled by ``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``.
Enabled by ``priority_reservation_settings.priority_class_share_config``.
"""
import time
@ -40,6 +40,16 @@ def team_member(team_id: str) -> str:
return f"team{MEMBER_SEPARATOR}{team_id}"
def key_member(key_hash: str) -> str:
"""The registry id and counter owner of a key that takes its own slice of its class, apart from every team id."""
return f"key{MEMBER_SEPARATOR}{key_hash}"
def customer_member(kind: str, entity_id: str) -> str:
"""The registry id and counter owner of a customer organization or customer member taking its own slice."""
return f"{kind}{MEMBER_SEPARATOR}{entity_id}"
def is_pool_member(member: str) -> bool:
return member.startswith(_POOL_PREFIX)

View file

@ -1595,7 +1595,11 @@ def _tenant_destination(
name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None
) -> "OtelDestination | None":
from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys
from litellm.integrations.otel.model.config import parse_capture_message_content, parse_excluded_services
from litellm.integrations.otel.model.config import (
langfuse_destination_span_scope,
parse_capture_message_content,
parse_excluded_services,
)
from litellm.integrations.otel.presets.destinations import destination_for
callback_vars: Final = MappingProxyType(
@ -1605,6 +1609,7 @@ def _tenant_destination(
)
)
)
otel_settings: Final = as_str_mapping((litellm.callback_settings or {}).get("otel")) or {}
return destination_for(
name,
_tenant_otel_params(callback_vars),
@ -1612,6 +1617,9 @@ def _tenant_destination(
parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)),
parse_excluded_services(callback_vars.get(EXCLUDED_SERVICES_VAR)),
parse_destination_metadata_keys(callback_vars.get(BAGGAGE_METADATA_KEYS_VAR)),
langfuse_destination_span_scope=langfuse_destination_span_scope(
otel_settings.get("langfuse_destination_span_scope")
),
)

View file

@ -278,7 +278,12 @@ async def update_budget(
return response
@router.post("/budget/new", tags=["budget management"], description=new_budget.__doc__)
@router.post(
"/budget/new",
tags=["budget management"],
description=new_budget.__doc__,
operation_id="new_budget_budget_new_post",
)
async def new_budget_endpoint(
budget_obj: BudgetWriteRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
@ -286,7 +291,12 @@ async def new_budget_endpoint(
return await new_budget(budget_obj, user_api_key_dict)
@router.post("/budget/update", tags=["budget management"], description=update_budget.__doc__)
@router.post(
"/budget/update",
tags=["budget management"],
description=update_budget.__doc__,
operation_id="update_budget_budget_update_post",
)
async def update_budget_endpoint(
budget_obj: BudgetWriteRequest,
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],

View file

@ -103,7 +103,7 @@ from litellm.proxy.utils import handle_exception_on_proxy, hash_password, query_
from litellm.redis.stores.spend_counters import (
user_counter,
)
from litellm.repositories.chunked_in import update_many_in
from litellm.repositories.chunked_in import find_many_in, update_many_in
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import TableActions
from litellm.repositories.table_repositories import (
@ -159,6 +159,7 @@ class _AdminTeamAndKeyRows(TypedDict):
_ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter(
list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True)
)
_TEAM_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
_ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter(
dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)
)
@ -468,8 +469,12 @@ async def add_new_user_to_default_team(
user_api_key_dict: UserAPIKeyAuth,
teams: Sequence[str | NewUserRequestTeam],
prisma_client: "PrismaClient",
teams_are_defaults: bool = True,
):
memberships: Final = tuple(_default_team_membership(team) for team in teams)
requested: Final = tuple(_default_team_membership(team) for team in teams)
memberships: Final = (
await _memberships_on_team_member_budget(requested, prisma_client) if teams_are_defaults else requested
)
for membership in memberships:
with contextlib.suppress(Exception):
await _add_user_to_team(
@ -482,6 +487,25 @@ async def add_new_user_to_default_team(
)
async def _memberships_on_team_member_budget(
memberships: tuple[NewUserRequestTeam, ...], prisma_client: "PrismaClient"
) -> tuple[NewUserRequestTeam, ...]:
"""A default team's max_budget_in_team is already the team's member budget, which also carries the reset."""
capped_team_ids: Final = [m.team_id for m in memberships if m.max_budget_in_team is not None]
if not capped_team_ids:
return memberships
team_rows: Final = await find_many_in(_team_table(prisma_client), "team_id", capped_team_ids)
budgeted_team_ids: Final = frozenset(
row.team_id
for row in team_rows
if isinstance(row.metadata, dict)
and _TEAM_METADATA_ADAPTER.validate_python(row.metadata).get("team_member_budget_id") is not None
)
return tuple(
m.model_copy(update={"max_budget_in_team": None}) if m.team_id in budgeted_team_ids else m for m in memberships
)
def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam:
if isinstance(team, str):
return NewUserRequestTeam(team_id=team)
@ -666,6 +690,7 @@ async def new_user(
user_api_key_dict=user_api_key_dict,
teams=teams,
prisma_client=prisma_client,
teams_are_defaults=data.teams is None,
)
user_id: Final = cast(str | None, response.get("user_id", None))

View file

@ -17,6 +17,7 @@ from litellm.proxy.list_api.common import (
)
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
from litellm.proxy.management_helpers.customer_hierarchy import (
BudgetSort,
CustomerHierarchy,
CustomerPortalLimits,
ErrorCode,
@ -54,13 +55,18 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy
{
ErrorCode.CAPACITY_BUSY: (
503,
"Customer portal busy",
"Customer portal capacity is being updated. Retry the request.",
"Customer Management busy",
"Customer Management capacity is being updated. Retry the request.",
),
ErrorCode.TRANSACTION_TIMEOUT: (
504,
"Customer Management operation timed out",
"Customer Management operation timed out. The transaction was rolled back. Retry the request.",
),
ErrorCode.FREE_TIER_LIMIT: (
403,
"Free customer portal limit reached",
"Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and "
"Customer Management free tier limit reached",
"Customer Management free tier limit reached: up to 5 customer organizations, 5 teams per organization and "
"5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: "
"https://www.litellm.ai/enterprise",
),
@ -213,7 +219,8 @@ AuthCache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache
@router.get(
"/customer_portal/limits",
response_model=CustomerPortalLimits,
description="Get customer portal free-tier limits and current capacity. Proxy admins only.",
summary="Get Customer Management Limits",
description="Get Customer Management free-tier limits and current capacity. Proxy admins only.",
)
@management_endpoint_wrapper
async def get_customer_portal_limits(
@ -407,9 +414,12 @@ async def list_customer_teams(
page: Annotated[int, Query(ge=1)] = 1,
page_size: Annotated[int, Query(ge=1)] = 50,
parent_org_id: str | None = None,
sort: Annotated[
BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.")
] = None,
) -> ListResponse[Team]:
size: Final = min(page_size, 100)
result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size))
result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size, sort))
return list_response(result, request, page, size)
@ -470,7 +480,9 @@ async def get_customer_member(
@router.patch(
"/customer_members/{customer_member_id}",
response_model=CustomerMember,
description="Update a customer member and its budgets. Proxy admins only.",
description="Update a customer member and its budgets. Supply parent_team_ids to replace its teams within the "
"same organization, with the first team as default. Identity, budget counters and historical usage are retained. "
"Historical usage stays attributed to the original team. Proxy admins only.",
)
@management_endpoint_wrapper
async def update_customer_member(
@ -505,7 +517,10 @@ async def delete_customer_member(
"/customer_members",
response_model=ListResponse[CustomerMember],
dependencies=(Depends(reject_duplicate_list_params),),
description="List customer members with page-number pagination and resolved budgets. Proxy admins only.",
description=(
"List customer members with page-number pagination and resolved budgets. "
"search filters by customer member ID or alias before paginating. Proxy admins only."
),
)
@management_endpoint_wrapper
async def list_customer_members(
@ -515,9 +530,17 @@ async def list_customer_members(
page_size: Annotated[int, Query(ge=1)] = 50,
parent_team_id: str | None = None,
parent_org_id: str | None = None,
search: Annotated[str, Query(description="Case-insensitive substring of the customer member ID or alias")] = "",
sort: Annotated[
BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.")
] = None,
) -> ListResponse[CustomerMember]:
size: Final = min(page_size, 100)
result: Final = resolve(await hierarchy.list_members(parent_team_id, page, size, parent_org_id=parent_org_id))
result: Final = resolve(
await hierarchy.list_members(
parent_team_id, page, size, parent_org_id=parent_org_id, search=search.strip(), sort=sort
)
)
return list_response(result, request, page, size)

View file

@ -108,7 +108,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset(
"phoenix_project_name_override",
"phoenix_project_name",
"opik",
"team_model_limit_max_borrow_multiple",
"model_limit_max_borrow_multiple",
)
)
_NOT_COLUMNS: Final[frozenset[str]] = frozenset({"team_id", "metadata"})

View file

@ -301,7 +301,10 @@ async def add_team_callbacks(
- langfuse_secret: The secret for the Langfuse callback
- langfuse_host: The host for the Langfuse callback
- langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
- langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans
- langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace,
"llm_only" sends only model-call spans.
Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as
callback_settings.otel.langfuse_destination_span_scope.
- capture_message_content: For OTel v2 callbacks, "no_content" or "span_only".
An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting.
"span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out

View file

@ -238,7 +238,7 @@ from litellm.proxy.utils import (
)
from litellm.redis.stores.spend_counters import team_counter
from litellm.repositories.budget_repository import BudgetRepository
from litellm.repositories.chunked_in import delete_many_in, find_many_in
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
from litellm.repositories.organization_repository import OrganizationRepository
from litellm.repositories.prisma_protocols import TableActions
@ -919,6 +919,50 @@ class TeamMemberBudgetHandler:
TeamMemberBudgetHandler.clean_team_member_fields(updated_kv)
return updated_kv
@staticmethod
async def member_budgets_on_team_reset_window(
team_id: str, team_member_budget_id: str | None, prisma_client: PrismaClient
) -> tuple[str | None, tuple[str, ...]]:
"""The team default's reset window, and the budgets of members who take their cap from it and reset on a copy."""
team_default: Final = (
await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id})
if team_member_budget_id is not None
else None
)
team_window: Final = team_default.budget_duration if team_default is not None else None
memberships: Final = await _team_membership_db(prisma_client).find_many(
where={"team_id": team_id, "budget_id": {"not": team_member_budget_id}},
include={"litellm_budget_table": True},
)
return team_window, tuple(
m.litellm_budget_table.budget_id
for m in memberships
if m.litellm_budget_table is not None
and m.litellm_budget_table.max_budget is None
and m.litellm_budget_table.budget_duration == team_window
)
@staticmethod
async def move_members_to_team_reset_window(
team_id: str, team_member_budget_id: str | None, budget_duration: str | None, prisma_client: PrismaClient
) -> None:
"""Runs before the team default is written, so a retry after a failure still finds the members left behind."""
team_window, budget_ids = await TeamMemberBudgetHandler.member_budgets_on_team_reset_window(
team_id=team_id, team_member_budget_id=team_member_budget_id, prisma_client=prisma_client
)
if team_window == budget_duration or not budget_ids:
return
_ = await update_many_in(
_budget_db(prisma_client),
"budget_id",
budget_ids,
data={
"budget_duration": budget_duration,
"budget_reset_at": get_budget_reset_time(budget_duration) if budget_duration else None,
},
atomicity="per_chunk_ok",
)
@staticmethod
async def backfill_team_member_budget_entries(
team_id: str,
@ -2825,6 +2869,15 @@ async def update_team(
},
}
_current_member_budget_id: Final = (existing_team_row.metadata or {}).get("team_member_budget_id")
if "team_member_budget_duration" in _team_member_fields_in_request:
await TeamMemberBudgetHandler.move_members_to_team_reset_window(
team_id=data.team_id,
team_member_budget_id=_current_member_budget_id if isinstance(_current_member_budget_id, str) else None,
budget_duration=data.team_member_budget_duration,
prisma_client=prisma_client,
)
if _team_member_fields_in_request and TeamMemberBudgetHandler.should_create_budget(
team_member_budget=data.team_member_budget,
team_member_rpm_limit=data.team_member_rpm_limit,

View file

@ -22,7 +22,7 @@ from litellm.repositories.table_repositories import AuditLogRepository
from litellm.types.utils import StandardAuditLogPayload
if TYPE_CHECKING:
from litellm.integrations.s3_v3 import S3Logger
from litellm.integrations.s3_v2 import S3Logger
_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {}
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header"
@ -78,9 +78,9 @@ def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None":
instance: CustomLogger | S3Logger | None
audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None)
if name == "s3_v2" and audit_params is not None:
from litellm.integrations import s3_v3
from litellm.integrations import s3_v2
instance = s3_v3.S3Logger(s3_callback_params_override=audit_params)
instance = s3_v2.S3Logger(s3_callback_params_override=audit_params)
else:
from litellm.litellm_core_utils.litellm_logging import (
_init_custom_logger_compatible_class,

View file

@ -4,7 +4,7 @@ from dataclasses import dataclass
from datetime import datetime, timezone
from enum import Enum
from itertools import chain
from typing import TYPE_CHECKING, Final, Generic, Protocol, TypeVar
from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar
from uuid import uuid4
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
@ -20,7 +20,7 @@ from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
from litellm.proxy.list_api.common import escape_like
from litellm.proxy.management_helpers.customer_budget_usage import customer_budget_usage
from litellm.repositories.budget_repository import create_budget_row, update_budget_row
from litellm.repositories.chunked_in import count_in, delete_many_in, find_many_in
from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE, count_in, delete_many_in, find_many_in
from litellm.repositories.prisma_protocols import TableActions
from litellm.types.proxy.management_endpoints.customer_hierarchy import (
Budget,
@ -29,6 +29,7 @@ from litellm.types.proxy.management_endpoints.customer_hierarchy import (
BudgetTemplate,
CustomerMember,
MemberCreate,
MemberFields,
MemberPatch,
Organization,
OrganizationBudget,
@ -45,9 +46,53 @@ if TYPE_CHECKING:
from prisma.models import LiteLLM_BudgetWindowSpend
BudgetSort: TypeAlias = Literal["budget_asc", "budget_desc"]
def customer_budget_limit(budget: Budget | None) -> float | None:
if budget is None:
return None
limits: Final = budget.budget_limits or ()
monthly: Final = next(
(
limit.max_budget
for limit in limits
if limit.budget_duration.strip().lower() in ("1mo", "1month", "1months", "30d", "30days", "monthly")
),
None,
)
if monthly is not None:
return monthly
if budget.max_budget is not None:
return budget.max_budget
return next(
(
limit.max_budget
for limit in limits
if limit.budget_duration.strip().lower() in ("12mo", "12month", "12months")
),
None,
)
def _budget_sort_key(budget: Budget | None, sort: BudgetSort) -> tuple[bool, float]:
limit: Final = customer_budget_limit(budget)
return limit is None, (limit or 0) * (-1 if sort == "budget_desc" else 1)
class _BudgetSortRow(BaseModel):
model_config = ConfigDict(frozen=True)
entity_id: str
budget_id: str | None
budget_override_id: str | None
budget_override_expires_at: datetime | None
class ErrorCode(str, Enum):
FREE_TIER_LIMIT = "free_tier_limit"
CAPACITY_BUSY = "capacity_busy"
TRANSACTION_TIMEOUT = "transaction_timeout"
NOT_FOUND = "not_found"
PARENT_NOT_FOUND = "parent_not_found"
ALREADY_EXISTS = "already_exists"
@ -371,7 +416,7 @@ def team_response(row: PrismaRecord, now: datetime) -> Team:
def override_expiry(
row: Team | CustomerMember | None, payload: TeamPatch | MemberPatch, override_id: str | None
row: Team | CustomerMember | None, payload: TeamPatch | MemberFields, override_id: str | None
) -> datetime | None:
if override_id is None:
return None
@ -486,6 +531,12 @@ _MEMBER_INCLUDE: Final = {"budget": True, "budget_override": True}
_ID: Final = TypeAdapter(str)
def _search_filter(search: str, fields: tuple[str, ...]) -> Mapping[str, object]:
if not search:
return {}
return {"OR": tuple({field: {"contains": escape_like(search), "mode": "insensitive"}} for field in fields)}
def _member_ids(rows: Sequence[PrismaRecord]) -> tuple[str, ...]:
return tuple(_ID.validate_python(row.model_dump()["customer_member_id"]) for row in rows)
@ -995,9 +1046,20 @@ class CustomerHierarchyStore:
return await self.update_budget(current_id, payload.budget_override, actor)
async def update_member(self, member_id: str, payload: MemberPatch, actor: str) -> CustomerMember | Failure:
if payload.parent_team_ids is not None:
await self.lock_customer_identities()
member: Final = await self.get_member(member_id)
if isinstance(member, Failure):
return member
if payload.parent_team_ids is not None:
teams: Final = await find_many_in(self.teams, "customer_team_id", payload.parent_team_ids)
if len(teams) != len(payload.parent_team_ids):
return Failure(ErrorCode.PARENT_NOT_FOUND)
if any(team.model_dump()["parent_org_id"] != member.parent_org_id for team in teams):
return Failure(ErrorCode.CUSTOMER_ORG_MISMATCH)
added: Final = tuple(team for team in payload.parent_team_ids if team not in member.parent_team_ids)
if await self.member_capacity_reached(added):
return Failure(ErrorCode.FREE_TIER_LIMIT)
selected: Final = await self.member_template_patch(member, payload)
if isinstance(selected, Failure):
return selected
@ -1012,6 +1074,7 @@ class CustomerHierarchyStore:
where={"customer_member_id": member_id},
data={
**alias_patch(payload),
**({"parent_team_ids": payload.parent_team_ids} if payload.parent_team_ids is not None else {}),
**({"budget_id": selected} if "budget_id" in payload.model_fields_set else {}),
**(
{
@ -1028,16 +1091,7 @@ class CustomerHierarchyStore:
return await self.get_member(member_id)
async def list_organizations(self, page: int, page_size: int, search: str = "") -> Page[Organization]:
where: Final = (
{
"OR": tuple(
{field: {"contains": escape_like(search), "mode": "insensitive"}}
for field in ("alias", "customer_org_id")
)
}
if search
else {}
)
where: Final = _search_filter(search, ("alias", "customer_org_id"))
count: Final = await self.organizations.count(where=where)
rows: Final = await self.organizations.find_many(
where=where,
@ -1048,9 +1102,49 @@ class CustomerHierarchyStore:
)
return Page(await self.organization_responses(rows), count)
async def list_teams(self, parent_org_id: str | None, page: int, page_size: int) -> Page[Team]:
async def list_teams(
self, parent_org_id: str | None, page: int, page_size: int, sort: BudgetSort | None = None
) -> Page[Team]:
where: Final = {"parent_org_id": parent_org_id} if parent_org_id is not None else {}
count: Final = await self.teams.count(where=where)
if sort is not None:
ids: Final = await self._budget_sort_ids(
await self.db.query_raw(
"SELECT customer_team_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at "
'FROM "LiteLLM_CustomerTeamTable" WHERE ($1::text IS NULL OR parent_org_id = $1) '
"ORDER BY created_at ASC, customer_team_id ASC",
parent_org_id,
),
sort,
page,
page_size,
)
chunks: Final = tuple(
[
await self.teams.find_many(
# bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids.
where={"customer_team_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}},
include=_MEMBER_INCLUDE,
)
for start in range(0, len(ids), IN_LIST_CHUNK_SIZE)
]
)
rows_by_id: Final = {
_ID.validate_python(row.model_dump()["customer_team_id"]): row for row in chain.from_iterable(chunks)
}
return Page(
await customer_budget_usage(
tuple(
team_response(rows_by_id[identifier], self.now)
for identifier in ids
if identifier in rows_by_id
),
self.window_spend,
self.now,
self.settings,
),
count,
)
rows: Final = await self.teams.find_many(
where=where,
skip=(page - 1) * page_size,
@ -1060,19 +1154,72 @@ class CustomerHierarchyStore:
)
return Page(
await customer_budget_usage(
tuple(team_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings
tuple(team_response(row, self.now) for row in rows),
self.window_spend,
self.now,
self.settings,
),
count,
)
async def list_members(
self, parent_team_id: str | None, page: int, page_size: int, *, parent_org_id: str | None = None
self,
parent_team_id: str | None,
page: int,
page_size: int,
*,
parent_org_id: str | None = None,
search: str = "",
sort: BudgetSort | None = None,
) -> Page[CustomerMember]:
where: Final[Mapping[str, object]] = {
**({"parent_team_ids": {"has": parent_team_id}} if parent_team_id is not None else {}),
**({"parent_org_id": parent_org_id} if parent_org_id is not None else {}),
**_search_filter(search, ("alias", "customer_member_id")),
}
count: Final = await self.members.count(where=where)
if sort is not None:
ids: Final = await self._budget_sort_ids(
await self.db.query_raw(
"SELECT customer_member_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at "
'FROM "LiteLLM_CustomerMemberTable" WHERE ($1::text IS NULL OR parent_org_id = $1) '
"AND ($2::text IS NULL OR parent_team_ids @> ARRAY[$2]::text[]) "
"AND ($3 = '' OR alias ILIKE $3 OR customer_member_id ILIKE $3) "
"ORDER BY created_at ASC, customer_member_id ASC",
parent_org_id,
parent_team_id,
f"%{escape_like(search)}%" if search else "",
),
sort,
page,
page_size,
)
chunks: Final = tuple(
[
await self.members.find_many(
# bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids.
where={"customer_member_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}},
include=_MEMBER_INCLUDE,
)
for start in range(0, len(ids), IN_LIST_CHUNK_SIZE)
]
)
rows_by_id: Final = {
_ID.validate_python(row.model_dump()["customer_member_id"]): row for row in chain.from_iterable(chunks)
}
return Page(
await customer_budget_usage(
tuple(
member_response(rows_by_id[identifier], self.now)
for identifier in ids
if identifier in rows_by_id
),
self.window_spend,
self.now,
self.settings,
),
count,
)
rows: Final = await self.members.find_many(
where=where,
skip=(page - 1) * page_size,
@ -1082,11 +1229,36 @@ class CustomerHierarchyStore:
)
return Page(
await customer_budget_usage(
tuple(member_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings
tuple(member_response(row, self.now) for row in rows),
self.window_spend,
self.now,
self.settings,
),
count,
)
async def _budget_sort_ids(self, rows: object, sort: BudgetSort, page: int, page_size: int) -> tuple[str, ...]:
candidates: Final = TypeAdapter(tuple[_BudgetSortRow, ...]).validate_python(rows)
references: Final = chain.from_iterable((row.budget_id, row.budget_override_id) for row in candidates)
budgets: Final = {
_ID.validate_python(row.model_dump()["budget_id"]): row.model_dump()
for row in await find_many_in(self.budgets, "budget_id", (ref for ref in references if ref is not None))
}
def key(row: _BudgetSortRow) -> tuple[bool, float]:
template: Final = related_budget(budgets.get(row.budget_id or ""), self.now)
override: Final = related_budget(
budgets.get(row.budget_override_id or ""),
self.now,
template.max_budget if template is not None else None,
)
return _budget_sort_key(
effective_member_budget(template, override, row.budget_override_expires_at, self.now), sort
)
ordered: Final = sorted(candidates, key=key)
return tuple(row.entity_id for row in ordered[(page - 1) * page_size : page * page_size])
async def _delete_unused_budgets(self, budget_ids: Sequence[str]) -> None:
where: Final[Mapping[str, object]] = {field: {"none": {}} for field in _BUDGET_REFERENCE_FIELDS}
await delete_many_in(
@ -1098,8 +1270,10 @@ class CustomerHierarchyStore:
chunk_size=_DELETE_BATCH_SIZE,
)
async def _delete_team_member_batch(self, team_id: str) -> bool:
rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE)
async def _delete_team_member_batch(self, team_ids: tuple[str, ...]) -> bool:
rows: Final = await self.members.find_many(
where={"parent_team_ids": {"has_some": list(team_ids)}}, take=_DELETE_BATCH_SIZE
)
members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows)
if not members:
return False
@ -1111,9 +1285,9 @@ class CustomerHierarchyStore:
await self._delete_unused_budgets(budget_ids)
return True
async def _delete_team_members(self, team_id: str) -> None:
async def _delete_team_members(self, team_ids: tuple[str, ...]) -> None:
while True:
if not await self._delete_team_member_batch(team_id):
if not await self._delete_team_member_batch(team_ids):
return
async def _delete_organization_team_batch(self, organization_id: str) -> bool:
@ -1121,9 +1295,8 @@ class CustomerHierarchyStore:
teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows)
if not teams:
return False
for team in teams:
await self._delete_team_members(team.customer_team_id)
team_ids: Final = tuple(team.customer_team_id for team in teams)
await self._delete_team_members(team_ids)
team_budget_ids: Final = chain.from_iterable((team.budget_id, team.budget_override_id) for team in teams)
budget_ids: Final = tuple(budget_id for budget_id in team_budget_ids if budget_id is not None)
await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction")
@ -1154,11 +1327,12 @@ class CustomerHierarchyStore:
return None
async def delete_team(self, team_id: str, *, cascade: bool = False) -> None | Failure:
await self.lock_customer_identities()
team: Final = await self.get_team(team_id)
if isinstance(team, Failure):
return team
if cascade:
await self._delete_team_members(team_id)
await self._delete_team_members((team_id,))
elif await self.members.count(where={"parent_team_ids": {"has": team_id}}):
return Failure(ErrorCode.HAS_CHILDREN)
await self.teams.delete(where={"customer_team_id": team_id})
@ -1166,6 +1340,7 @@ class CustomerHierarchyStore:
return None
async def delete_organization(self, organization_id: str, *, cascade: bool = False) -> None | Failure:
await self.lock_customer_identities()
organization: Final = await self.get_organization(organization_id)
if isinstance(organization, Failure):
return organization
@ -1196,7 +1371,13 @@ class CustomerHierarchy:
*,
operation_type: TransactionOperation = TransactionOperation.READ_WRITE,
) -> T | Failure:
from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError
from prisma.errors import (
DataError,
ForeignKeyViolationError,
RecordNotFoundError,
TransactionExpiredError,
UniqueViolationError,
)
try:
async with self.db.tx() as tx:
@ -1205,6 +1386,8 @@ class CustomerHierarchy:
return Failure(ErrorCode.ALREADY_EXISTS)
except RecordNotFoundError:
return Failure(ErrorCode.NOT_FOUND)
except TransactionExpiredError:
return Failure(ErrorCode.TRANSACTION_TIMEOUT)
except ForeignKeyViolationError:
return Failure(
ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND
@ -1278,15 +1461,24 @@ class CustomerHierarchy:
return await self.transaction(lambda store: store.list_organizations(page, page_size, search))
async def list_teams(
self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50
self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50, sort: BudgetSort | None = None
) -> Page[Team] | Failure:
return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size))
return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size, sort))
async def list_members(
self, parent_team_id: str | None = None, page: int = 1, page_size: int = 50, *, parent_org_id: str | None = None
self,
parent_team_id: str | None = None,
page: int = 1,
page_size: int = 50,
*,
parent_org_id: str | None = None,
search: str = "",
sort: BudgetSort | None = None,
) -> Page[CustomerMember] | Failure:
return await self.transaction(
lambda store: store.list_members(parent_team_id, page, page_size, parent_org_id=parent_org_id)
lambda store: store.list_members(
parent_team_id, page, page_size, parent_org_id=parent_org_id, search=search, sort=sort
)
)
async def add_template(self, organization_id: str, payload: BudgetTemplate, actor: str) -> Organization | Failure:

View file

@ -460,6 +460,8 @@ async def _resolve_member_budget_id(
changes reach them; ``/team/member_update`` clones that row on first write.
A lone ``budget_duration`` clones the default with the reset window
overridden, or creates a window-only budget when there is no team default.
``allowed_models`` with no cap creates a budget that takes the default's reset
window and leaves the cap to the default.
With nothing set the member gets no budget, though ``add_new_member`` still writes its membership row.
"""
has_explicit_limit: Final = max_budget_in_team is not None or allowed_models is not None
@ -493,14 +495,34 @@ async def _resolve_member_budget_id(
budget_data["max_budget"] = max_budget_in_team
if allowed_models is not None:
budget_data["allowed_models"] = allowed_models
if budget_duration is not None:
budget_data["budget_duration"] = budget_duration
budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=budget_duration)
member_budget_duration: Final = budget_duration or (
await _team_default_budget_duration(prisma_client, default_team_budget_id, tx)
if max_budget_in_team is None and default_team_budget_id is not None
else None
)
if member_budget_duration is not None:
budget_data["budget_duration"] = member_budget_duration
budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=member_budget_duration)
budget_table: Final[_PrismaBudgetTable] = _budget_table(prisma_client, tx)
response: Final = await budget_table.create(data=budget_data)
return response.budget_id
async def _team_default_budget_duration(
prisma_client: PrismaClient, default_team_budget_id: str, tx: MemberWriteTx | None
) -> str | None:
"""The reset period a member with models of their own and no cap of their own must carry.
Their cap is read from the team default at auth time, but spend is only reset for a
member whose own budget row has a reset date.
"""
default_budget: Final = await _budget_table(prisma_client, tx).find_unique(
where={"budget_id": default_team_budget_id}
)
duration: Final = default_budget.model_dump().get("budget_duration") if default_budget is not None else None
return duration if isinstance(duration, str) and duration else None
async def _append_team_id_if_absent(
prisma_client: PrismaClient, user_id: str, team_id: str, tx: MemberWriteTx | None = None
) -> None:

View file

@ -598,6 +598,13 @@ def _fal_target(endpoint: str) -> httpx.URL:
)
def _fal_upstream_endpoint(target: httpx.URL) -> str:
base_url: Final = httpx.URL(fal_ai_queue_base())
base_path: Final = base_url.copy_with(path=base_url.path).raw_path.partition(b"?")[0].decode("ascii").strip("/")
upstream_path: Final = target.raw_path.partition(b"?")[0].decode("ascii").lstrip("/")
return upstream_path.removeprefix(f"{base_path}/") if base_path else upstream_path
@router.api_route(
"/fal_ai/{endpoint:path}",
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
@ -619,10 +626,14 @@ async def fal_ai_proxy_route(
status_code=401,
detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured",
)
if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await read_request_body(request)) is None:
upstream_endpoint: Final = _fal_upstream_endpoint(updated_url)
if (
"/requests/" not in upstream_endpoint
and fal_ai_passthrough_cost(upstream_endpoint, await read_request_body(request)) is None
):
raise HTTPException(
status_code=400,
detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai",
detail=f"fal_ai/{upstream_endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai",
)
endpoint_func: Final = create_pass_through_route(
endpoint=endpoint,

View file

@ -79,7 +79,7 @@ def _build_db_connection_url_params(
socket_timeout: float | None = None,
disable_prepared_statements: bool = False,
extra_params: dict | None = None,
) -> dict:
) -> dict[str, str | int | float]:
"""Build the Prisma DATABASE_URL query params controlling connection pool behavior.
`connect_timeout` / `socket_timeout` map to the Prisma URL params of the same
@ -91,7 +91,7 @@ def _build_db_connection_url_params(
`extra_params` is an untyped passthrough — keys it provides win over the
named arguments above, so it can be used to override any default we set here.
"""
params: Final[dict] = {
params: Final[dict[str, str | int | float]] = {
"connection_limit": connection_limit,
}
if pool_timeout is not None:
@ -1637,11 +1637,13 @@ def run_server(
db_connection_timeout = LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value
if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None:
from litellm.proxy.db.connection_budget import ConnectionBudget
from litellm.proxy.db.db_url_settings import (
DISABLE_PREPARED_STATEMENTS_ENV_VAR,
add_missing_query_params,
idle_lifetime_params,
postgres_connection_budget_message,
prisma_default_params,
reader_shareable_params,
translate_libpq_ssl_params,
unsupported_db_scheme,
@ -1676,7 +1678,13 @@ def run_server(
disable_prepared_statements=disable_prepared_statements,
extra_params=db_extra_connection_params,
)
writer_budget: Final = ConnectionBudget.of(connection_url_params["connection_limit"])
rust_writer_connections: Final = 0 if writer_budget is None else writer_budget.rust
prisma_url_params: Final = (
connection_url_params if writer_budget is None else writer_budget.prisma_params(connection_url_params)
)
lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime"))
prisma_defaults: Final = prisma_default_params(lifetime_params, rust_writer_connections)
if os.getenv("DATABASE_URL", None) is not None:
database_url = get_secret("DATABASE_URL", default_value=None)
resolved_url: Final[str | None] = str(database_url) if database_url else None
@ -1693,10 +1701,10 @@ def run_server(
)
modified_url = append_query_params(
writer_url,
connection_url_params,
dict(prisma_url_params),
)
os.environ["DATABASE_URL"] = translate_libpq_ssl_params(
add_missing_query_params(modified_url, lifetime_params)
add_missing_query_params(modified_url, prisma_defaults)
)
if os.getenv("DIRECT_URL", None) is not None:
database_url = os.getenv("DIRECT_URL")
@ -1710,6 +1718,14 @@ def run_server(
# on the writer. Anything pinned on the replica URL wins, unlike the
# writer where the config is applied on top.
read_replica_url: Final[str | None] = os.getenv("DATABASE_URL_READ_REPLICA")
reader_budget: Final = (
ConnectionBudget.of(
_url_query_value(read_replica_url, "connection_limit") or connection_url_params["connection_limit"]
)
if read_replica_url
else None
)
reader_prisma_limit: Final = None if reader_budget is None else reader_budget.prisma_limit
if read_replica_url:
reader_options: Final[str] = _pg_options_with_timeouts(
_url_query_value(read_replica_url, "options"),
@ -1717,27 +1733,34 @@ def run_server(
db_lock_timeout,
db_idle_in_transaction_timeout,
)
reader_url: Final = (
_with_query_value(read_replica_url, "options", reader_options)
if reader_options
else read_replica_url
)
os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params(
add_missing_query_params(
add_missing_query_params(
_with_query_value(read_replica_url, "options", reader_options)
if reader_options
else read_replica_url,
reader_shareable_params(connection_url_params),
reader_url
if reader_prisma_limit is None
else _with_query_value(reader_url, "connection_limit", str(reader_prisma_limit)),
reader_shareable_params(prisma_url_params),
),
lifetime_params,
prisma_defaults,
)
)
print_startup_message(
postgres_connection_budget_message(
writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit")
or f"{db_connection_pool_limit}",
rust_writer_limit=rust_writer_connections,
reader_limit=(
_url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit")
or f"{db_connection_pool_limit}"
)
if read_replica_url
else None,
rust_reader_limit=0 if reader_budget is None else reader_budget.rust,
num_workers=f"{1 if run_hypercorn else num_workers}",
)
)

View file

@ -213,7 +213,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
snapshot_tuning_baselines,
tuning_limit_violation,
)
from litellm.rust_bridge.catalog import Target
from litellm.rust_bridge.catalog import NATIVE_MESSAGES_TARGETS, Target
from litellm.types.utils import (
PRICING_OVERRIDES_KEY,
CallTypes,
@ -681,6 +681,7 @@ from litellm.proxy.config_resolvers.settings_rules import (
from litellm.proxy.config_resolvers.sso import SSO_FIELD_ENV_VARS
from litellm.proxy.container_endpoints.endpoints import router as container_router
from litellm.proxy.credential_endpoints.endpoints import router as credential_router
from litellm.proxy.db.connection_budget import RUST_RESERVED_CONNECTIONS
from litellm.proxy.db.create_views import SupportsRawQueries
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import (
@ -1620,7 +1621,12 @@ async def _connect_to_count_stored_values(resources: AsyncExitStack) -> Supports
async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) -> database_runtime.Database | None:
if client is None:
return None
database: Final = await database_runtime.Database.open(client)
database: Final = await database_runtime.Database.open(
client,
os.environ["DATABASE_URL"],
os.environ.get("DATABASE_URL_READ_REPLICA") or None,
RUST_RESERVED_CONNECTIONS,
)
resources.push_async_callback(database.aclose)
return database
@ -7446,7 +7452,7 @@ class ProxyConfig:
if key == "request_timeout":
litellm.request_timeout_explicitly_set = True
if key in {"s3_audit_callback_params", "s3_callback_params"}:
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
from litellm.litellm_core_utils.litellm_logging import (
_in_memory_loggers,
)
@ -16124,7 +16130,6 @@ def _count_context_management(
_GATEWAY_EDIT_ROUTES: Final = frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})
_MESSAGES_ROUTES: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH})
def _messages_route(deployment: _TokenCountDeployment, request_model: str) -> Target:
@ -16663,7 +16668,7 @@ async def _count_request_tokens(
local_view: Final = (
None
if route not in _MESSAGES_ROUTES or context_edits is None or request_messages is None
if route not in NATIVE_MESSAGES_TARGETS or context_edits is None or request_messages is None
else await _bridged_request_view(
request_messages,
system,

View file

@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable {
updated_at DateTime @default(now()) @updatedAt
updated_by String
@@index([parent_org_id])
@@index([budget_id])
@@index([budget_override_id])
@@index([spend])
}
@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable {
updated_at DateTime @default(now()) @updatedAt
updated_by String
@@index([parent_org_id])
@@index([parent_team_ids], type: Gin)
@@index([budget_id])
@@index([budget_override_id])
@@index([spend])
}
@ -2308,7 +2313,7 @@ enum AccountingDeliveryState {
}
model LiteLLM_SpendLogDelivery {
delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text"))
delivery_id String @id @default(uuid())
request_id String
state AccountingDeliveryState @default(pending)
accounting_payload Json

View file

@ -23,6 +23,7 @@ from litellm.litellm_core_utils.duration_parser import (
duration_in_seconds,
get_annual_budget_window_start,
is_annual_budget_duration,
normalize_budget_window_start,
)
from litellm.litellm_core_utils.duration_parser import (
get_budget_window_start as calendar_budget_window_start,
@ -30,7 +31,6 @@ from litellm.litellm_core_utils.duration_parser import (
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
from litellm.models.budget import BudgetLimit
from litellm.proxy._types import (
Litellm_EntityType,
LiteLLM_TeamMembership,
LiteLLM_TeamTable,
LiteLLM_UserTable,
@ -60,6 +60,11 @@ from litellm.proxy.spend_tracking.input_tokens import (
models_counted_for,
openai_chat_pdf_file_tokens,
)
from litellm.proxy.spend_tracking.native_budgets import (
new_reservation_id,
reserve_native,
settle_native_reservation,
)
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.proxy.utils import PrismaClient, ProxyLogging
from litellm.redis.stores.spend_counters import (
@ -145,21 +150,6 @@ def _customer_window_counter(
)
_COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
"Key": Litellm_EntityType.KEY.value,
"Team": Litellm_EntityType.TEAM.value,
"TeamMember": Litellm_EntityType.TEAM_MEMBER.value,
"User": Litellm_EntityType.USER.value,
"EndUser": Litellm_EntityType.END_USER.value,
"Tag": Litellm_EntityType.TAG.value,
"Model access group": Litellm_EntityType.MODEL_ACCESS_GROUP.value,
"Organization": Litellm_EntityType.ORGANIZATION.value,
"Project": Litellm_EntityType.PROJECT.value,
"customer_org": Litellm_EntityType.CUSTOMER_ORG.value,
"customer_team": Litellm_EntityType.CUSTOMER_TEAM.value,
"customer_member": Litellm_EntityType.CUSTOMER_MEMBER.value,
}
_CACHED_VALUE: Final = TypeAdapter(object)
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
@ -203,7 +193,11 @@ def get_reserved_counter_keys(budget_reservation: dict | None) -> set:
_lease_renewals: Final[set[asyncio.Task[None]]] = set() # mutable-ok: asyncio only weak-refs pending tasks
def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], counter_keys: frozenset[str]) -> None:
def start_reservation_lease_renewal(
budget_reservation: Mapping[str, object],
counter_keys: frozenset[str],
request_task: asyncio.Task[object] | None = None,
) -> None:
"""A reservation lives inside spend counter keys that expire on their Redis TTL. Renew the TTL
while the request is in flight so a request longer than the TTL does not drop its
reservation and admit concurrent requests against the DB floor on any worker."""
@ -217,7 +211,7 @@ def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], c
budget_reservation=budget_reservation,
counter_keys=counter_keys,
interval=redis_ttl / 2,
request_task=asyncio.current_task(),
request_task=request_task or asyncio.current_task(),
)
)
_lease_renewals.add(task)
@ -243,80 +237,6 @@ async def _renew_reservation_lease(
await refresh_spend_counter_ttl(counter_key=counter_key)
def _key_reservation_should_release_for_throttle(counter_key: str, valid_token: UserAPIKeyAuth | None) -> bool:
"""
Whether an over-budget key's own ``max_budget`` reservation should be
released rather than blocked, because the key opted into throttling: the
rate limiter slows it instead. Only the key's own ``max_budget`` counter is
exempt; team/user/window counters still enforce normally, and under-budget
requests never reach this branch so their concurrent-overspend protection is
untouched.
"""
if valid_token is None:
return False
return counter_key == key_counter(str(valid_token.token)) and should_throttle_budget_exceeded(valid_token)
async def _apply_over_budget_reservation_policy(
counter: _BudgetCounter,
valid_token: UserAPIKeyAuth | None,
entry: dict[str, float | str],
applied_entries: list[dict[str, float | str]],
reservation_cost: float,
current_spend: float,
fail_closed_budget_enforcement: bool = False,
) -> float:
"""
Decide what to do when a counter is over budget, and return the reservation
cost to carry into the next counter. Three outcomes: an over-budget key that
opted into throttling releases its own reservation (the rate limiter slows
it) and keeps the cost; a partially-remaining budget resizes the reservation
down to what is left, unless strict enforcement is on, because the known
estimate already does not fit; anything else hard-blocks by raising.
"""
if _key_reservation_should_release_for_throttle(counter.counter_key, valid_token):
await _release_applied_entries_best_effort(entries=[entry], default_reserved_cost=reservation_cost)
applied_entries.remove(entry)
return reservation_cost
remaining_before_reservation: Final = counter.max_budget - (current_spend - reservation_cost)
if remaining_before_reservation <= 1e-12:
_raise_counter_budget_exceeded(counter=counter, current_cost=current_spend)
if fail_closed_budget_enforcement and current_spend - counter.max_budget > 1e-12:
_raise_counter_budget_exceeded(
counter=counter,
current_cost=current_spend - reservation_cost,
estimated_cost=reservation_cost,
)
await _resize_applied_reservation(
entries=applied_entries,
current_reserved_cost=reservation_cost,
new_reserved_cost=remaining_before_reservation,
)
return remaining_before_reservation
def _raise_counter_budget_exceeded(
counter: _BudgetCounter,
current_cost: float,
estimated_cost: float | None = None,
) -> NoReturn:
estimate_detail: Final = "" if estimated_cost is None else f"Estimated request cost: {estimated_cost}, "
raise litellm.BudgetExceededError(
current_cost=current_cost,
max_budget=counter.max_budget,
message=(
"Budget has been exceeded! "
f"{counter.entity_type}={counter.entity_id} "
f"Current cost: {current_cost}, "
f"{estimate_detail}"
f"Max budget: {counter.max_budget}"
),
entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type),
entity_id=counter.spend_log_entity_id or counter.entity_id,
)
async def reserve_budget_for_request(
request_body: dict,
route: str,
@ -365,7 +285,7 @@ async def reserve_budget_for_request(
path_params=path_params,
)
reservation_cost = estimate_request_max_cost(
reservation_cost: Final = estimate_request_max_cost(
request_body=request_body,
route=route,
llm_router=llm_router,
@ -377,51 +297,64 @@ async def reserve_budget_for_request(
# In that case we fall back to read-time enforcement only.
if reservation_cost is None or reservation_cost <= 0:
return None
applied_entries: Final[list[dict[str, float | str]]] = []
try:
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
reservable: Final = await _initialize_reservation_counters(
counters=counters,
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
reservation_cost = await _reserve_reservable_counters(
reservable=reservable,
valid_token=valid_token,
applied_entries=applied_entries,
reservation_cost=reservation_cost,
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
except Exception:
await _release_applied_entries_best_effort(
entries=applied_entries,
default_reserved_cost=reservation_cost,
)
raise
if not applied_entries:
return None
input_cost: Final = estimate_request_input_cost(
request_body=request_body,
route=route,
llm_router=llm_router,
return await _native_budget_plan(
counters=counters,
valid_token=valid_token,
reservation_cost=reservation_cost,
input_token_counts=input_token_counts,
path_params=path_params,
input_cost=estimate_request_input_cost(
request_body=request_body,
route=route,
llm_router=llm_router,
input_token_counts=input_token_counts,
path_params=path_params,
),
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
budget_reservation: Final = {
"reserved_cost": reservation_cost,
"entries": applied_entries,
"finalized": False,
"callback_bound": False,
"input_cost": min(float(input_cost or 0.0), reservation_cost),
"input_tokens": max(input_token_counts.values(), default=None),
}
_start_reservation_lease_renewal(
budget_reservation=budget_reservation,
counter_keys=frozenset(get_reserved_counter_keys(budget_reservation=budget_reservation)),
async def _native_budget_plan(
counters: Sequence[_BudgetCounter],
valid_token: UserAPIKeyAuth,
reservation_cost: float,
input_token_counts: Mapping[str, int],
input_cost: float | None,
fail_closed_budget_enforcement: bool,
) -> dict[str, object] | None: # mutable-ok: settlement stamps finalized on the reservation it returns
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
reservable: Final = await _initialize_reservation_counters(
counters=counters,
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
if not reservable:
return None
input_tokens: Final = max(input_token_counts.values(), default=None)
return await reserve_native(
{
"counters": [
{
"counter_key": counter.counter_key,
"max_budget": counter.max_budget,
"fallback_spend": counter.fallback_spend,
"entity_type": counter.entity_type,
"entity_id": counter.entity_id,
"spend_log_entity_id": counter.spend_log_entity_id,
}
for counter in reservable
],
"reservation_cost": float(reservation_cost),
"input_cost": float(input_cost or 0.0),
"input_tokens": input_tokens,
"throttle_counter": key_counter(str(valid_token.token))
if should_throttle_budget_exceeded(valid_token)
else None,
"fail_closed": fail_closed_budget_enforcement,
"status_code": int(litellm.budget_exceeded_status_code),
"reservation_id": new_reservation_id(),
},
valid_token,
{counter.counter_key: counter.reset_at.isoformat() for counter in reservable if counter.reset_at is not None},
)
return budget_reservation
async def reconcile_budget_reservation(
@ -430,19 +363,11 @@ async def reconcile_budget_reservation(
finalize: bool = True,
apply_consistent: bool = True,
) -> tuple[PendingSpendIncrement, ...]:
"""Settle every reserved counter on ``actual_cost``. With ``apply_consistent`` False the adjustments for
counters that still hold the reservation are returned instead of written, so the caller can pipeline them with
its own increments and then call ``stamp_budget_reservation_actual_cost``."""
if not budget_reservation or budget_reservation.get("finalized") is True:
return ()
reserved_cost: Final = float(budget_reservation.get("reserved_cost") or 0.0)
actual: Final = float(actual_cost or 0.0)
pending: Final = await _set_reserved_entries_actual_cost(
entries=budget_reservation.get("entries") or [],
actual_cost=actual,
default_reserved_cost=reserved_cost,
apply_consistent=apply_consistent,
_ = float(budget_reservation.get("reserved_cost") or 0.0)
pending: Final = await settle_native_reservation(
budget_reservation, float(actual_cost or 0.0), apply_consistent=apply_consistent
)
if finalize:
budget_reservation["finalized"] = True
@ -1088,121 +1013,6 @@ async def _initialize_reservation_counter(counter: _BudgetCounter) -> None:
raise _CounterReservationUnavailable
async def _reserve_reservable_counters(
reservable: Sequence[_BudgetCounter],
valid_token: UserAPIKeyAuth | None,
applied_entries: list[dict[str, float | str]],
reservation_cost: float,
fail_closed_budget_enforcement: bool,
) -> float:
"""Charge the counters group by group (see ``_reservation_groups``), settling the over-budget policy on each
group before the next is charged, and hand back the reservation cost the policy left standing."""
current_spend_by_counter_key: Final = {
counter.counter_key: await _get_current_counter_value(counter=counter) for counter in reservable
}
for group in _reservation_groups(
counters=reservable,
current_spend_by_counter_key=current_spend_by_counter_key,
reservation_cost=reservation_cost,
):
charged_cost = reservation_cost
entries = tuple(_counter_to_reservation_entry(counter=counter, reserved_cost=charged_cost) for counter in group)
applied_entries.extend(entries)
reserved_values = await _reserve_counters(counters=group, entries=entries, reservation_cost=charged_cost)
if reserved_values is None:
for entry in entries:
applied_entries.remove(entry)
if fail_closed_budget_enforcement:
_raise_reservation_unavailable(counter_key=group[0].counter_key)
continue
for counter, entry, reserved_value in zip(group, entries, reserved_values):
if entry not in applied_entries:
continue
if reserved_value is not None:
current_spend = reserved_value - (charged_cost - reservation_cost)
else:
current_spend = current_spend_by_counter_key[counter.counter_key] + reservation_cost
if current_spend > counter.max_budget:
reservation_cost = await _apply_over_budget_reservation_policy(
counter=counter,
valid_token=valid_token,
entry=entry,
applied_entries=applied_entries,
reservation_cost=reservation_cost,
current_spend=current_spend,
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
)
return reservation_cost
def _reservation_groups(
counters: Sequence[_BudgetCounter],
current_spend_by_counter_key: Mapping[str, float],
reservation_cost: float,
) -> tuple[tuple[_BudgetCounter, ...], ...]:
"""Every counter the batch read says still has room for the estimate is charged in one pipeline. As soon as one
does not, the counters are charged one at a time so the over-budget policy settles each before the next is
touched, and a rejection charges nothing after it."""
if not counters:
return ()
if all(
current_spend_by_counter_key[counter.counter_key] + reservation_cost <= counter.max_budget
for counter in counters
):
return (tuple(counters),)
return tuple((counter,) for counter in counters)
async def _reserve_counters(
counters: Sequence[_BudgetCounter],
entries: Sequence[dict[str, float | str]],
reservation_cost: float,
) -> tuple[float | None, ...] | None:
"""One INCRBYFLOAT pipeline reserves every counter. When it fails each counter is dropped, and one that cannot
be dropped is released instead in case its increment landed, so nothing is left to release by the caller."""
from litellm.proxy.proxy_server import invalidate_spend_counter, run_spend_counter_pipeline
if not counters:
return ()
try:
reserved: Final = await run_spend_counter_pipeline(
pending=tuple(
PendingSpendIncrement(counter_key=counter.counter_key, increment=reservation_cost)
for counter in counters
)
)
except Exception:
verbose_proxy_logger.warning(
"Skipping budget reservation for %s because spend counter reservation failed",
tuple(counter.counter_key for counter in counters),
exc_info=True,
)
for counter, entry in zip(counters, entries):
try:
await invalidate_spend_counter(counter_key=counter.counter_key)
except Exception:
verbose_proxy_logger.warning(
"Failed to invalidate spend counter after budget reservation failure for %s",
counter.counter_key,
exc_info=True,
)
await _release_applied_entries_best_effort(
entries=[entry],
default_reserved_cost=reservation_cost,
)
return None
return tuple(reserved) + (None,) * (len(counters) - len(reserved))
async def _get_current_counter_value(counter: _BudgetCounter) -> float:
from litellm.proxy.proxy_server import get_current_spend
return await get_current_spend(
counter_key=counter.counter_key,
fallback_spend=counter.fallback_spend,
)
def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_scope:
"""Each counter is read once, then written, so one MGET up front serves every read in the loop."""
from litellm.proxy.proxy_server import spend_counter_store
@ -1210,176 +1020,17 @@ def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_s
return spend_counter_batch_scope(spend_counter_store, counter_keys=counter_keys)
@dataclass(frozen=True, slots=True)
class _EntryAdjustment:
entry: dict[str, float | str]
counter_key: str
target_adjustment: float
adjustment: float
actual_cost: float
def _entry_settled_cost(entry: Mapping[str, object], actual_cost: float, settlement_time: datetime | None) -> float:
reset_at: Final = _coerce_datetime(entry.get("window_reset_at"))
return 0.0 if settlement_time is not None and reset_at is not None and reset_at <= settlement_time else actual_cost
def _entry_adjustment(
entry: dict[str, float | str],
actual_cost: float,
default_reserved_cost: float,
settlement_time: datetime | None = None,
) -> _EntryAdjustment | None:
counter_key: Final = entry.get("counter_key")
if counter_key is None:
return None
settled_cost: Final = _entry_settled_cost(entry, actual_cost, settlement_time)
target_adjustment: Final = settled_cost - _get_entry_reserved_cost(
entry=entry, default_reserved_cost=default_reserved_cost
)
adjustment: Final = target_adjustment - float(entry.get("applied_adjustment") or 0.0)
if adjustment == 0:
return None
return _EntryAdjustment(
entry=entry,
counter_key=str(counter_key),
target_adjustment=target_adjustment,
adjustment=adjustment,
actual_cost=settled_cost,
)
async def _set_reserved_entries_actual_cost(
entries: list[dict],
actual_cost: float,
default_reserved_cost: float,
reseed_on_inconsistent: bool = True,
apply_consistent: bool = True,
) -> tuple[PendingSpendIncrement, ...]:
"""Every reserved counter is read from one MGET and the consistent adjustments go out in one pipeline, or are
returned unwritten when ``apply_consistent`` is False. A counter that was flushed or reseeded since reservation
is settled on its own after the pipeline."""
from litellm.proxy.proxy_server import increment_spend_counters_pipeline
settlement_time: Final = datetime.now(timezone.utc) if reseed_on_inconsistent else None
with _counters_batch_scope(frozenset(str(entry["counter_key"]) for entry in entries if "counter_key" in entry)):
adjustments: Final = tuple(
adjustment
for entry in entries
if (adjustment := _entry_adjustment(entry, actual_cost, default_reserved_cost, settlement_time)) is not None
)
consistent: Final = tuple(
await asyncio.gather(
*(
_counter_can_apply_adjustment(counter_key=item.counter_key, adjustment=item.adjustment)
for item in adjustments
)
)
)
inconsistent: Final = tuple(item for item, ok in zip(adjustments, consistent) if not ok)
if inconsistent and not reseed_on_inconsistent:
# Pre-call admission resize: the in-flight reservation cost is not yet
# persisted, so the DB floor would discard it. Keep the original
# fail-closed behavior (raise -> reserve_budget_for_request releases and
# denies) rather than admitting against an inconsistent counter.
raise RuntimeError(
f"Cannot resize budget reservation against inconsistent counter {inconsistent[0].counter_key}"
)
applicable: Final = tuple(item for item, ok in zip(adjustments, consistent) if ok)
applicable_pending: Final = tuple(
PendingSpendIncrement(counter_key=item.counter_key, increment=item.adjustment) for item in applicable
)
if apply_consistent:
await increment_spend_counters_pipeline(pending=applicable_pending)
for item in inconsistent:
await _reseed_reserved_entry(item=item, actual_cost=item.actual_cost)
for item in adjustments if apply_consistent else inconsistent:
item.entry["applied_adjustment"] = item.target_adjustment
return () if apply_consistent else applicable_pending
async def _reseed_reserved_entry(item: _EntryAdjustment, actual_cost: float) -> None:
"""Post-call reconcile / release of a counter that was flushed, expired or reseeded between reservation and
reconcile: the optimistic delta no longer applies, so reseed from the DB floor and add the settled cost, since
increment_spend_counters skips reserved keys. The reconcile runs before this request's spend is enqueued to the
DB, so the reseeded floor excludes it."""
async def reseed_reserved_counter(counter_key: str, actual_cost: float) -> None:
from litellm.proxy.proxy_server import increment_spend_counter_cache, reseed_spend_counter_from_db
reseeded: Final = await reseed_spend_counter_from_db(counter_key=item.counter_key)
reseeded: Final = await reseed_spend_counter_from_db(counter_key=counter_key)
if reseeded and actual_cost > 0:
await increment_spend_counter_cache(counter_key=item.counter_key, increment=actual_cost)
async def _counter_can_apply_adjustment(
counter_key: str,
adjustment: float,
) -> bool:
from litellm.proxy.proxy_server import read_spend_counter_cache_value
try:
current_value, _ = await read_spend_counter_cache_value(counter_key=counter_key)
except (TypeError, ValueError):
return False
if current_value is None:
return False
return not (adjustment < 0 and current_value + adjustment < -1e-12)
async def _release_applied_entries_best_effort(
entries: list[dict],
default_reserved_cost: float,
) -> None:
for entry in entries:
try:
await _set_reserved_entries_actual_cost(
entries=[entry],
actual_cost=0.0,
default_reserved_cost=default_reserved_cost,
)
except Exception:
counter_key = entry.get("counter_key")
verbose_proxy_logger.exception("Failed to release partial budget reservation during exception cleanup")
if counter_key is None:
continue
try:
from litellm.proxy.proxy_server import invalidate_spend_counter
await invalidate_spend_counter(counter_key=counter_key)
except Exception:
verbose_proxy_logger.exception(
"Failed to invalidate partial budget reservation counter during exception cleanup"
)
async def _resize_applied_reservation(
entries: list[dict],
current_reserved_cost: float,
new_reserved_cost: float,
) -> None:
await _set_reserved_entries_actual_cost(
entries=entries,
actual_cost=new_reserved_cost,
default_reserved_cost=current_reserved_cost,
reseed_on_inconsistent=False,
)
for entry in entries:
entry["reserved_cost"] = new_reserved_cost
entry["applied_adjustment"] = 0.0
def _counter_to_reservation_entry(
counter: _BudgetCounter,
reserved_cost: float,
) -> dict[str, float | str]:
return {
"counter_key": counter.counter_key,
"entity_type": counter.entity_type,
"entity_id": counter.entity_id,
"reserved_cost": reserved_cost,
"applied_adjustment": 0.0,
**({"window_reset_at": counter.reset_at.isoformat()} if counter.reset_at is not None else {}),
}
await increment_spend_counter_cache(counter_key=counter_key, increment=actual_cost)
def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float:
@ -1416,10 +1067,10 @@ def get_budget_window_start(window: object, reset_settings: BudgetResetSettings
reset_at = _coerce_datetime(window_dict.get("reset_at"))
if reset_at is None:
return datetime.now(timezone.utc) - timedelta(seconds=duration_seconds)
return normalize_budget_window_start(datetime.now(timezone.utc) - timedelta(seconds=duration_seconds))
if reset_at.tzinfo is None:
reset_at = reset_at.replace(tzinfo=timezone.utc)
return reset_at - timedelta(seconds=duration_seconds)
return normalize_budget_window_start(reset_at - timedelta(seconds=duration_seconds))
def _coerce_datetime(value: object) -> datetime | None:

View file

@ -0,0 +1,320 @@
from __future__ import annotations
import asyncio
import uuid
from collections.abc import Iterable, Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import TYPE_CHECKING, Annotated, Final, Literal
from fastapi import HTTPException, status
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
from pydantic import Field as PydanticField
import litellm
from litellm.integrations.custom_logger_v2.derivations import (
budget_reservation,
mapping,
write_budget_reservation,
)
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
from litellm.redis.stores.spend_counters import SpendCounterStore
from litellm.types.integrations.custom_logger_v2 import BudgetPlan, BudgetReservation, coerce_record
from litellm.types.proxy.native_budgets import BudgetCheck, NativeSettlement
if TYPE_CHECKING:
from litellm.rust_bridge._native import NativeBudgets
def new_reservation_id() -> str:
return uuid.uuid4().hex
class _BudgetExceededFields(BaseModel):
model_config = ConfigDict(frozen=True)
current_cost: float
max_budget: float
message: str
entity_type: str | None
entity_id: str | None
class _DetailFields(BaseModel):
model_config = ConfigDict(frozen=True)
detail: str
class _MessageFields(BaseModel):
model_config = ConfigDict(frozen=True)
message: str
class _Held(BaseModel):
model_config = ConfigDict(frozen=True)
admission: Literal["held"]
reservation: Mapping[str, object]
class _Skipped(BaseModel):
model_config = ConfigDict(frozen=True)
admission: Literal["skipped"]
class _Refused(BaseModel):
model_config = ConfigDict(frozen=True)
admission: Literal["refused"]
kind: Literal["budget_exceeded", "budget_unavailable", "budget_reservation_failed"]
fields: Mapping[str, object]
_ADMISSION: Final[TypeAdapter[_Held | _Skipped | _Refused]] = TypeAdapter(
Annotated[_Held | _Skipped | _Refused, PydanticField(discriminator="admission")]
)
_SETTLEMENT: Final = TypeAdapter(NativeSettlement)
_CHECK: Final[TypeAdapter[BudgetCheck]] = TypeAdapter(BudgetCheck)
_HELD_ENTRIES: Final = TypeAdapter(Iterable[object])
def _refusal(refused: _Refused) -> Exception:
match refused.kind:
case "budget_exceeded":
exceeded: Final = _BudgetExceededFields.model_validate(refused.fields)
return litellm.BudgetExceededError(
current_cost=exceeded.current_cost,
max_budget=exceeded.max_budget,
message=exceeded.message,
entity_type=exceeded.entity_type,
entity_id=exceeded.entity_id,
)
case "budget_unavailable":
return HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail=_DetailFields.model_validate(refused.fields).detail,
)
case "budget_reservation_failed":
return ProxyException(
message="Authentication Error, " + _MessageFields.model_validate(refused.fields).message,
type=ProxyErrorTypes.auth_error,
param=None,
code=status.HTTP_401_UNAUTHORIZED,
)
def _with_window_reset(entry: object, window_resets: Mapping[str, str]) -> object:
held: Final = mapping(entry)
if held is None:
return entry
reset: Final = window_resets.get(str(held.get("counter_key")))
return entry if reset is None else {**held, "window_reset_at": reset}
def _with_window_resets(value: Mapping[str, object], window_resets: Mapping[str, str]) -> Mapping[str, object]:
entries: Final = value.get("entries")
if not window_resets or not isinstance(entries, list | tuple):
return value
return {
**value,
"entries": [_with_window_reset(entry, window_resets) for entry in _HELD_ENTRIES.validate_python(entries)],
}
def _planned_counters(plan: BudgetPlan) -> tuple[tuple[str, float], ...]:
return tuple(
(counter.counter_key, float(counter.fallback_spend or 0.0))
for counter in plan.counters or ()
if counter.counter_key is not None
)
async def observed_counter_spend(plan: BudgetPlan) -> tuple[Mapping[str, object], ...]:
from litellm.proxy.proxy_server import get_current_spend, spend_counter_store
counters: Final = _planned_counters(plan)
with spend_counter_batch_scope(spend_counter_store, counter_keys=frozenset(key for key, _ in counters)):
return tuple(
[
{
"counter_key": counter_key,
"spend": await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend),
}
for counter_key, fallback_spend in counters
]
)
_engine: Final[list[tuple[SpendCounterStore, NativeBudgets]]] = [] # mutable-ok: the engine for the proxy's store
def native_budgets() -> NativeBudgets | None:
from litellm.proxy.proxy_server import spend_counter_store
if _engine and _engine[0][0] is spend_counter_store:
return _engine[0][1]
from litellm.rust_bridge._native import NativeBudgets, NativeSpendCounters
handle: Final = spend_counter_store.handle
if not isinstance(handle, NativeSpendCounters):
return None
budgets: Final = NativeBudgets(handle)
_engine[:] = [(spend_counter_store, budgets)]
return budgets
async def reserve_native(
plan: Mapping[str, object],
identity: UserAPIKeyAuth,
window_resets: Mapping[str, str],
) -> dict[str, object] | None: # mutable-ok: settlement stamps the reservation it returns
budgets: Final = native_budgets()
if budgets is None:
return None
record: Final = coerce_record(BudgetPlan, plan)
admission: Final = _ADMISSION.validate_python(await budgets.reserve(record, await observed_counter_spend(record)))
if isinstance(admission, _Refused):
raise _refusal(admission)
if isinstance(admission, _Skipped):
return None
held: Final = _with_window_resets(admission.reservation, window_resets)
reservation: Final[dict[str, object]] = { # mutable-ok: settlement stamps the reservation it returns
"reserved_cost": 0.0,
"entries": [],
"finalized": False,
"callback_bound": False,
"input_cost": 0.0,
"input_tokens": None,
}
identity.budget_reservation = reservation
write_budget_reservation(HookArgs(auth=identity), held)
from litellm.proxy.spend_tracking.budget_reservation import start_reservation_lease_renewal
start_reservation_lease_renewal(
budget_reservation=reservation,
counter_keys=frozenset(
key for key in map(_held_counter_key, _held_entries(reservation.get("entries") or ())) if key is not None
),
request_task=asyncio.current_task(),
)
return reservation
@dataclass(frozen=True, slots=True)
class _Reserved:
budget_reservation: object
def _reservation_record(reservation: object) -> BudgetReservation | None:
return budget_reservation(HookArgs(auth=_Reserved(reservation)))
def _window_reset(entry: object) -> str | None:
held: Final = mapping(entry)
reset: Final = None if held is None else held.get("window_reset_at")
if isinstance(reset, datetime):
return reset.isoformat()
return reset if isinstance(reset, str) else None
def _held_entries(raw: object) -> tuple[object, ...]:
try:
return tuple(_HELD_ENTRIES.validate_python(raw))
except ValidationError:
raise TypeError(f"'{type(raw).__name__}' object is not iterable") from None
def _held_counter_key(entry: object) -> str | None:
held: Final = mapping(entry)
counter_key: Final = None if held is None else held.get("counter_key")
return counter_key if isinstance(counter_key, str) else None
async def _observed_spend(counter_key: str | None) -> float | None:
if counter_key is None:
return None
from litellm.proxy.proxy_server import read_spend_counter_cache_value
try:
value, _ = await read_spend_counter_cache_value(counter_key=counter_key)
except (TypeError, ValueError):
return None
return value
async def settle_native_reservation(
reservation: Mapping[str, object],
actual_cost: float,
apply_consistent: bool = True,
) -> tuple[PendingSpendIncrement, ...]:
budgets: Final = native_budgets()
if budgets is None:
return ()
entries: Final = _held_entries(reservation.get("entries") or ())
record: Final = _reservation_record(reservation)
if record is None:
return ()
from litellm.proxy.proxy_server import spend_counter_store
from litellm.proxy.spend_tracking.budget_reservation import reseed_reserved_counter
counter_keys: Final = tuple(map(_held_counter_key, entries))
with spend_counter_batch_scope(
spend_counter_store, counter_keys=frozenset(key for key in counter_keys if key is not None)
):
observed: Final = tuple([await _observed_spend(counter_key) for counter_key in counter_keys])
settled: Final = _SETTLEMENT.validate_python(
await budgets.settle(record, tuple(map(_window_reset, entries)), observed, actual_cost, apply_consistent)
)
for reseed in settled.reseed:
await reseed_reserved_counter(reseed.counter_key, reseed.actual_cost)
for entry, target in zip(entries, settled.adjustments):
if target is not None and isinstance(entry, dict):
entry["applied_adjustment"] = target
return tuple(
PendingSpendIncrement(counter_key=pending.counter_key, increment=pending.increment)
for pending in settled.pending
)
class BudgetDecision(BaseModel):
model_config = ConfigDict(frozen=True)
decision: Literal["skip", "pass", "alert", "throttle", "refuse"]
pct: int | float | None = None
current_cost: int | float | None = None
max_budget: int | float | None = None
message: str | None = None
entity_type: str | None = None
entity_id: str | None = None
@property
def skipped(self) -> bool:
return self.decision == "skip"
@property
def alerts(self) -> bool:
return self.decision == "alert"
@property
def refused(self) -> bool:
return self.decision in ("refuse", "throttle")
def error(self) -> litellm.BudgetExceededError:
return litellm.BudgetExceededError(
current_cost=self.current_cost if self.current_cost is not None else 0.0,
max_budget=self.max_budget if self.max_budget is not None else 0.0,
message=self.message,
entity_type=self.entity_type,
entity_id=self.entity_id,
)
def native_budget_decision(check: Mapping[str, object]) -> BudgetDecision:
from litellm.rust_bridge._native import budget_check_decision
return BudgetDecision.model_validate(budget_check_decision(_CHECK.validate_python(check)))

View file

@ -78,6 +78,10 @@ class SpendCounterStore:
if redis.client is not None:
self._handle.attach(redis.client)
@property
def handle(self) -> SpendCountersHandle:
return self._handle
@property
def redis_ttl(self) -> float | None:
"""The TTL every Redis write gives a counter, None when there is no Redis."""

View file

@ -1,34 +1,110 @@
import asyncio
from collections.abc import AsyncGenerator, Coroutine
import urllib.parse
from collections.abc import AsyncGenerator, Callable, Coroutine
from contextlib import asynccontextmanager
from dataclasses import dataclass
from typing import TYPE_CHECKING, Final
from litellm.rust_bridge._native import NativeDatabase
from litellm._logging import verbose_proxy_logger
from litellm.proxy.db.db_span import proxy_service_logging
from litellm.proxy.db.native_database import (
NativeEndpointSettings,
ServiceLoggingEvents,
TokenRotation,
keep_token_fresh,
reader_token_rotation,
resolve_native_endpoint,
writer_token_rotation,
)
from litellm.rust_bridge._native import NativeDatabase, NativeDatabaseFailure
if TYPE_CHECKING:
from litellm.proxy.utils import PrismaClient
def _token_refresh(
install: Callable[[str], None], rotation: TokenRotation | None, url: str
) -> "tuple[asyncio.Task[None], ...]":
if rotation is None:
return ()
current: Final = urllib.parse.unquote(urllib.parse.urlsplit(url).password or "")
return (asyncio.create_task(keep_token_fresh(install, rotation, current)),)
def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None:
if settings.ignored_options:
verbose_proxy_logger.error(
"Native database pool cannot send the %s options %s; only -c name=value settings carry over",
env_var,
settings.ignored_options,
)
async def _warn_if_unreachable(native: NativeDatabase) -> None:
try:
await native.health_check()
except NativeDatabaseFailure as error:
verbose_proxy_logger.warning(
"Native database pool cannot reach the database, so operations ported to it will fail until it"
" connects. Prisma is checked separately. Cause: %s",
error,
)
@dataclass(frozen=True, slots=True)
class Database:
prisma: "PrismaClient"
native: NativeDatabase
native: NativeDatabase | None
tasks: "tuple[asyncio.Task[None], ...]" = ()
boot_probe: "asyncio.Task[None] | None" = None
@classmethod
async def open(cls, prisma: "PrismaClient") -> "Database":
async def open(
cls, prisma: "PrismaClient", writer_url: str, reader_url: str | None, native_connections: int
) -> "Database":
if native_connections == 0:
return cls(prisma=prisma, native=None)
try:
native: Final = await NativeDatabase.connect()
writer: Final = await resolve_native_endpoint(writer_url, native_connections)
reader: Final = (
None if reader_url is None else await resolve_native_endpoint(reader_url, native_connections)
)
native: Final = await NativeDatabase.connect(
ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=proxy_service_logging),
writer.endpoint,
None if reader is None else reader.endpoint,
)
except BaseException:
await prisma.aclose()
raise
return cls(prisma=prisma, native=native)
_log_ignored_options("DATABASE_URL", writer)
if reader is not None:
_log_ignored_options("DATABASE_URL_READ_REPLICA", reader)
writer_refresh: Final = _token_refresh(
native.set_writer_password, writer_token_rotation(writer_url), writer_url
)
reader_refresh: Final = (
()
if reader_url is None
else _token_refresh(native.set_reader_password, reader_token_rotation(reader_url), reader_url)
)
return cls(
prisma=prisma,
native=native,
tasks=(*writer_refresh, *reader_refresh),
boot_probe=asyncio.create_task(_warn_if_unreachable(native)),
)
async def aclose(self) -> None:
background: Final = (*self.tasks, *(() if self.boot_probe is None else (self.boot_probe,)))
for task in background:
task.cancel()
await asyncio.gather(*background, return_exceptions=True)
try:
await self.prisma.aclose()
finally:
await self.native.close()
if self.native is not None:
await self.native.close()
@asynccontextmanager

View file

@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N
verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name)
def _params_base_model(core: RustCore, deployment_id: str) -> str | None:
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
return base_model if isinstance(base_model, str) else None
class _Driver:
"""Answers the attempt and fallback steps of one call, and shapes its result.
@ -4496,7 +4503,7 @@ class _Driver:
self._route_phase: contextlib.ExitStack | None = None
self._route_model: str = ""
self._walk: Final = WalkMetadata(
request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt
request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt
)
self.outcomes: Final = FallbackOutcomes(request)
# The fallback level the walk is at, from its fallback events.
@ -4548,12 +4555,6 @@ class _Driver:
endpoints.counters.retract_success(self._plan, call)
endpoints.attempt_failed(self._plan, error, deployment, call, call.group)
def _params_base_model(self, deployment_id: str) -> str | None:
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
return base_model if isinstance(base_model, str) else None
def _logged_error(self, event: Mapping[str, object]) -> BaseException | None:
"""The error a ``retry_logged`` event records."""
return self._cause_error(event)
@ -5544,6 +5545,7 @@ class _Driver:
def finish(self, step: Mapping[str, object]) -> object:
"""The response with the router's headers, in the order ``make_call`` and the wrappers above it
stamp them: a usage read that raises leaves the counts off."""
self._clear_errors()
if not self._plan.sets_headers:
self.log_outcomes_sync()
return self._last_response
@ -5564,6 +5566,7 @@ class _Driver:
async def finish_async(self, step: Mapping[str, object]) -> object:
"""``finish``, with the usage read on the call's own loop."""
self._clear_errors()
if not self._plan.sets_headers:
await await_all(self.outcomes.take_pending())
return self._last_response
@ -5585,6 +5588,18 @@ class _Driver:
return self._request.get("messages") is not None or self._request.get("input") is not None
def failed(self, step: Mapping[str, object]) -> BaseException:
try:
return self._failure(step)
finally:
self._clear_errors()
def _clear_errors(self) -> None:
self._last_error = None
self._level_errors.clear()
self._hop_first.clear()
self._level_picks.clear()
def _failure(self, step: Mapping[str, object]) -> BaseException:
model: Final = str(self._request.get("model"))
primary: Final = self._level_errors.get(self._base_level)
if primary is None:

View file

@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own
self._async_generator: Final = async_generator
self.attempt: Final = attempt
self._billing_source: object = source_iterator
self._last_sequence_number = -1
self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source
@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
if response_id_exposed:
_raise_released_stream_error(attempt, e)
async with contextlib.aclosing(
_responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held)
) as fallback_stream:
async for fallback_item in fallback_stream:
yield fallback_item
_responses_recover_stream_error(
attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held
)
) as recovered_stream:
async for recovered_item in recovered_stream:
yield recovered_item
except Exception:
for held_event in held:
yield held_event
@ -1791,6 +1794,99 @@ def _responses_continuation_request(
}
def _responses_stream_retries(
attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator"
) -> bool:
"""Whether a broken Responses stream retries in its own group first: it broke before any output and
before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a
``/v1/messages`` stream does. After output the continuation fallback takes over instead."""
if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call:
return False
status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e))
return status_code is None or attempt.should_retry_status(status_code)
async def _responses_recover_stream_error(
attempt: StreamAttempt,
e: MidStreamFallbackError,
source_iterator: "BaseResponsesAPIStreamingIterator",
wrapper: object,
adopt_headers: Callable[[object], _Prepared],
held: tuple[object, ...],
) -> AsyncGenerator[object, None]:
"""A Responses stream that broke before any output is retried in its own group, then falls back
with the newest failure once the retries are spent. The primary's held lifecycle events are dropped
when a retry or a fallback answers."""
last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed
if _responses_stream_retries(attempt, e, source_iterator):
if hasattr(source_iterator, "aclose"):
with anyio.CancelScope(shield=True), contextlib.suppress(Exception):
await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
await attempt.failure_recorded()
trigger: Final = _mid_stream_retry_trigger(e)
retried: Final = await attempt.retry(trigger)
if not isinstance(retried, BaseException):
async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers):
yield retried_item
return
if retried is not trigger:
last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest
retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider
)
async with contextlib.aclosing(
_responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held)
) as fallback_stream:
async for fallback_item in fallback_stream:
yield fallback_item
async def _responses_yield_retried(
retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared]
) -> AsyncGenerator[object, None]:
"""Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and
closes it after. A retried stream that breaks again counts on from the retries already made, and when
it recovers onto another source (its own fallback) the wrapper and the events follow that source."""
response, attempted, budget = retried
inner_attempt: Final = getattr(response, "attempt", None)
if isinstance(inner_attempt, StreamAttempt):
inner_attempt.attempted_retries = attempted
inner_attempt.committed_budget = budget
served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group")
def follow_source() -> _Prepared:
prepared: Final = adopt_headers(response)
if served_as is not None:
_ = prepared[1].setdefault("x-litellm-model-group", served_as)
newest: Final = prepared[1].get("x-litellm-attempted-retries")
_ = add_retry_headers_to_response(
response=wrapper,
attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted,
max_retries=budget,
)
return prepared
followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source
source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source
try:
if hasattr(response, "__aiter__"):
async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above
if (current_source := getattr(response, "_billing_source", None)) is not source:
source = current_source
followed = follow_source()
_apply_fallback_hidden_params_to_item(retried_item, followed)
yield retried_item
else:
yield response
finally:
if hasattr(response, "aclose"):
with anyio.CancelScope(shield=True):
try:
await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
except Exception as exc: # noqa: BLE001 # closing is best-effort
verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc)
async def _responses_fallback_attempt(
attempt: StreamAttempt,
e: MidStreamFallbackError,

View file

@ -133,6 +133,7 @@ from litellm.types.utils import (
)
from litellm.utils import (
ModelInfoLookupMiss,
base_fields_replaced_by,
catalog_with_runtime_registrations,
get_potential_model_names,
get_secret,
@ -340,10 +341,11 @@ def _inherited_off_peak_base_rates(
backend_entry: Final = _backend_cost_entry(host, backend_model, provider)
if backend_entry is None:
return {}
superseded: Final = base_fields_replaced_by(model_info)
return {
field: copy.deepcopy(value)
for field, value in backend_entry.items()
if model_info.get(field) is None and value is not None
if model_info.get(field) is None and value is not None and field not in superseded
}

View file

@ -258,6 +258,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
dependencies = [
"aws-credential-types",
"aws-runtime",
"aws-sdk-sso",
"aws-sdk-ssooidc",
"aws-sdk-sts",
"aws-smithy-async",
"aws-smithy-http",
@ -269,11 +271,14 @@ dependencies = [
"aws-types",
"bytes",
"fastrand",
"hex",
"http 1.4.2",
"sha1 0.10.7",
"time",
"tokio",
"tracing",
"url",
"zeroize",
]
[[package]]
@ -428,6 +433,58 @@ dependencies = [
"tracing",
]
[[package]]
name = "aws-sdk-sso"
version = "1.111.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90"
dependencies = [
"arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand",
"http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-ssooidc"
version = "1.113.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a"
dependencies = [
"arc-swap",
"aws-credential-types",
"aws-runtime",
"aws-smithy-async",
"aws-smithy-http",
"aws-smithy-json",
"aws-smithy-observability",
"aws-smithy-runtime",
"aws-smithy-runtime-api",
"aws-smithy-schema",
"aws-smithy-types",
"aws-types",
"bytes",
"fastrand",
"http 0.2.12",
"http 1.4.2",
"regex-lite",
"tracing",
]
[[package]]
name = "aws-sdk-sts"
version = "1.108.0"
@ -3455,6 +3512,7 @@ dependencies = [
"aws-smithy-types",
"aws-types",
"bytes",
"chrono",
"futures-util",
"litellm-auth-types",
"litellm-error",
@ -3467,6 +3525,7 @@ dependencies = [
"serde",
"serde_json",
"sha2 0.10.9",
"tempfile",
"thiserror 2.0.19",
"tokio",
"url",
@ -3705,6 +3764,22 @@ dependencies = [
"thiserror 2.0.19",
]
[[package]]
name = "litellm-callbacks-budgets"
version = "0.1.0"
dependencies = [
"litellm-callbacks",
"litellm-coroutine",
"litellm-python-compat",
"litellm-redis-store",
"litellm-tracing",
"rstest",
"serde",
"serde_json",
"tokio",
"tracing",
]
[[package]]
name = "litellm-callbacks-legacy-python"
version = "0.1.0"
@ -3836,6 +3911,10 @@ name = "litellm-db"
version = "0.1.0"
dependencies = [
"chrono",
"futures-util",
"litellm-circuit-breaker",
"litellm-error",
"litellm-secret-value",
"macro_rules_attribute",
"rstest",
"serde",
@ -3844,6 +3923,7 @@ dependencies = [
"strum",
"thiserror 2.0.19",
"tokio",
"url",
]
[[package]]
@ -3983,6 +4063,7 @@ dependencies = [
"litellm-auth",
"litellm-cache-memory",
"litellm-cache-response",
"litellm-core-utils",
"litellm-error",
"litellm-gateway-auth",
"litellm-host",
@ -4248,6 +4329,7 @@ dependencies = [
"litellm-inference-testing",
"litellm-llms",
"litellm-llms-types",
"litellm-python-compat",
"litellm-router-types",
"litellm-secret-value",
"litellm-secrets",
@ -4370,6 +4452,7 @@ dependencies = [
"aws-smithy-types",
"base64 0.22.1",
"bytes",
"crc32fast",
"data-url",
"fancy-regex 0.19.2",
"futures-util",
@ -4399,6 +4482,7 @@ dependencies = [
"thiserror 2.0.19",
"time",
"tokio",
"tokio-util",
"url",
]
@ -4467,6 +4551,7 @@ name = "litellm-python-bridge"
version = "0.1.0"
dependencies = [
"async-trait",
"aws-credential-types",
"aws-sdk-secretsmanager",
"azure_core",
"bytes",
@ -4480,11 +4565,13 @@ dependencies = [
"litellm-cache",
"litellm-cache-response",
"litellm-callbacks",
"litellm-callbacks-budgets",
"litellm-callbacks-legacy-python",
"litellm-callbacks-python",
"litellm-callbacks-s3-log",
"litellm-circuit-breaker",
"litellm-core-utils",
"litellm-db",
"litellm-error",
"litellm-host",
"litellm-host-python",

View file

@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" }
litellm-callbacks = { path = "crates/callbacks" }
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
litellm-callbacks-python = { path = "crates/callbacks-python" }
litellm-callbacks-budgets = { path = "crates/callbacks-budgets" }
litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
litellm-framer = { path = "crates/framer" }
litellm-auth = { path = "crates/auth" }

View file

@ -16,12 +16,13 @@ litellm-secret-value.workspace = true
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
sha2.workspace = true
chrono.workspace = true
thiserror.workspace = true
tokio.workspace = true
url.workspace = true
veil.workspace = true
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process", "sso"] }
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] }
aws-sigv4 = "1.5.1"
@ -33,6 +34,7 @@ litellm-secrets.workspace = true
aws-smithy-types = "1.6.4"
aws-runtime = { version = "1.9.4", default-features = false }
rstest.workspace = true
tempfile = "3"
litellm-http = { workspace = true, features = ["test-support"] }
bytes.workspace = true
futures-util.workspace = true

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1 @@
{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:CountTokens","bedrock:Rerank","bedrock:Retrieve","bedrock:ListKnowledgeBases","bedrock:InvokeAgent","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"BedrockAgentCoreLiteLLM","Effect":"Allow","Action":["bedrock-agentcore:InvokeAgentRuntime","bedrock-agentcore:InvokeAgentRuntimeForUser","bedrock-agentcore:InvokeGateway"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformLiteLLM","Effect":"Allow","Action":["aws-external-anthropic:CreateInference","aws-external-anthropic:CreateBatchInference","aws-external-anthropic:CancelBatchInference","aws-external-anthropic:DeleteBatchInference","aws-external-anthropic:CountTokens","aws-external-anthropic:Get*","aws-external-anthropic:List*"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformOutboundIdentityLiteLLM","Effect":"Allow","Action":["sts:GetWebIdentityToken","sts:TagGetWebIdentityToken"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringEquals":{"aws:CalledViaLast":"aws-external-anthropic.amazonaws.com"}}},{"Sid":"BedrockMantleLiteLLM","Effect":"Allow","Action":["bedrock-mantle:CreateInference","bedrock-mantle:CountTokens"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}

View file

@ -42,5 +42,6 @@ pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[
pub const BEDROCK_SERVICE: &str = "bedrock";
pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session";
pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2";
pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str =
"https://bedrock-runtime.{region}.amazonaws.com";
pub const BEDROCK_SESSION_POLICY: &str = include_str!("bedrock_session_policy.json");
pub const WEB_IDENTITY_SESSION_SECONDS: i32 = 3600;
pub const AWS4_AUTHORIZATION_PREFIX: &str = "AWS4-HMAC-SHA256";

View file

@ -1,3 +1,4 @@
use aws_credential_types::provider::error::CredentialsError;
use litellm_error::Source;
#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
@ -10,6 +11,14 @@ pub enum Error {
#[source]
source: Source,
},
#[error(
"the AWS web identity token must name a secret or an oidc/ provider, not an environment variable"
)]
WebIdentityTokenReference,
#[error("the AWS web identity token reference resolved to no token")]
MissingWebIdentityToken,
#[error("could not read the AWS web identity token")]
WebIdentityToken(#[source] Source),
#[error("AWS web identity response had no credentials")]
MissingWebIdentityCredentials,
#[error("AWS web identity response had an invalid expiration")]
@ -49,6 +58,20 @@ pub enum SigningStage {
}
impl Error {
pub fn is_empty_default_chain(&self) -> bool {
let Self::Credentials {
flow: CredentialFlow::DefaultChain,
source,
} = self
else {
return false;
};
matches!(
source.downcast_ref::<CredentialsError>(),
Some(CredentialsError::CredentialsNotLoaded(_))
)
}
pub(crate) fn credentials<E: std::error::Error + Send + Sync + 'static>(
flow: CredentialFlow,
) -> impl FnOnce(E) -> Self {
@ -73,3 +96,40 @@ impl From<Error> for litellm_auth_types::Error {
Self::provider(error)
}
}
#[cfg(test)]
mod tests {
use super::*;
#[rstest::rstest]
#[case::empty_default_chain(
CredentialFlow::DefaultChain,
CredentialsError::not_loaded("no providers in chain provided credentials"),
true
)]
#[case::failed_default_chain(
CredentialFlow::DefaultChain,
CredentialsError::provider_error("IMDS answered 500"),
false
)]
#[case::empty_profile(
CredentialFlow::Profile,
CredentialsError::not_loaded("the profile has no credentials"),
false
)]
fn only_a_default_chain_that_loaded_nothing_is_empty(
#[case] flow: CredentialFlow,
#[case] failure: CredentialsError,
#[case] empty: bool,
) {
assert_eq!(
Error::credentials(flow)(failure).is_empty_default_chain(),
empty
);
}
#[test]
fn a_web_identity_failure_is_not_an_empty_default_chain() {
assert!(!Error::MissingWebIdentityToken.is_empty_default_chain());
}
}

View file

@ -2,10 +2,12 @@ mod aws;
pub mod constants;
mod error;
mod provider;
mod region;
mod signer;
pub use aws::*;
pub use aws_credential_types::Credentials;
pub use error::{CredentialFlow, Error, SigningStage};
pub use provider::AwsCredentialsProvider;
pub use signer::SigV4Signer;
pub use region::{AwsPartition, aws_partition, bedrock_runtime_endpoint};
pub use signer::{CallerSigningHeaders, SigV4Signer};

View file

@ -0,0 +1,239 @@
use std::path::PathBuf;
use crate::constants::AWS_DEFAULT_REGION;
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct AwsPartition {
pub partition: &'static str,
pub dns_suffix: &'static str,
}
const COMMERCIAL_PARTITION: AwsPartition = AwsPartition {
partition: "aws",
dns_suffix: "amazonaws.com",
};
const PARTITIONS_BY_REGION_PREFIX: [(&str, AwsPartition); 6] = [
(
"cn-",
AwsPartition {
partition: "aws-cn",
dns_suffix: "amazonaws.com.cn",
},
),
(
"us-gov-",
AwsPartition {
partition: "aws-us-gov",
dns_suffix: "amazonaws.com",
},
),
(
"us-isob-",
AwsPartition {
partition: "aws-iso-b",
dns_suffix: "sc2s.sgov.gov",
},
),
(
"us-isof-",
AwsPartition {
partition: "aws-iso-f",
dns_suffix: "csp.hci.ic.gov",
},
),
(
"us-iso-",
AwsPartition {
partition: "aws-iso",
dns_suffix: "c2s.ic.gov",
},
),
(
"eu-isoe-",
AwsPartition {
partition: "aws-iso-e",
dns_suffix: "cloud.adc-e.uk",
},
),
];
pub fn aws_partition(region: &str) -> AwsPartition {
PARTITIONS_BY_REGION_PREFIX
.iter()
.find(|(prefix, _)| region.starts_with(prefix))
.map_or(COMMERCIAL_PARTITION, |(_, partition)| *partition)
}
pub fn bedrock_runtime_endpoint(region: &str) -> String {
format!(
"https://bedrock-runtime.{region}.{}",
aws_partition(region).dns_suffix
)
}
pub(crate) fn session_region(
env_lookup: &dyn Fn(&str) -> Option<String>,
process_env: &dyn Fn(&str) -> Option<String>,
) -> Option<String> {
env_lookup(AWS_DEFAULT_REGION)
.filter(|region| !region.is_empty())
.or_else(|| profile_region(process_env))
}
fn profile_region(process_env: &dyn Fn(&str) -> Option<String>) -> Option<String> {
let profile = process_env("AWS_PROFILE")
.or_else(|| process_env("AWS_DEFAULT_PROFILE"))
.filter(|profile| !profile.is_empty())
.unwrap_or_else(|| "default".to_string());
let home = process_env("HOME");
let path = match process_env("AWS_CONFIG_FILE").filter(|path| !path.is_empty()) {
Some(path) => match (path.strip_prefix("~/"), &home) {
(Some(rest), Some(home)) => PathBuf::from(home).join(rest),
_ => PathBuf::from(path),
},
None => PathBuf::from(home?).join(".aws").join("config"),
};
config_file_region(&std::fs::read_to_string(path).ok()?, &profile)
}
fn config_file_region(contents: &str, profile: &str) -> Option<String> {
let named = format!("profile {profile}");
let selects = |section: &str| section == named || (profile == "default" && section == profile);
let mut selected = false;
for line in contents.lines().map(str::trim) {
if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
continue;
}
if let Some(section) = line
.strip_prefix('[')
.and_then(|line| line.strip_suffix(']'))
{
selected = selects(&section.split_whitespace().collect::<Vec<_>>().join(" "));
continue;
}
if selected
&& let Some((key, value)) = line.split_once('=')
&& key.trim() == "region"
{
return Some(value.trim().to_string()).filter(|region| !region.is_empty());
}
}
None
}
pub(crate) fn sts_endpoint_region(endpoint: &str) -> Option<String> {
let url = url::Url::parse(endpoint).ok()?;
let labels: Vec<&str> = url.host_str()?.split('.').collect();
(0..labels.len()).find_map(|index| {
let ["sts" | "sts-fips", region, rest @ ..] = &labels[index..] else {
return None;
};
let rest = rest.join(".");
let valid_region = !region.is_empty()
&& region
.chars()
.all(|char| char.is_ascii_lowercase() || char.is_ascii_digit() || char == '-');
(valid_region
&& (rest.starts_with("amazonaws.com") || rest.starts_with("vpce.amazonaws.com")))
.then(|| region.to_string())
})
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case::commercial("us-east-1", "https://bedrock-runtime.us-east-1.amazonaws.com")]
#[case::china("cn-north-1", "https://bedrock-runtime.cn-north-1.amazonaws.com.cn")]
#[case::gov_cloud("us-gov-west-1", "https://bedrock-runtime.us-gov-west-1.amazonaws.com")]
#[case::iso("us-iso-east-1", "https://bedrock-runtime.us-iso-east-1.c2s.ic.gov")]
#[case::iso_b(
"us-isob-east-1",
"https://bedrock-runtime.us-isob-east-1.sc2s.sgov.gov"
)]
#[case::iso_f(
"us-isof-south-1",
"https://bedrock-runtime.us-isof-south-1.csp.hci.ic.gov"
)]
#[case::iso_e(
"eu-isoe-west-1",
"https://bedrock-runtime.eu-isoe-west-1.cloud.adc-e.uk"
)]
fn the_runtime_endpoint_uses_the_partition_dns_suffix(
#[case] region: &str,
#[case] expected: &str,
) {
assert_eq!(bedrock_runtime_endpoint(region), expected);
}
#[rstest]
#[case::china("cn-northwest-1", "aws-cn")]
#[case::gov_cloud("us-gov-east-1", "aws-us-gov")]
#[case::commercial("eu-central-1", "aws")]
fn a_region_names_its_partition(#[case] region: &str, #[case] expected: &str) {
assert_eq!(aws_partition(region).partition, expected);
}
#[rstest]
#[case::regional("https://sts.eu-west-1.amazonaws.com", Some("eu-west-1"))]
#[case::fips("https://sts-fips.us-east-2.amazonaws.com/", Some("us-east-2"))]
#[case::china("https://sts.cn-north-1.amazonaws.com.cn", Some("cn-north-1"))]
#[case::vpc_endpoint(
"https://vpce-0abc-1234.sts.ap-south-1.vpce.amazonaws.com",
Some("ap-south-1")
)]
#[case::global("https://sts.amazonaws.com", None)]
#[case::custom_host("https://sts.internal.example.com", None)]
#[case::no_scheme("sts.eu-west-1.amazonaws.com", None)]
fn the_sts_region_is_read_from_a_regional_endpoint(
#[case] endpoint: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(sts_endpoint_region(endpoint).as_deref(), expected);
}
const CONFIG: &str = "[default]\nregion = eu-west-3\n\n[profile work]\noutput = json\nregion=ap-northeast-1\n\n[profile empty]\nregion =\n";
#[rstest]
#[case::default_profile("default", Some("eu-west-3"))]
#[case::named_profile("work", Some("ap-northeast-1"))]
#[case::blank_region("empty", None)]
#[case::unknown_profile("missing", None)]
fn the_profile_region_comes_from_its_config_section(
#[case] profile: &str,
#[case] expected: Option<&str>,
) {
assert_eq!(config_file_region(CONFIG, profile).as_deref(), expected);
}
#[rstest]
#[case::default_region_wins(Some("sa-east-1"), None, Some("sa-east-1"))]
#[case::blank_default_region_reads_the_profile(Some(""), Some("work"), Some("ap-northeast-1"))]
#[case::profile_from_the_environment(None, Some("work"), Some("ap-northeast-1"))]
#[case::default_profile(None, None, Some("eu-west-3"))]
fn the_session_region_reads_the_default_region_then_the_profile(
#[case] default_region: Option<&str>,
#[case] profile: Option<&str>,
#[case] expected: Option<&str>,
) {
let file = tempfile::NamedTempFile::new().unwrap();
std::fs::write(file.path(), CONFIG).unwrap();
let config = file.path().to_string_lossy().into_owned();
let env = |name: &str| {
(name == AWS_DEFAULT_REGION)
.then(|| default_region.map(str::to_string))
.flatten()
};
let process_env = |name: &str| match name {
"AWS_CONFIG_FILE" => Some(config.clone()),
"AWS_PROFILE" => profile.map(str::to_string),
_ => None,
};
assert_eq!(session_region(&env, &process_env).as_deref(), expected);
}
}

View file

@ -1,11 +1,18 @@
use std::collections::BTreeMap;
use std::time::SystemTime;
use aws_credential_types::Credentials;
use reqwest::{Request, header::HeaderMap};
use aws_sigv4::sign::v4;
use chrono::{DateTime, Utc};
use reqwest::{
Request,
header::{HeaderMap, HeaderName, HeaderValue},
};
use sha2::{Digest, Sha256};
use crate::{
AwsAuthService, AwsCredentialSource, Error, SigningScope, aws_signature_headers,
is_sigv4_computed_header, sign_headers,
AwsAuthService, AwsCredentialSource, Error, SigningScope, SigningStage, aws_signature_headers,
constants::AWS4_AUTHORIZATION_PREFIX, is_sigv4_computed_header, sign_headers,
};
use litellm_http::outbound::RequestSigner;
@ -15,6 +22,14 @@ pub struct SigV4Signer {
service: &'static str,
credentials: Credentials,
clock: fn() -> SystemTime,
caller_headers: CallerSigningHeaders,
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
pub enum CallerSigningHeaders {
#[default]
Dropped,
Botocore,
}
impl SigV4Signer {
@ -24,6 +39,7 @@ impl SigV4Signer {
service,
credentials,
clock: SystemTime::now,
caller_headers: CallerSigningHeaders::Dropped,
}
}
@ -31,6 +47,18 @@ impl SigV4Signer {
Self { clock, ..self }
}
pub fn with_caller_headers(self, caller_headers: CallerSigningHeaders) -> Self {
Self {
caller_headers,
..self
}
}
fn keeps_caller_security_token(&self) -> bool {
self.caller_headers == CallerSigningHeaders::Botocore
&& self.credentials.session_token().is_none()
}
pub async fn resolve(
auth: &AwsAuthService,
region: String,
@ -38,48 +66,246 @@ impl SigV4Signer {
credentials: AwsCredentialSource,
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
) -> Result<Self, Error> {
Ok(Self::new(
region,
service,
credentials.resolve(auth, env_lookup).await?,
))
let credentials = credentials
.resolve(auth, &region, service, env_lookup)
.await?;
Ok(Self::new(region, service, credentials))
}
}
impl RequestSigner for SigV4Signer {
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
if let Some(name) = request
.headers()
.keys()
.find(|name| is_sigv4_computed_header(name.as_str()))
{
return Err(litellm_http::Error::computed_header(name.as_str()));
const CALLER_HEADERS_THE_SIGNER_REPLACES: &[&str] = &["x-amz-date", "x-amz-security-token", "date"];
const CALLER_DATE_HEADERS: &[&str] = &["x-amz-date", "date"];
const SECURITY_TOKEN: &str = "x-amz-security-token";
const BOTOCORE_UNSIGNED_HEADERS: &[&str] = &["expect", "user-agent", "x-amzn-trace-id"];
fn signable(headers: &HeaderMap, keep_security_token: bool) -> HeaderMap {
aws_signature_headers(headers)
.iter()
.filter(|(name, _)| {
!is_sigv4_computed_header(name.as_str())
|| (keep_security_token && name.as_str() == SECURITY_TOKEN)
})
.map(|(name, value)| (name.clone(), value.clone()))
.collect()
}
fn hex_sha256(bytes: &[u8]) -> String {
Sha256::digest(bytes)
.iter()
.map(|byte| format!("{byte:02x}"))
.collect()
}
fn quote_path(path: &str) -> String {
path.bytes()
.map(|byte| match byte {
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => {
char::from(byte).to_string()
}
other => format!("%{other:02X}"),
})
.collect()
}
fn canonical_uri(path: &str) -> String {
if path.is_empty() {
return "/".to_string();
}
let segments = path.split('/').fold(Vec::new(), |mut kept, segment| {
match segment {
"" | "." => {}
".." => {
kept.pop();
}
other => kept.push(other),
}
kept
});
let leading = if path.starts_with('/') { "/" } else { "" };
let trailing = if path.ends_with('/') && !segments.is_empty() {
"/"
} else {
""
};
quote_path(&format!("{leading}{}{trailing}", segments.join("/")))
}
fn canonical_query(query: Option<&str>) -> String {
let mut pairs: Vec<(&str, &str)> = query
.filter(|query| !query.is_empty())
.map(|query| {
query
.split('&')
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
.collect()
})
.unwrap_or_default();
pairs.sort_unstable();
pairs
.iter()
.map(|(key, value)| format!("{key}={value}"))
.collect::<Vec<_>>()
.join("&")
}
fn canonical_host(url: &reqwest::Url) -> String {
let host = url.host_str().unwrap_or_default();
match url.port() {
Some(port) => format!("{host}:{port}"),
None => host.to_string(),
}
}
fn header_pair(name: &str, value: &str) -> Result<(HeaderName, HeaderValue), Error> {
Ok((
HeaderName::from_bytes(name.as_bytes()).map_err(Error::signing(SigningStage::Output))?,
HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?,
))
}
fn sign_with_date_header(
request: &Request,
body: &[u8],
headers: &HeaderMap,
scope: SigningScope<'_>,
) -> Result<HeaderMap, Error> {
let time: DateTime<Utc> = scope.time.into();
let amz_date = time.format("%Y%m%dT%H%M%SZ").to_string();
let http_date = time.format("%a, %d %b %Y %H:%M:%S -0000").to_string();
let session_token = scope.credentials.session_token();
let mut canonical: BTreeMap<String, Vec<String>> = BTreeMap::new();
for (name, value) in headers {
if BOTOCORE_UNSIGNED_HEADERS.contains(&name.as_str()) {
continue;
}
let value = value.to_str().map_err(|_| Error::NonAsciiHeader {
name: name.to_string(),
})?;
canonical
.entry(name.as_str().to_string())
.or_default()
.push(value.split_whitespace().collect::<Vec<_>>().join(" "));
}
canonical.insert("date".into(), vec![http_date.clone()]);
if let Some(token) = session_token {
canonical.insert(SECURITY_TOKEN.into(), vec![token.to_string()]);
}
canonical
.entry("host".into())
.or_insert_with(|| vec![canonical_host(request.url())]);
let payload = headers
.get("x-amz-content-sha256")
.and_then(|value| value.to_str().ok())
.map_or_else(|| hex_sha256(body), str::to_string);
let canonical_headers: String = canonical
.iter()
.map(|(name, values)| format!("{name}:{}\n", values.join(",")))
.collect();
let signed_headers = canonical.keys().cloned().collect::<Vec<_>>().join(";");
let canonical_request = format!(
"{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload}",
request.method().as_str(),
canonical_uri(request.url().path()),
canonical_query(request.url().query()),
);
let credential_scope = format!(
"{}/{}/{}/aws4_request",
&amz_date[..8],
scope.region,
scope.service
);
let string_to_sign = format!(
"{AWS4_AUTHORIZATION_PREFIX}\n{amz_date}\n{credential_scope}\n{}",
hex_sha256(canonical_request.as_bytes())
);
let signing_key = v4::generate_signing_key(
scope.credentials.secret_access_key(),
scope.time,
scope.region,
scope.service,
);
let signature = v4::calculate_signature(signing_key, string_to_sign.as_bytes());
let authorization = format!(
"{AWS4_AUTHORIZATION_PREFIX} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}",
scope.credentials.access_key_id()
);
[
Some(("date", http_date.as_str())),
session_token.map(|token| (SECURITY_TOKEN, token)),
Some(("authorization", authorization.as_str())),
]
.into_iter()
.flatten()
.map(|(name, value)| header_pair(name, value))
.collect()
}
fn overrides_the_signature(headers: &HeaderMap) -> bool {
headers
.get(reqwest::header::AUTHORIZATION)
.is_some_and(|value| {
!value
.as_bytes()
.starts_with(AWS4_AUTHORIZATION_PREFIX.as_bytes())
})
}
impl RequestSigner for SigV4Signer {
fn owned_headers(&self) -> &'static [&'static str] {
if self.keeps_caller_security_token() {
CALLER_DATE_HEADERS
} else {
CALLER_HEADERS_THE_SIGNER_REPLACES
}
}
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
let body = match request.body() {
None => &[],
Some(body) => body
.as_bytes()
.ok_or_else(litellm_http::Error::unsignable_body)?,
};
sign_headers(
request.method().as_str(),
request.url().as_str(),
body,
&aws_signature_headers(request.headers()),
SigningScope {
region: &self.region,
service: self.service,
credentials: &self.credentials,
time: (self.clock)(),
},
)
let headers = signable(request.headers(), self.keeps_caller_security_token());
let scope = SigningScope {
region: &self.region,
service: self.service,
credentials: &self.credentials,
time: (self.clock)(),
};
let signed = match self.caller_headers == CallerSigningHeaders::Botocore
&& request.headers().contains_key(reqwest::header::DATE)
{
true => sign_with_date_header(request, body, &headers, scope),
false => sign_headers(
request.method().as_str(),
request.url().as_str(),
body,
&headers,
scope,
),
}
.map_err(|error| match error {
Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error),
Error::ExpiredCredentials
| Error::Credentials { .. }
| Error::WebIdentityTokenReference
| Error::MissingWebIdentityToken
| Error::WebIdentityToken(_)
| Error::MissingWebIdentityCredentials
| Error::WebIdentityExpiration(_)
| Error::Signing { .. } => litellm_http::Error::signature(error),
})?;
Ok(match overrides_the_signature(request.headers()) {
true => signed
.iter()
.filter(|(name, _)| *name != reqwest::header::AUTHORIZATION)
.map(|(name, value)| (name.clone(), value.clone()))
.collect(),
false => signed,
})
}
}
@ -176,22 +402,166 @@ mod tests {
assert!(original.contains("/us-east-1/textract/aws4_request"));
}
#[test]
fn a_forwarded_computed_header_is_refused_instead_of_sent_twice() {
let error = signed(
fn sent(headers: &[(&str, &str)], credentials: Credentials) -> Request {
signed(
json(
URL,
&[("authorization".into(), "Bearer caller".into())],
&headers
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect::<Vec<_>>(),
&json_value!({}),
None,
)
.unwrap(),
&signer("textract"),
&SigV4Signer::new("us-east-1".into(), "bedrock", credentials).with_clock(fixed_clock),
)
.unwrap_err();
.unwrap()
}
assert!(error.is_computed_header());
assert!(error.to_string().contains("authorization"), "{error}");
fn header<'a>(request: &'a Request, name: &str) -> Vec<&'a str> {
request
.headers()
.get_all(name)
.iter()
.map(|value| value.to_str().unwrap())
.collect()
}
#[rstest::rstest]
#[case::caller_date("date", None)]
#[case::caller_amz_date("x-amz-date", Some("20231114T221320Z"))]
#[case::caller_security_token_without_a_session("x-amz-security-token", None)]
fn a_header_the_signer_computes_is_dropped_from_the_callers_copy(
#[case] name: &str,
#[case] expected: Option<&str>,
) {
let request = sent(
&[(name, "caller-value")],
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
);
assert_eq!(
header(&request, name),
expected.into_iter().collect::<Vec<_>>()
);
assert!(header(&request, "authorization")[0].starts_with("AWS4-HMAC-SHA256 "));
}
#[test]
fn a_session_token_replaces_the_callers_security_token() {
let request = sent(
&[("X-Amz-Security-Token", "caller-value")],
Credentials::new(
"AKIDEXAMPLE",
"secret",
Some("session".into()),
None,
"test",
),
);
assert_eq!(header(&request, "x-amz-security-token"), ["session"]);
}
#[rstest::rstest]
#[case::bearer_overrides_the_signature("Bearer gateway", "Bearer gateway")]
#[case::a_stale_signature_is_replaced(
"AWS4-HMAC-SHA256 Credential=stale",
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date"
)]
fn a_callers_authorization_overrides_the_signature_unless_it_is_sigv4(
#[case] caller: &str,
#[case] expected: &str,
) {
let request = sent(
&[("Authorization", caller)],
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
);
let authorization = header(&request, "authorization");
assert_eq!(authorization.len(), 1);
assert_eq!(
authorization[0].split(", Signature=").next(),
Some(expected)
);
assert_eq!(header(&request, "x-amz-date"), ["20231114T221320Z"]);
}
const BEDROCK_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke";
const ARN_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Ainference-profile%2Fx/invoke?b=2&a=1";
const CALLER_SIGNING_HEADERS: &[(&str, &str)] = &[
("date", "Wed, 01 Jan 2020 00:00:00 GMT"),
("x-amz-date", "20200101T000000Z"),
("x-amz-security-token", "caller-token"),
];
const CALLER_TOKEN: &[(&str, &str)] = &[("x-amz-security-token", "caller-token")];
const CALLER_DATE: &[(&str, &str)] = &[("Date", "Wed, 01 Jan 2020 00:00:00 GMT")];
const HTTP_DATE: &str = "Tue, 14 Nov 2023 22:13:20 -0000";
const AMZ_DATE: &str = "20231114T221320Z";
#[rstest::rstest]
#[case::date_and_caller_token(URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "1b3652d523d5296b3904a80981e936a201167da9cb7bd200c2da8171dadb2c08")]
#[case::caller_token_only(URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "0fbd1a8e98efbfbcdb6fa9d665fe89ffffcdd8a7656865d63def21a5344d02dd")]
#[case::date_only(URL, CALLER_DATE, None, (Some(HTTP_DATE), None, None), "content-type;date;host", "cccdac7b04f20ebfe37a156c4e806a14e6c0a4b0aad11ca67e25dca337237085")]
#[case::session_replaces_the_caller_token_under_a_date(URL, CALLER_SIGNING_HEADERS, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "3406b8200cae95c8b66037dc5a9a838e1fe4da94b57b89fada3c05cde8a2c15c")]
#[case::session_replaces_the_caller_token(URL, CALLER_TOKEN, Some("session"), (None, Some(AMZ_DATE), Some("session")), "content-type;host;x-amz-date;x-amz-security-token", "d75425952d9f5ef7465a6ef3acabf15908c0edfd8ce2529cc3f0f8d2e9232e76")]
#[case::colon_in_the_model_path(BEDROCK_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "47d75a0dcb00499a025fc0f9f010f6dce89c7661eb54cb97c3a293e9c585b153")]
#[case::encoded_arn_and_query(ARN_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "8fb7d97a0ad5df9d1c3bbfc55897266059a5a2f35a1aaed9687f370047e78363")]
#[case::encoded_arn_and_query_without_a_date(ARN_URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "f5eb993af90c7896203bf6b3b85c0087087f712a0634e44ac0a04c9d87d6fe12")]
#[case::encoded_arn_and_query_with_a_session(ARN_URL, CALLER_DATE, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "15e8d5a9f64eb64366363f951db64de2a747959573a8cda9b5faa0fcbabc713c")]
fn botocore_caller_headers_sign_like_botocore(
#[case] url: &str,
#[case] caller: &[(&str, &str)],
#[case] session: Option<&str>,
#[case] sent: (Option<&str>, Option<&str>, Option<&str>),
#[case] signed_headers: &str,
#[case] signature: &str,
) {
let request = signed(
json(
url,
&caller
.iter()
.map(|(name, value)| (name.to_string(), value.to_string()))
.collect::<Vec<_>>(),
&json_value!({}),
None,
)
.unwrap(),
&SigV4Signer::new(
"us-east-1".into(),
"bedrock",
Credentials::new(
"AKIDEXAMPLE",
"secret",
session.map(str::to_string),
None,
"test",
),
)
.with_clock(fixed_clock)
.with_caller_headers(CallerSigningHeaders::Botocore),
)
.unwrap();
assert_eq!(
(
header(&request, "date"),
header(&request, "x-amz-date"),
header(&request, "x-amz-security-token"),
header(&request, "authorization"),
),
(
sent.0.into_iter().collect::<Vec<_>>(),
sent.1.into_iter().collect::<Vec<_>>(),
sent.2.into_iter().collect::<Vec<_>>(),
vec![format!(
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders={signed_headers}, Signature={signature}"
)
.as_str()],
)
);
}
#[test]

View file

@ -52,6 +52,6 @@ pub use credential::{
};
pub use error::{Error, ErrorDetail};
pub use http::CredentialPlacement;
pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams};
pub use params::{AwsParams, AwsSessionTag, ParamKind, ParamSpec, VertexParams};
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};

View file

@ -21,6 +21,14 @@ const fn spec(
}
}
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
pub struct AwsSessionTag {
#[serde(rename = "Key")]
pub key: String,
#[serde(rename = "Value")]
pub value: String,
}
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
#[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct AwsParams {
@ -50,11 +58,15 @@ pub struct AwsParams {
pub aws_external_id: Option<String>,
#[serde(default)]
pub aws_bedrock_runtime_endpoint: Option<String>,
#[serde(default)]
pub aws_session_tags: Option<Vec<AwsSessionTag>>,
}
impl AwsParams {
pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN";
pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
pub const DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION";
pub const SESSION_TAGS: &str = "aws_session_tags";
pub const ACCESS_KEY_ID: ParamSpec = spec(
"access key id",
@ -144,6 +156,7 @@ impl AwsParams {
Self::SPECS
.iter()
.flat_map(|spec| spec.wire.iter().copied())
.chain([Self::SESSION_TAGS])
}
/// The environment names the specs read, for hosts that resolve secrets up front.
@ -153,6 +166,7 @@ impl AwsParams {
.iter()
.flat_map(|spec| spec.env.iter().copied())
.chain([
AwsParams::DEFAULT_REGION_ENV,
AwsParams::ROLE_ARN_ENV,
AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV,
])
@ -205,6 +219,9 @@ impl AwsParams {
aws_sts_endpoint: value("aws_sts_endpoint"),
aws_external_id: value("aws_external_id"),
aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"),
aws_session_tags: optional_params
.get(Self::SESSION_TAGS)
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
}
}
}

View file

@ -1,7 +1,7 @@
mod aws;
mod vertex;
pub use aws::AwsParams;
pub use aws::{AwsParams, AwsSessionTag};
pub use vertex::VertexParams;
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads

View file

@ -1,11 +1,16 @@
use litellm_auth_types::AwsParams;
use rstest::rstest;
use serde_json::{Map, Value};
use serde_json::{Map, Value, json};
#[rstest]
fn every_named_param_roundtrips_without_loss() {
let filled: Map<String, Value> = AwsParams::fields()
.map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
.map(|name| match name {
AwsParams::SESSION_TAGS => {
(name.to_string(), json!([{"Key": "team", "Value": "genai"}]))
}
_ => (name.to_string(), Value::from(format!("value-of-{name}"))),
})
.collect();
let typed = AwsParams::from_optional_params(&filled);
let serialized = serde_json::to_value(&typed).unwrap();

View file

@ -0,0 +1,20 @@
[package]
name = "litellm-callbacks-budgets"
version = "0.1.0"
edition.workspace = true
license.workspace = true
repository.workspace = true
[dependencies]
litellm-callbacks.workspace = true
litellm-python-compat.workspace = true
litellm-redis-store.workspace = true
litellm-tracing.workspace = true
serde.workspace = true
serde_json.workspace = true
tracing.workspace = true
[dev-dependencies]
litellm-coroutine.workspace = true
rstest.workspace = true
tokio = { workspace = true, features = ["macros", "rt", "time", "test-util"] }

View file

@ -0,0 +1,128 @@
use std::sync::Arc;
use litellm_callbacks::{BudgetPlan, BudgetReservation, ReservationEntry};
use litellm_redis_store::spend_counters::SpendCounters;
use litellm_tracing::Logger;
use serde::Serialize;
use serde_json::{Map, Value, json};
use crate::{
engine::{self, BudgetCounterSpend, EngineError, Reservation, Settle, Settlement, UNAVAILABLE},
plan::Plan,
window::Moment,
};
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(tag = "admission", rename_all = "snake_case")]
pub enum Admission {
Skipped,
Held {
reservation: BudgetReservation,
},
Refused {
kind: &'static str,
fields: Map<String, Value>,
},
}
pub struct Budgets {
store: Arc<SpendCounters>,
logger: Option<Logger>,
}
fn refusal(error: &EngineError) -> Admission {
let (kind, fields) = match error {
EngineError::Exceeded(exceeded) => (
"budget_exceeded",
Map::from_iter([
("current_cost".to_owned(), json!(exceeded.current_cost)),
("max_budget".to_owned(), json!(exceeded.max_budget)),
("message".to_owned(), json!(exceeded.message)),
("entity_type".to_owned(), json!(exceeded.entity_type)),
("entity_id".to_owned(), json!(exceeded.entity_id)),
]),
),
EngineError::Unavailable => (
"budget_unavailable",
Map::from_iter([("detail".to_owned(), json!(UNAVAILABLE))]),
),
other => (
"budget_reservation_failed",
Map::from_iter([("message".to_owned(), json!(other.to_string()))]),
),
};
Admission::Refused { kind, fields }
}
fn reservation_record(reservation: &Reservation) -> BudgetReservation {
BudgetReservation {
reserved_cost: Some(reservation.reserved_cost),
entries: Some(
reservation
.entries
.iter()
.map(|entry| {
Some(ReservationEntry {
counter_key: Some(entry.counter_key.clone()),
entity_type: Some(entry.entity_type.clone()),
entity_id: Some(entry.entity_id.clone()),
reserved_cost: Some(entry.reserved_cost),
applied_adjustment: Some(entry.applied_adjustment),
..ReservationEntry::default()
})
})
.collect(),
),
input_cost: Some(reservation.input_cost),
input_tokens: reservation.input_tokens,
..BudgetReservation::default()
}
}
impl Budgets {
pub fn new(store: Arc<SpendCounters>) -> Self {
Self {
store,
logger: None,
}
}
pub fn with_logger(self, logger: Logger) -> Self {
Self {
logger: Some(logger),
..self
}
}
async fn logged<T>(&self, future: impl Future<Output = T>) -> T {
match &self.logger {
Some(logger) => logger.instrument(future).await,
None => future.await,
}
}
pub async fn settle(&self, request: &Settle<'_>) -> Result<Settlement, EngineError> {
self.logged(engine::settle(&self.store, request, &Moment::now()))
.await
}
pub async fn admit(
&self,
plan: BudgetPlan,
observed: &[BudgetCounterSpend],
) -> Result<Admission, String> {
let plan = Plan::try_from(plan).map_err(|error| error.to_string())?;
Ok(
match self
.logged(engine::reserve(&self.store, &plan, observed))
.await
{
Ok(None) => Admission::Skipped,
Ok(Some(reservation)) => Admission::Held {
reservation: reservation_record(&reservation),
},
Err(error) => refusal(&error),
},
)
}
}

View file

@ -0,0 +1,496 @@
use litellm_python_compat::{number::PyNumber, unicode::isspace};
use serde::{Deserialize, Serialize};
fn is_masked_key_name(name: &str) -> bool {
let Some(rest) = name.strip_prefix("sk-...") else {
return false;
};
rest.is_empty()
|| (rest.chars().count() == 4
&& rest
.chars()
.all(|c| !isspace(c) && !matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}')))
}
fn py_str(value: Option<&str>) -> &str {
value.unwrap_or("None")
}
#[derive(Clone, Debug, Deserialize)]
#[serde(tag = "check", rename_all = "snake_case", deny_unknown_fields)]
pub enum Check {
KeyMax {
spend: PyNumber,
max_budget: PyNumber,
token: Option<String>,
key_alias: Option<String>,
key_name: Option<String>,
throttle_pct: Option<PyNumber>,
},
KeyWindow {
spend: PyNumber,
max_budget: PyNumber,
budget_duration: String,
token: Option<String>,
},
KeySoft {
spend: PyNumber,
soft_budget: Option<PyNumber>,
},
TeamMax {
spend: PyNumber,
max_budget: PyNumber,
team_id: Option<String>,
},
TeamWindow {
spend: PyNumber,
max_budget: PyNumber,
budget_duration: String,
team_id: Option<String>,
},
UserWindow {
spend: PyNumber,
max_budget: PyNumber,
budget_duration: String,
user_id: String,
},
TeamSoft {
spend: Option<PyNumber>,
soft_budget: Option<PyNumber>,
},
ProjectMax {
spend: PyNumber,
max_budget: PyNumber,
project_id: Option<String>,
},
ProjectSoft {
spend: Option<PyNumber>,
soft_budget: Option<PyNumber>,
},
Organization {
spend: PyNumber,
max_budget: PyNumber,
org_id: String,
},
Tag {
spend: PyNumber,
max_budget: PyNumber,
tag: String,
},
ModelAccessGroup {
spend: PyNumber,
max_budget: PyNumber,
group: String,
},
User {
spend: PyNumber,
max_budget: PyNumber,
user_id: Option<String>,
},
TeamMember {
spend: PyNumber,
max_budget: PyNumber,
user_id: Option<String>,
team_id: Option<String>,
},
TeamMemberPositive {
spend: PyNumber,
max_budget: PyNumber,
user_id: Option<String>,
team_id: Option<String>,
},
EndUser {
spend: PyNumber,
max_budget: PyNumber,
end_user_id: Option<String>,
},
Proxy {
spend: PyNumber,
max_budget: PyNumber,
},
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Refusal {
pub current_cost: PyNumber,
pub max_budget: PyNumber,
pub message: String,
pub entity_type: &'static str,
pub entity_id: Option<String>,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(tag = "decision", rename_all = "snake_case")]
pub enum Decision {
Skip,
Pass,
Alert,
Throttle { pct: PyNumber },
Refuse(Refusal),
}
fn refuse(
spend: PyNumber,
max_budget: PyNumber,
message: String,
entity_type: &'static str,
entity_id: Option<String>,
) -> Decision {
Decision::Refuse(Refusal {
current_cost: spend,
max_budget,
message,
entity_type,
entity_id,
})
}
fn current(spend: PyNumber, max_budget: PyNumber, label: &str) -> String {
format!(
"Budget has been exceeded! {label} Current cost: {}, Max budget: {}",
spend.text(),
max_budget.text()
)
}
fn window(spend: PyNumber, max_budget: PyNumber) -> String {
format!("Spend=${}, Limit=${}", spend.fixed(4), max_budget.fixed(2))
}
fn at_or_over(spend: PyNumber, max_budget: PyNumber) -> bool {
max_budget.value().is_finite() && spend.value() >= max_budget.value()
}
fn over(spend: PyNumber, max_budget: PyNumber) -> bool {
max_budget.value().is_finite() && spend.value() > max_budget.value()
}
fn crossed(spend: Option<PyNumber>, soft_budget: Option<PyNumber>) -> Decision {
match (spend, soft_budget) {
(Some(spend), Some(soft)) if spend.value() >= soft.value() => Decision::Alert,
_ => Decision::Pass,
}
}
pub fn decide(check: &Check) -> Decision {
match check.clone() {
Check::KeyMax {
spend,
max_budget,
token,
key_alias,
key_name,
throttle_pct,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
if let Some(pct) = throttle_pct {
return Decision::Throttle { pct };
}
let label = key_alias.filter(|alias| !alias.is_empty());
let label = label.as_deref().unwrap_or("key");
let descriptor = match key_name.as_deref() {
Some(name) if !name.is_empty() && is_masked_key_name(name) => {
format!("{label} ({name})")
}
_ => label.to_owned(),
};
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("Key={descriptor}")),
"key",
token,
)
}
Check::KeyWindow {
spend,
max_budget,
budget_duration,
token,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"ExceededBudget: Key over {budget_duration} budget. {}",
window(spend, max_budget)
),
"key",
token,
)
}
Check::KeySoft { spend, soft_budget } => match soft_budget {
Some(soft) if soft.value() != 0.0 && spend.value() >= soft.value() => Decision::Alert,
_ => Decision::Pass,
},
Check::TeamMax {
spend,
max_budget,
team_id,
} => {
if !over(spend, max_budget) {
return Decision::Pass;
}
let label = format!("Team={}", py_str(team_id.as_deref()));
refuse(
spend,
max_budget,
current(spend, max_budget, &label),
"team",
team_id,
)
}
Check::TeamWindow {
spend,
max_budget,
budget_duration,
team_id,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"ExceededBudget: Team={} over {budget_duration} budget. {}",
py_str(team_id.as_deref()),
window(spend, max_budget)
),
"team",
team_id,
)
}
Check::UserWindow {
spend,
max_budget,
budget_duration,
user_id,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"ExceededBudget: User={user_id} over {budget_duration} budget. {}",
window(spend, max_budget)
),
"user",
Some(user_id),
)
}
Check::TeamSoft { spend, soft_budget } | Check::ProjectSoft { spend, soft_budget } => {
crossed(spend, soft_budget)
}
Check::ProjectMax {
spend,
max_budget,
project_id,
} => {
if !max_budget.value().is_finite() {
return Decision::Skip;
}
if spend.value() < max_budget.value() || spend.value().is_nan() {
return Decision::Pass;
}
let label = format!("Project={}", py_str(project_id.as_deref()));
refuse(
spend,
max_budget,
current(spend, max_budget, &label),
"project",
project_id,
)
}
Check::Organization {
spend,
max_budget,
org_id,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("Organization={org_id}")),
"organization",
Some(org_id),
)
}
Check::Tag {
spend,
max_budget,
tag,
} => {
if spend.value() <= max_budget.value() {
return Decision::Pass;
}
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("Tag={tag}")),
"tag",
Some(tag),
)
}
Check::ModelAccessGroup {
spend,
max_budget,
group,
} => {
if max_budget.value() <= 0.0 {
return Decision::Skip;
}
if spend.value() < max_budget.value() {
return Decision::Pass;
}
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("Model access group={group}")),
"model_access_group",
Some(group),
)
}
Check::User {
spend,
max_budget,
user_id,
} => {
if !at_or_over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"ExceededBudget: User={} over budget. Spend={}, Budget={}",
py_str(user_id.as_deref()),
spend.text(),
max_budget.text()
),
"user",
user_id,
)
}
Check::TeamMember {
spend,
max_budget,
user_id,
team_id,
} => {
if !max_budget.value().is_finite() {
return Decision::Skip;
}
if spend.value() < max_budget.value() || spend.value().is_nan() {
return Decision::Pass;
}
let (user, team) = (py_str(user_id.as_deref()), py_str(team_id.as_deref()));
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("User={user} in Team={team}")),
"team_member",
Some(format!("{user}:{team}")),
)
}
Check::TeamMemberPositive {
spend,
max_budget,
user_id,
team_id,
} => {
if spend.value() < max_budget.value()
|| spend.value().is_nan()
|| max_budget.value().is_nan()
{
return Decision::Pass;
}
let entity = format!(
"{}:{}",
py_str(user_id.as_deref()),
py_str(team_id.as_deref())
);
refuse(
spend,
max_budget,
current(spend, max_budget, &format!("TeamMember={entity}")),
"team_member",
Some(entity),
)
}
Check::EndUser {
spend,
max_budget,
end_user_id,
} => {
if spend.value() <= max_budget.value()
|| spend.value().is_nan()
|| max_budget.value().is_nan()
{
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"ExceededBudget: End User={} over budget. Spend={}, Budget={}",
py_str(end_user_id.as_deref()),
spend.text(),
max_budget.text()
),
"end_user",
end_user_id,
)
}
Check::Proxy { spend, max_budget } => {
if !over(spend, max_budget) {
return Decision::Pass;
}
refuse(
spend,
max_budget,
format!(
"Budget has been exceeded! Current cost: {}, Max budget: {}",
spend.text(),
max_budget.text()
),
"proxy",
None,
)
}
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
#[rstest]
#[case("sk-...", true)]
#[case("sk-...abcd", true)]
#[case("sk-...abc", false)]
#[case("sk-...ab d", false)]
#[case("sk-...abcde", false)]
#[case("sk-...ab\u{85}d", false)]
#[case("sk-...ab\u{3000}d", false)]
#[case("sk-...ab\u{202f}d", false)]
#[case("sk-...ab\u{2028}d", false)]
#[case("sk-...ab\u{200b}d", true)]
#[case("sk-...ab\u{180e}d", true)]
#[case("sk-...ab\u{feff}d", true)]
#[case("my-key", false)]
fn masked_key_names_match_the_proxy_pattern(#[case] name: &str, #[case] masked: bool) {
assert_eq!(is_masked_key_name(name), masked);
}
}

View file

@ -0,0 +1,992 @@
use std::sync::atomic::{AtomicUsize, Ordering};
use litellm_callbacks::{BudgetReservation, ReservationEntry};
use litellm_python_compat::repr::float_repr;
use litellm_redis_store::{
Number,
spend_counters::{NotASpendCounter, SpendCounter, SpendCounters},
};
use serde::{Deserialize, Serialize};
use crate::{
plan::{Counter, Plan, entity_type_value},
window::{self, Moment, ReservationError},
};
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
#[serde(default, deny_unknown_fields)]
pub struct BudgetCounterSpend {
pub counter_key: Option<String>,
pub spend: Option<f64>,
}
const EPSILON: f64 = 1e-12;
static ENTRY_IDS: AtomicUsize = AtomicUsize::new(0);
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Entry {
pub counter_key: String,
pub entity_type: String,
pub entity_id: String,
pub reserved_cost: f64,
pub applied_adjustment: f64,
#[serde(skip)]
id: usize,
#[serde(skip)]
counter: SpendCounter,
}
impl Entry {
pub fn counter(&self) -> &SpendCounter {
&self.counter
}
fn new(counter: &Counter, reserved_cost: f64) -> Self {
Self {
counter_key: counter.planned.counter_key.clone(),
entity_type: counter.planned.entity_type.clone(),
entity_id: counter.planned.entity_id.clone(),
reserved_cost,
applied_adjustment: 0.0,
id: ENTRY_IDS.fetch_add(1, Ordering::Relaxed),
counter: counter.counter.clone(),
}
}
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Reservation {
pub reserved_cost: f64,
pub entries: Vec<Entry>,
pub input_cost: f64,
pub input_tokens: Option<i64>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Exceeded {
pub current_cost: f64,
pub max_budget: f64,
pub message: String,
pub entity_type: Option<&'static str>,
pub entity_id: String,
}
#[derive(Debug)]
pub enum EngineError {
Exceeded(Exceeded),
Unavailable,
Inconsistent(String),
Store(litellm_redis_store::Error),
Plan(NotASpendCounter),
Reservation(ReservationError),
}
impl std::fmt::Display for EngineError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Self::Exceeded(exceeded) => f.write_str(&exceeded.message),
Self::Unavailable => f.write_str(UNAVAILABLE),
Self::Inconsistent(message) => f.write_str(message),
Self::Store(error) => write!(f, "{error}"),
Self::Plan(error) => write!(f, "{error}"),
Self::Reservation(error) => write!(f, "{error}"),
}
}
}
impl From<ReservationError> for EngineError {
fn from(error: ReservationError) -> Self {
Self::Reservation(error)
}
}
impl From<litellm_redis_store::Error> for EngineError {
fn from(error: litellm_redis_store::Error) -> Self {
Self::Store(error)
}
}
pub const UNAVAILABLE: &str = "Budget enforcement unavailable: the budget reservation could not be written to the spend counter backend, and fail_closed_budget_enforcement is enabled, so the request was rejected to avoid exceeding the configured budget. Retry shortly.";
fn exceeded(counter: &Counter, current_cost: f64, estimated_cost: Option<f64>) -> EngineError {
let planned = &counter.planned;
let estimate = estimated_cost
.map(|cost| format!("Estimated request cost: {}, ", float_repr(cost)))
.unwrap_or_default();
EngineError::Exceeded(Exceeded {
current_cost,
max_budget: planned.max_budget,
message: format!(
"Budget has been exceeded! {}={} Current cost: {}, {}Max budget: {}",
planned.entity_type,
planned.entity_id,
float_repr(current_cost),
estimate,
float_repr(planned.max_budget),
),
entity_type: entity_type_value(&planned.entity_type),
entity_id: planned
.spend_log_entity_id
.clone()
.unwrap_or_else(|| planned.entity_id.clone()),
})
}
fn observed_spend(observed: &[BudgetCounterSpend], counter: &Counter) -> f64 {
observed
.iter()
.find(|spend| spend.counter_key.as_deref() == Some(counter.key()))
.and_then(|spend| spend.spend)
.unwrap_or(counter.planned.fallback_spend)
}
fn applies_to(value: Option<f64>, adjustment: f64) -> bool {
match value {
None => false,
Some(value) => !(adjustment < 0.0 && value + adjustment < -EPSILON),
}
}
async fn can_apply(
store: &SpendCounters,
counter: &SpendCounter,
adjustment: f64,
) -> Result<bool, EngineError> {
Ok(applies_to(
store.read(counter).await?.value().map(Number::as_f64),
adjustment,
))
}
async fn increment_or_invalidate(
store: &SpendCounters,
applicable: Vec<(SpendCounter, Number)>,
) -> Result<(), EngineError> {
if applicable.is_empty() {
return Ok(());
}
if let Err(error) = store.increment_many(&applicable).await {
let counters: Vec<SpendCounter> = applicable.into_iter().map(|(c, _)| c).collect();
store.invalidate(&counters).await;
return Err(error.into());
}
Ok(())
}
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) enum Inconsistent {
Drop,
Refuse,
}
struct Adjustment {
index: usize,
counter: SpendCounter,
counter_key: String,
target: f64,
adjustment: f64,
}
async fn apply(
store: &SpendCounters,
adjustments: &[Adjustment],
inconsistent: Inconsistent,
) -> Result<Vec<bool>, EngineError> {
let mut consistent = Vec::with_capacity(adjustments.len());
for adjustment in adjustments {
consistent.push(can_apply(store, &adjustment.counter, adjustment.adjustment).await?);
}
if inconsistent == Inconsistent::Refuse
&& let Some((adjustment, _)) = adjustments
.iter()
.zip(&consistent)
.find(|(_, consistent)| !**consistent)
{
return Err(EngineError::Inconsistent(format!(
"Cannot resize budget reservation against inconsistent counter {}",
adjustment.counter_key
)));
}
let applicable: Vec<(SpendCounter, Number)> = adjustments
.iter()
.zip(&consistent)
.filter(|(_, consistent)| **consistent)
.map(|(adjustment, _)| {
(
adjustment.counter.clone(),
Number::Float(adjustment.adjustment),
)
})
.collect();
increment_or_invalidate(store, applicable).await?;
if inconsistent == Inconsistent::Drop {
for (adjustment, _) in adjustments
.iter()
.zip(&consistent)
.filter(|(_, consistent)| !**consistent)
{
store
.invalidate(std::slice::from_ref(&adjustment.counter))
.await;
}
}
Ok(consistent)
}
pub(crate) async fn apply_actual(
store: &SpendCounters,
entries: &mut [Entry],
actual: f64,
inconsistent: Inconsistent,
) -> Result<(), EngineError> {
let adjustments: Vec<Adjustment> = entries
.iter()
.enumerate()
.filter_map(|(index, entry)| {
let target = actual - entry.reserved_cost;
let adjustment = target - entry.applied_adjustment;
(adjustment != 0.0).then(|| Adjustment {
index,
counter: entry.counter.clone(),
counter_key: entry.counter_key.clone(),
target,
adjustment,
})
})
.collect();
apply(store, &adjustments, inconsistent).await?;
for adjustment in adjustments {
entries[adjustment.index].applied_adjustment = adjustment.target;
}
Ok(())
}
pub(crate) async fn release_best_effort(store: &SpendCounters, entries: &mut [Entry]) {
for entry in entries {
if apply_actual(store, std::slice::from_mut(entry), 0.0, Inconsistent::Drop)
.await
.is_err()
{
tracing::error!(
counter = entry.counter_key,
"Failed to release partial budget reservation during exception cleanup"
);
store.invalidate(std::slice::from_ref(&entry.counter)).await;
}
}
}
struct Admission<'a> {
store: &'a SpendCounters,
observed: &'a [BudgetCounterSpend],
plan: &'a Plan,
applied: Vec<Entry>,
cost: f64,
}
impl Admission<'_> {
async fn charge(&mut self, counters: &[Counter]) -> Result<(), EngineError> {
let fits = counters.iter().all(|counter| {
observed_spend(self.observed, counter) + self.cost <= counter.planned.max_budget
});
let groups: Vec<&[Counter]> = if fits {
vec![counters]
} else {
counters.chunks(1).collect()
};
for group in groups {
self.charge_group(group).await?;
}
Ok(())
}
async fn charge_group(&mut self, group: &[Counter]) -> Result<(), EngineError> {
let charged = self.cost;
let entries: Vec<Entry> = group.iter().map(|c| Entry::new(c, charged)).collect();
let ids: Vec<usize> = entries.iter().map(|entry| entry.id).collect();
self.applied.extend(entries);
let increments: Vec<(SpendCounter, Number)> = group
.iter()
.map(|counter| (counter.counter.clone(), Number::Float(charged)))
.collect();
let totals = match self.store.increment_many(&increments).await {
Ok(totals) => totals,
Err(error) => {
tracing::warn!(%error, "Skipping budget reservation because spend counter reservation failed");
let counters: Vec<SpendCounter> = increments.into_iter().map(|(c, _)| c).collect();
self.store.invalidate(&counters).await;
self.applied.retain(|entry| !ids.contains(&entry.id));
if self.plan.fail_closed {
return Err(EngineError::Unavailable);
}
return Ok(());
}
};
for ((counter, id), total) in group.iter().zip(ids).zip(totals) {
if !self.applied.iter().any(|entry| entry.id == id) {
continue;
}
let current = total.as_f64() - (charged - self.cost);
if current > counter.planned.max_budget {
self.over_budget(counter, id, current).await?;
}
}
Ok(())
}
async fn over_budget(
&mut self,
counter: &Counter,
id: usize,
current: f64,
) -> Result<(), EngineError> {
if self.plan.throttle_counter.as_deref() == Some(counter.key()) {
if let Some(position) = self.applied.iter().position(|entry| entry.id == id) {
let mut entry = self.applied.remove(position);
release_best_effort(self.store, std::slice::from_mut(&mut entry)).await;
}
return Ok(());
}
let remaining = counter.planned.max_budget - (current - self.cost);
if remaining <= EPSILON {
return Err(exceeded(counter, current, None));
}
if self.plan.fail_closed && current - counter.planned.max_budget > EPSILON {
return Err(exceeded(counter, current - self.cost, Some(self.cost)));
}
apply_actual(
self.store,
&mut self.applied,
remaining,
Inconsistent::Refuse,
)
.await?;
for entry in &mut self.applied {
entry.reserved_cost = remaining;
entry.applied_adjustment = 0.0;
}
self.cost = remaining;
Ok(())
}
}
pub async fn reserve(
store: &SpendCounters,
plan: &Plan,
observed: &[BudgetCounterSpend],
) -> Result<Option<Reservation>, EngineError> {
let counters = plan.counters().map_err(EngineError::Plan)?;
let mut admission = Admission {
store,
observed,
plan,
applied: Vec::new(),
cost: plan.reservation_cost,
};
if let Err(error) = admission.charge(&counters).await {
release_best_effort(store, &mut admission.applied).await;
return Err(error);
}
if admission.applied.is_empty() {
return Ok(None);
}
Ok(Some(Reservation {
reserved_cost: admission.cost,
input_cost: plan.input_cost.min(admission.cost),
input_tokens: plan.input_tokens,
entries: admission.applied,
}))
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Reseed {
pub counter_key: String,
pub actual_cost: f64,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct PendingIncrement {
pub counter_key: String,
pub increment: f64,
}
#[derive(Clone, Debug, PartialEq, Serialize)]
pub struct Settlement {
pub adjustments: Vec<Option<f64>>,
pub reseed: Vec<Reseed>,
pub pending: Vec<PendingIncrement>,
}
pub struct Settle<'a> {
pub reservation: &'a BudgetReservation,
pub window_resets: &'a [Option<String>],
pub observed: &'a [Option<f64>],
pub actual: f64,
pub apply_consistent: bool,
}
fn rejected(rejected: Option<&[String]>, member: &str) -> bool {
rejected.is_some_and(|names| names.iter().any(|name| name == member))
}
fn held_adjustment(
index: usize,
entry: &ReservationEntry,
window_reset: Option<&str>,
actual: f64,
default_reserved_cost: f64,
now: &Moment,
) -> Result<Option<(Adjustment, f64)>, EngineError> {
let names = entry.rejected.as_deref();
if rejected(names, "counter_key") {
return Err(ReservationError::type_error(
"str() of a reservation entry counter_key raises",
)
.into());
}
let Some(counter_key) = &entry.counter_key else {
return Ok(None);
};
let settled_cost = match window_reset.map(window::parse_reset).transpose()?.flatten() {
Some(reset_at) if reset_at <= *now => 0.0,
_ => actual,
};
let reserved_cost = match entry.reserved_cost {
Some(cost) if !rejected(names, "reserved_cost") => cost,
_ => default_reserved_cost,
};
if rejected(names, "applied_adjustment") {
return Err(ReservationError::value_error(
"could not convert the reservation entry applied_adjustment to float",
)
.into());
}
let target = settled_cost - reserved_cost;
let adjustment = target - entry.applied_adjustment.unwrap_or(0.0);
if adjustment == 0.0 {
return Ok(None);
}
Ok(Some((
Adjustment {
index,
counter: SpendCounter::parse(counter_key).map_err(EngineError::Plan)?,
counter_key: counter_key.clone(),
target,
adjustment,
},
settled_cost,
)))
}
pub async fn settle(
store: &SpendCounters,
request: &Settle<'_>,
now: &Moment,
) -> Result<Settlement, EngineError> {
let entries = request.reservation.entries.as_deref().unwrap_or_default();
if entries.iter().any(Option::is_none) {
return Err(
ReservationError::type_error("a budget reservation entry is not a mapping").into(),
);
}
let default_reserved_cost = request.reservation.reserved_cost.unwrap_or(0.0);
let mut adjustments = Vec::new();
let mut settled_costs = Vec::new();
for (index, entry) in entries.iter().enumerate() {
let Some(entry) = entry else { continue };
let window_reset = request.window_resets.get(index).and_then(Option::as_deref);
if let Some((adjustment, settled_cost)) = held_adjustment(
index,
entry,
window_reset,
request.actual,
default_reserved_cost,
now,
)? {
adjustments.push(adjustment);
settled_costs.push(settled_cost);
}
}
let consistent: Vec<bool> = adjustments
.iter()
.map(|adjustment| {
applies_to(
request.observed.get(adjustment.index).copied().flatten(),
adjustment.adjustment,
)
})
.collect();
if request.apply_consistent {
let applicable = adjustments
.iter()
.zip(&consistent)
.filter(|(_, consistent)| **consistent)
.map(|(adjustment, _)| {
(
adjustment.counter.clone(),
Number::Float(adjustment.adjustment),
)
})
.collect();
increment_or_invalidate(store, applicable).await?;
}
let mut settlement = Settlement {
adjustments: vec![None; entries.len()],
reseed: Vec::new(),
pending: Vec::new(),
};
for ((adjustment, settled_cost), consistent) in
adjustments.iter().zip(settled_costs).zip(consistent)
{
if !consistent {
settlement.reseed.push(Reseed {
counter_key: adjustment.counter_key.clone(),
actual_cost: settled_cost,
});
} else if !request.apply_consistent {
settlement.pending.push(PendingIncrement {
counter_key: adjustment.counter_key.clone(),
increment: adjustment.adjustment,
});
continue;
}
settlement.adjustments[adjustment.index] = Some(adjustment.target);
}
Ok(settlement)
}
#[cfg(test)]
mod tests {
use std::sync::Arc;
use litellm_redis_store::{
Number,
spend_counters::{SpendCounter, SpendCounters},
};
use rstest::rstest;
use serde_json::json;
use super::*;
use crate::{
plan::{Plan, PlannedCounter},
window::ErrorKind,
};
fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> PlannedCounter {
PlannedCounter {
counter_key: key.to_owned(),
max_budget,
fallback_spend: 0.0,
entity_type: entity_type.to_owned(),
entity_id: entity_id.to_owned(),
spend_log_entity_id: None,
}
}
fn plan(counters: Vec<PlannedCounter>, cost: f64) -> Plan {
Plan {
counters,
reservation_cost: cost,
input_cost: cost / 4.0,
input_tokens: Some(12),
throttle_counter: None,
fail_closed: false,
status_code: 400,
}
}
fn store(seed: &[(&str, f64)]) -> Arc<SpendCounters> {
let store = Arc::new(SpendCounters::new(None));
for (key, value) in seed {
store
.set_local(
&SpendCounter::parse(key).unwrap(),
Number::Float(*value),
None,
)
.unwrap();
}
store
}
fn value(store: &SpendCounters, key: &str) -> Option<f64> {
store
.get_local(&SpendCounter::parse(key).unwrap())
.unwrap()
.map(Number::as_f64)
}
const KEY: &str = "spend:key:hashed";
const TEAM: &str = "spend:team:t1";
#[rstest]
#[tokio::test]
async fn counters_that_fit_are_charged_the_whole_estimate() {
let store = store(&[(KEY, 1.0), (TEAM, 2.0)]);
let plan = plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
3.0,
);
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
assert_eq!(reservation.reserved_cost, 3.0);
assert_eq!(reservation.input_cost, 0.75);
assert_eq!(value(&store, KEY), Some(4.0));
assert_eq!(value(&store, TEAM), Some(5.0));
assert_eq!(
serde_json::to_value(&reservation).unwrap()["entries"][1],
json!({"counter_key": TEAM, "entity_type": "Team", "entity_id": "t1", "reserved_cost": 3.0, "applied_adjustment": 0.0})
);
}
#[rstest]
#[tokio::test]
async fn a_partial_fit_shrinks_every_applied_entry_to_what_is_left() {
let store = store(&[(KEY, 1.0), (TEAM, 9.0)]);
let plan = plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
3.0,
);
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
assert_eq!(reservation.reserved_cost, 1.0);
assert_eq!(value(&store, KEY), Some(2.0));
assert_eq!(value(&store, TEAM), Some(10.0));
}
#[rstest]
#[case::spent(
10.0,
false,
"Budget has been exceeded! Team=t1 Current cost: 11.0, Max budget: 10.0",
11.0
)]
#[case::fail_closed(
9.5,
true,
"Budget has been exceeded! Team=t1 Current cost: 9.5, Estimated request cost: 1.0, Max budget: 10.0",
9.5
)]
#[rstest]
#[tokio::test]
async fn an_over_budget_counter_refuses_and_releases_what_was_charged(
#[case] team_spend: f64,
#[case] fail_closed: bool,
#[case] message: &str,
#[case] current_cost: f64,
) {
let store = store(&[(KEY, 1.0), (TEAM, team_spend)]);
let plan = Plan {
fail_closed,
..plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
1.0,
)
};
let Err(EngineError::Exceeded(exceeded)) = reserve(&store, &plan, &[]).await else {
panic!("expected a refusal");
};
assert_eq!(exceeded.message, message);
assert_eq!(exceeded.current_cost, current_cost);
assert_eq!(exceeded.entity_type, Some("team"));
assert_eq!(value(&store, KEY), Some(1.0));
assert_eq!(value(&store, TEAM), Some(team_spend));
}
#[rstest]
#[tokio::test]
async fn a_throttled_key_releases_its_own_entry_and_keeps_the_rest() {
let store = store(&[(KEY, 10.0), (TEAM, 0.0)]);
let plan = Plan {
throttle_counter: Some(KEY.to_owned()),
..plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
1.0,
)
};
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
assert_eq!(reservation.entries.len(), 1);
assert_eq!(reservation.entries[0].counter_key, TEAM);
assert_eq!(value(&store, KEY), Some(10.0));
assert_eq!(value(&store, TEAM), Some(1.0));
}
fn record(reservation: &Reservation) -> BudgetReservation {
serde_json::from_value(serde_json::to_value(reservation).unwrap()).unwrap()
}
fn stamp(record: &mut BudgetReservation, settlement: &Settlement) {
for (entry, target) in record
.entries
.iter_mut()
.flatten()
.zip(&settlement.adjustments)
{
if let (Some(entry), Some(target)) = (entry, target) {
entry.applied_adjustment = Some(*target);
}
}
}
async fn observed(store: &SpendCounters, record: &BudgetReservation) -> Vec<Option<f64>> {
let mut observed = Vec::new();
for entry in record.entries.iter().flatten() {
let counter = entry
.as_ref()
.and_then(|entry| entry.counter_key.as_deref())
.and_then(|key| SpendCounter::parse(key).ok());
observed.push(match counter {
Some(counter) => store
.read(&counter)
.await
.unwrap()
.value()
.map(Number::as_f64),
None => None,
});
}
observed
}
async fn settle_as(
store: &SpendCounters,
record: &BudgetReservation,
resets: &[Option<String>],
actual: f64,
apply_consistent: bool,
) -> Result<Settlement, EngineError> {
let observed = observed(store, record).await;
settle(
store,
&Settle {
reservation: record,
window_resets: resets,
observed: &observed,
actual,
apply_consistent,
},
&Moment::from_unix_micros(NOW),
)
.await
}
async fn settle_now(
store: &SpendCounters,
record: &BudgetReservation,
resets: &[Option<String>],
actual: f64,
) -> Result<Settlement, EngineError> {
settle_as(store, record, resets, actual, true).await
}
#[rstest]
#[tokio::test]
async fn a_settlement_that_leaves_writes_to_the_caller_hands_back_the_consistent_increments() {
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
let plan = plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
2.0,
);
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
store
.invalidate(&[SpendCounter::parse(TEAM).unwrap()])
.await;
let settlement = settle_as(&store, &record, &[], 0.5, false).await.unwrap();
assert_eq!(value(&store, KEY), Some(3.0));
assert_eq!(
settlement.pending,
vec![PendingIncrement {
counter_key: KEY.to_owned(),
increment: -1.5
}]
);
assert_eq!(
settlement.reseed,
vec![Reseed {
counter_key: TEAM.to_owned(),
actual_cost: 0.5
}]
);
assert_eq!(settlement.adjustments, vec![None, Some(-1.5)]);
}
#[rstest]
#[tokio::test]
async fn a_counter_the_caller_saw_missing_is_reseeded_whatever_the_store_holds_now() {
let store = store(&[(KEY, 1.0)]);
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
let settlement = settle(
&store,
&Settle {
reservation: &record,
window_resets: &[],
observed: &[None],
actual: 0.5,
apply_consistent: true,
},
&Moment::from_unix_micros(NOW),
)
.await
.unwrap();
assert_eq!(value(&store, KEY), Some(3.0));
assert_eq!(
settlement.reseed,
vec![Reseed {
counter_key: KEY.to_owned(),
actual_cost: 0.5
}]
);
}
const NOW: i128 = 1_763_035_200_000_000;
#[rstest]
#[case::success(0.4, 1.4)]
#[case::refund(0.0, 1.0)]
#[tokio::test]
async fn settling_moves_each_counter_from_the_hold_to_the_actual_cost(
#[case] actual: f64,
#[case] after: f64,
) {
let store = store(&[(KEY, 1.0)]);
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let mut record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
let settlement = settle_now(&store, &record, &[], actual).await.unwrap();
assert_eq!(settlement.adjustments, vec![Some(actual - 2.0)]);
stamp(&mut record, &settlement);
let again = settle_now(&store, &record, &[], actual).await.unwrap();
assert_eq!(again.adjustments, vec![None]);
assert_eq!(value(&store, KEY), Some(after));
}
#[rstest]
#[case::missing(None, 0.4, None)]
#[case::would_go_negative(Some(0.5), 0.0, Some(0.5))]
#[tokio::test]
async fn settling_an_inconsistent_counter_reports_it_and_leaves_it_alone(
#[case] before_settle: Option<f64>,
#[case] actual: f64,
#[case] after: Option<f64>,
) {
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
let plan = plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
2.0,
);
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
let key = SpendCounter::parse(KEY).unwrap();
match before_settle {
Some(spend) => store.set_local(&key, Number::Float(spend), None).unwrap(),
None => store.invalidate(std::slice::from_ref(&key)).await,
}
let settlement = settle_now(&store, &record, &[], actual).await.unwrap();
assert_eq!(
settlement.reseed,
vec![Reseed {
counter_key: KEY.to_owned(),
actual_cost: actual
}]
);
assert_eq!(value(&store, KEY), after);
assert_eq!(value(&store, TEAM), Some(2.0 + (actual - 2.0)));
assert_eq!(settlement.adjustments, vec![Some(actual - 2.0); 2]);
}
#[rstest]
#[case::reset_before_settlement("2025-11-13T11:59:59+00:00", 0.0)]
#[case::reset_at_settlement("2025-11-13T12:00:00Z", 0.0)]
#[case::still_open("2025-11-13T12:00:00.000001+00:00", 0.4)]
#[case::unparsable("next month", 0.4)]
#[tokio::test]
async fn an_entry_whose_window_reset_before_settlement_settles_at_no_cost(
#[case] reset: &str,
#[case] settled: f64,
) {
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
let plan = plan(
vec![
counter(KEY, 10.0, "Key", "hashed"),
counter(TEAM, 10.0, "Team", "t1"),
],
2.0,
);
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
store
.invalidate(&[SpendCounter::parse(TEAM).unwrap()])
.await;
let settlement = settle_now(&store, &record, &[Some(reset.to_owned()), None], 0.4)
.await
.unwrap();
assert_eq!(value(&store, KEY), Some(1.0 + settled));
assert_eq!(
settlement.adjustments,
vec![Some(settled - 2.0), Some(0.4 - 2.0)]
);
assert_eq!(
settlement.reseed,
vec![Reseed {
counter_key: TEAM.to_owned(),
actual_cost: 0.4
}]
);
}
fn held(entries: serde_json::Value) -> BudgetReservation {
serde_json::from_value(json!({"reserved_cost": 2.0, "entries": entries})).unwrap()
}
#[rstest]
#[case::not_a_mapping(json!([{"counter_key": KEY, "reserved_cost": 2.0}, null]), None, ErrorKind::Type)]
#[case::naive_reset(json!([{"counter_key": KEY, "reserved_cost": 2.0}]), Some("2025-11-13T12:00:00"), ErrorKind::Type)]
#[case::applied_adjustment_float_refuses(json!([{"counter_key": KEY, "rejected": ["applied_adjustment"]}]), None, ErrorKind::Value)]
#[tokio::test]
async fn an_entry_python_raises_on_fails_the_settlement_before_any_counter_moves(
#[case] entries: serde_json::Value,
#[case] reset: Option<&str>,
#[case] kind: ErrorKind,
) {
let store = store(&[(KEY, 3.0)]);
let Err(EngineError::Reservation(error)) =
settle_now(&store, &held(entries), &[reset.map(str::to_owned)], 0.5).await
else {
panic!("expected a reservation error");
};
assert_eq!(error.kind, kind);
assert_eq!(value(&store, KEY), Some(3.0));
}
#[rstest]
#[case::no_counter_key(json!([{"reserved_cost": 2.0}]), 3.0, vec![None])]
#[case::reserved_cost_float_refuses(json!([{"counter_key": KEY, "rejected": ["reserved_cost"]}]), 1.5, vec![Some(-1.5)])]
#[case::reserved_cost_missing(json!([{"counter_key": KEY}]), 1.5, vec![Some(-1.5)])]
#[case::own_reserved_cost(json!([{"counter_key": KEY, "reserved_cost": 1.0, "applied_adjustment": 0.25}]), 2.25, vec![Some(-0.5)])]
#[tokio::test]
async fn each_entry_member_is_read_as_budgets_reads_it(
#[case] entries: serde_json::Value,
#[case] after: f64,
#[case] adjustments: Vec<Option<f64>>,
) {
let store = store(&[(KEY, 3.0)]);
let settlement = settle_now(&store, &held(entries), &[], 0.5).await.unwrap();
assert_eq!(settlement.adjustments, adjustments);
assert_eq!(value(&store, KEY), Some(after));
}
}

View file

@ -0,0 +1,16 @@
mod admission;
mod checks;
mod engine;
mod plan;
mod window;
pub use admission::{Admission, Budgets};
pub use checks::{Check, Decision, Refusal, decide};
pub use engine::{
BudgetCounterSpend, EngineError, Entry, Exceeded, PendingIncrement, Reseed, Reservation,
Settle, Settlement, UNAVAILABLE, reserve, settle,
};
pub use litellm_callbacks::{BudgetPlan, BudgetReservation};
pub use litellm_python_compat::number::PyNumber;
pub use plan::{Plan, PlannedCounter};
pub use window::{ErrorKind, Moment, ReservationError};

View file

@ -0,0 +1,114 @@
use litellm_callbacks::{BudgetPlan, BudgetPlanCounter};
use litellm_redis_store::spend_counters::{NotASpendCounter, SpendCounter};
#[derive(Clone, Debug, PartialEq)]
pub struct PlannedCounter {
pub counter_key: String,
pub max_budget: f64,
pub fallback_spend: f64,
pub entity_type: String,
pub entity_id: String,
pub spend_log_entity_id: Option<String>,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Plan {
pub counters: Vec<PlannedCounter>,
pub reservation_cost: f64,
pub input_cost: f64,
pub input_tokens: Option<i64>,
pub throttle_counter: Option<String>,
pub fail_closed: bool,
pub status_code: u16,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct MissingMember(pub &'static str);
impl std::fmt::Display for MissingMember {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "the budget plan has no {}", self.0)
}
}
fn required<T>(value: Option<T>, member: &'static str) -> Result<T, MissingMember> {
value.ok_or(MissingMember(member))
}
impl TryFrom<BudgetPlanCounter> for PlannedCounter {
type Error = MissingMember;
fn try_from(counter: BudgetPlanCounter) -> Result<Self, Self::Error> {
Ok(Self {
counter_key: required(counter.counter_key, "counters[].counter_key")?,
max_budget: required(counter.max_budget, "counters[].max_budget")?,
fallback_spend: required(counter.fallback_spend, "counters[].fallback_spend")?,
entity_type: required(counter.entity_type, "counters[].entity_type")?,
entity_id: required(counter.entity_id, "counters[].entity_id")?,
spend_log_entity_id: counter.spend_log_entity_id,
})
}
}
impl TryFrom<BudgetPlan> for Plan {
type Error = MissingMember;
fn try_from(plan: BudgetPlan) -> Result<Self, Self::Error> {
Ok(Self {
counters: required(plan.counters, "counters")?
.into_iter()
.map(PlannedCounter::try_from)
.collect::<Result<_, _>>()?,
reservation_cost: required(plan.reservation_cost, "reservation_cost")?,
input_cost: required(plan.input_cost, "input_cost")?,
input_tokens: plan.input_tokens,
throttle_counter: plan.throttle_counter,
fail_closed: plan.fail_closed.unwrap_or_default(),
status_code: required(plan.status_code, "status_code")?,
})
}
}
#[derive(Clone, Debug)]
pub(crate) struct Counter {
pub planned: PlannedCounter,
pub counter: SpendCounter,
}
impl Counter {
pub fn key(&self) -> &str {
&self.planned.counter_key
}
}
impl Plan {
pub(crate) fn counters(&self) -> Result<Vec<Counter>, NotASpendCounter> {
self.counters
.iter()
.map(|planned| {
Ok(Counter {
counter: SpendCounter::parse(&planned.counter_key)?,
planned: planned.clone(),
})
})
.collect()
}
}
pub(crate) fn entity_type_value(entity_type: &str) -> Option<&'static str> {
match entity_type {
"Key" => Some("key"),
"Team" => Some("team"),
"TeamMember" => Some("team_member"),
"User" => Some("user"),
"EndUser" => Some("end_user"),
"Tag" => Some("tag"),
"Model access group" => Some("model_access_group"),
"Organization" => Some("organization"),
"Project" => Some("project"),
"customer_org" => Some("customer_org"),
"customer_team" => Some("customer_team"),
"customer_member" => Some("customer_member"),
_ => None,
}
}

View file

@ -0,0 +1,109 @@
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_python_compat::datetime::{IsoDateTime, fromisoformat};
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ErrorKind {
Type,
Value,
}
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct ReservationError {
pub kind: ErrorKind,
pub message: String,
}
impl ReservationError {
pub(crate) fn type_error(message: impl Into<String>) -> Self {
Self {
kind: ErrorKind::Type,
message: message.into(),
}
}
pub(crate) fn value_error(message: impl Into<String>) -> Self {
Self {
kind: ErrorKind::Value,
message: message.into(),
}
}
}
impl std::fmt::Display for ReservationError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.message)
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
pub struct Moment(i128);
impl Moment {
pub fn now() -> Self {
let since = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default();
Self(i128::try_from(since.as_micros()).unwrap_or(i128::MAX))
}
pub const fn from_unix_micros(micros: i128) -> Self {
Self(micros)
}
}
/// A reservation entry's `window_reset_at`, read as the proxy's `_coerce_datetime` reads it:
/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, `None` where that raises.
pub(crate) fn parse_reset(text: &str) -> Result<Option<Moment>, ReservationError> {
match fromisoformat(&text.replace('Z', "+00:00")) {
None => Ok(None),
Some(IsoDateTime::Aware(micros)) => Ok(Some(Moment(i128::from(micros)))),
Some(IsoDateTime::Naive(_)) => Err(ReservationError::type_error(
"can't compare offset-naive and offset-aware datetimes",
)),
}
}
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::*;
const NOON: i128 = 1_763_035_200_000_000;
#[rstest]
#[case::isoformat("2025-11-13T12:00:00+00:00", Some(NOON))]
#[case::zulu("2025-11-13T12:00:00Z", Some(NOON))]
#[case::micros("2025-11-13T12:00:00.000001+00:00", Some(NOON + 1))]
#[case::short_fraction("2025-11-13T12:00:00.5+00:00", Some(NOON + 500_000))]
#[case::long_fraction("2025-11-13T12:00:00.1234567+00:00", Some(NOON + 123_456))]
#[case::offset("2025-11-13T13:30:00+01:30", Some(NOON))]
#[case::negative_offset("2025-11-13T11:00:00-01:00", Some(NOON))]
#[case::space_separator("2025-11-13 12:00+00:00", Some(NOON))]
#[case::basic("20251113T120000+0000", Some(NOON))]
#[case::not_a_date("soon", None)]
#[case::bad_day("2025-02-30T12:00:00+00:00", None)]
#[case::bad_hour("2025-11-13T25:00:00+00:00", None)]
#[case::midnight_rollover("2025-11-13T24:00:00+00:00", Some(NOON + 43_200_000_000))]
#[case::trailing("2025-11-13T12:00:00+00:00 ", None)]
fn an_aware_reset_parses_as_python_fromisoformat_does(
#[case] text: &str,
#[case] expected: Option<i128>,
) {
assert_eq!(
parse_reset(text).unwrap(),
expected.map(Moment::from_unix_micros)
);
}
#[rstest]
#[case::date_only("2025-11-13")]
#[case::no_offset("2025-11-13T12:00:00")]
fn a_naive_reset_raises_the_comparison_type_error(#[case] text: &str) {
assert_eq!(
parse_reset(text).unwrap_err(),
ReservationError::type_error("can't compare offset-naive and offset-aware datetimes")
);
}
}

View file

@ -0,0 +1,230 @@
use std::sync::Arc;
use litellm_callbacks::{BudgetPlan, BudgetPlanCounter, BudgetReservation};
use litellm_callbacks_budgets::{Admission, BudgetCounterSpend, Budgets, Settle, Settlement};
use litellm_redis_store::{
Number,
spend_counters::{SpendCounter, SpendCounters},
};
use rstest::rstest;
use serde_json::json;
fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> BudgetPlanCounter {
BudgetPlanCounter {
counter_key: Some(key.to_owned()),
max_budget: Some(max_budget),
fallback_spend: Some(0.0),
entity_type: Some(entity_type.to_owned()),
entity_id: Some(entity_id.to_owned()),
spend_log_entity_id: None,
}
}
fn plan(counters: Vec<BudgetPlanCounter>, cost: f64) -> BudgetPlan {
BudgetPlan {
counters: Some(counters),
reservation_cost: Some(cost),
input_cost: Some(cost / 4.0),
input_tokens: Some(12),
throttle_counter: None,
fail_closed: Some(false),
status_code: Some(400),
reservation_id: Some("reservation-1".to_owned()),
..BudgetPlan::default()
}
}
fn store(seed: &[(&str, f64)]) -> Arc<SpendCounters> {
let store = Arc::new(SpendCounters::new(None));
for (key, value) in seed {
store
.set_local(
&SpendCounter::parse(key).unwrap(),
Number::Float(*value),
None,
)
.unwrap();
}
store
}
fn value(store: &SpendCounters, key: &str) -> Option<f64> {
store
.get_local(&SpendCounter::parse(key).unwrap())
.unwrap()
.map(Number::as_f64)
}
const KEY: &str = "spend:key:hashed";
fn observed(spends: &[(&str, f64)]) -> Vec<BudgetCounterSpend> {
spends
.iter()
.map(|(key, spend)| BudgetCounterSpend {
counter_key: Some((*key).to_owned()),
spend: Some(*spend),
})
.collect()
}
async fn admit(budgets: &Budgets, plan: &BudgetPlan, spends: &[(&str, f64)]) -> Admission {
budgets
.admit(plan.clone(), &observed(spends))
.await
.unwrap()
}
async fn held(budgets: &Budgets, plan: &BudgetPlan) -> BudgetReservation {
let Admission::Held { reservation } = admit(budgets, plan, &[]).await else {
panic!("expected a reservation");
};
reservation
}
fn stamp(record: &mut BudgetReservation, adjustments: &[Option<f64>]) {
for (entry, target) in record.entries.iter_mut().flatten().zip(adjustments) {
if let (Some(entry), Some(target)) = (entry, target) {
entry.applied_adjustment = Some(*target);
}
}
}
async fn settle(
budgets: &Budgets,
store: &SpendCounters,
record: &BudgetReservation,
actual: f64,
) -> Settlement {
let mut observed = Vec::new();
for entry in record.entries.iter().flatten().flatten() {
let counter = SpendCounter::parse(entry.counter_key.as_deref().unwrap()).unwrap();
observed.push(
store
.read(&counter)
.await
.unwrap()
.value()
.map(Number::as_f64),
);
}
budgets
.settle(&Settle {
reservation: record,
window_resets: &[],
observed: &observed,
actual,
apply_consistent: true,
})
.await
.unwrap()
}
#[rstest]
#[tokio::test]
async fn an_admission_holds_and_returns_the_hold_as_the_reservation() {
let store = store(&[(KEY, 1.0)]);
let budgets = Budgets::new(Arc::clone(&store));
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let mut record = held(&budgets, &plan).await;
assert_eq!(value(&store, KEY), Some(3.0));
assert_eq!(record.reserved_cost, Some(2.0));
assert_eq!(record.input_cost, Some(0.5));
assert_eq!(record.input_tokens, Some(12));
assert_eq!(record.finalized, None);
let settled = settle(&budgets, &store, &record, 0.5).await;
assert_eq!(settled.adjustments, vec![Some(-1.5)]);
assert!(settled.reseed.is_empty());
assert_eq!(value(&store, KEY), Some(1.5));
stamp(&mut record, &settled.adjustments);
let again = settle(&budgets, &store, &record, 0.5).await;
assert_eq!(again.adjustments, vec![None]);
assert_eq!(value(&store, KEY), Some(1.5));
}
#[rstest]
#[case::no_counters(BudgetPlan { counters: None, ..plan(vec![], 2.0) })]
#[case::no_status_code(BudgetPlan { status_code: None, ..plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0) })]
#[tokio::test]
async fn a_plan_missing_a_required_member_fails_without_holding(#[case] plan: BudgetPlan) {
let store = store(&[(KEY, 1.0)]);
let budgets = Budgets::new(Arc::clone(&store));
assert!(budgets.admit(plan, &[]).await.is_err());
assert_eq!(value(&store, KEY), Some(1.0));
}
#[rstest]
#[case::once(&[0.25])]
#[case::twice_on_the_same_cost(&[0.25, 0.25])]
#[case::grown_then_shrunk(&[0.75, 0.25])]
#[tokio::test]
async fn a_settlement_through_the_reservation_dict_lands_on_the_last_cost(#[case] costs: &[f64]) {
let store = store(&[(KEY, 1.0)]);
let budgets = Budgets::new(Arc::clone(&store));
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let mut record = held(&budgets, &plan).await;
for cost in costs {
let settled = settle(&budgets, &store, &record, *cost).await;
stamp(&mut record, &settled.adjustments);
}
assert_eq!(value(&store, KEY), Some(1.0 + costs[costs.len() - 1]));
assert_eq!(record.finalized, None);
}
#[rstest]
#[tokio::test]
async fn a_refusal_returns_the_typed_budget_error() {
let store = store(&[(KEY, 10.0)]);
let budgets = Budgets::new(Arc::clone(&store));
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let Admission::Refused { kind, fields } = admit(&budgets, &plan, &[]).await else {
panic!("expected a refusal");
};
assert_eq!(kind, "budget_exceeded");
assert_eq!(
serde_json::Value::Object(fields),
json!({
"current_cost": 12.0,
"max_budget": 10.0,
"message": "Budget has been exceeded! Key=hashed Current cost: 12.0, Max budget: 10.0",
"entity_type": "key",
"entity_id": "hashed",
})
);
assert_eq!(value(&store, KEY), Some(10.0));
}
#[rstest]
#[case::observed(&[(KEY, 1.0)])]
#[case::not_observed(&[])]
#[tokio::test]
async fn a_counter_python_observed_is_charged_on_the_store(#[case] observed: &[(&str, f64)]) {
let store = store(&[(KEY, 1.0)]);
let budgets = Budgets::new(Arc::clone(&store));
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
let Admission::Held { .. } = admit(&budgets, &plan, observed).await else {
panic!("expected the reservation");
};
assert_eq!(value(&store, KEY), Some(3.0));
}
#[rstest]
#[tokio::test]
async fn an_observed_spend_with_no_room_charges_counters_one_at_a_time_and_refuses_the_full_one() {
const TEAM: &str = "spend:team:team-1";
let store = store(&[(KEY, 10.0), (TEAM, 0.0)]);
let budgets = Budgets::new(Arc::clone(&store));
let plan = plan(
vec![
counter(TEAM, 10.0, "Team", "team-1"),
counter(KEY, 10.0, "Key", "hashed"),
],
2.0,
);
let Admission::Refused { kind, .. } = admit(&budgets, &plan, &[(KEY, 10.0), (TEAM, 0.0)]).await
else {
panic!("expected the refusal");
};
assert_eq!(kind, "budget_exceeded");
assert_eq!(value(&store, KEY), Some(10.0));
assert_eq!(value(&store, TEAM), Some(0.0));
}

View file

@ -159,6 +159,7 @@
"endpoint_type",
"request_body",
"chunks",
"error"
"error",
"head"
]
}

View file

@ -38,6 +38,7 @@ struct PassThroughStream {
/// What the Messages stream iterator keeps for its end-of-stream billing.
struct DeliveredStream {
head: Py<PyAny>,
chunks: Py<PyList>,
first_chunk: Option<Py<PyAny>>,
}
@ -65,22 +66,14 @@ pub struct LegacyLogging {
/// When the provider request went out, until its window is recorded (Messages only).
api_started: Option<f64>,
host_answered: bool,
/// Whether the call's Messages stream starts only on its first read, as Python's iterator
/// does. Only `azure_ai`: by the owner's ruling `anthropic` keeps main's billing of an
/// unread stream.
/// Whether the call's stream starts only on its first read, as Python's Messages iterator
/// does.
stream_starts_on_first_read: bool,
/// Set once the route sends through the host's own client, whose timed `post` already
/// reports the exchange.
host_client: Arc<AtomicBool>,
}
/// The one provider whose unread Messages stream bills nothing, as Python's does.
const STREAM_STARTS_ON_FIRST_READ_PROVIDER: &str = "azure_ai";
/// The one provider whose exchange through the host's client is timed only by that client, as
/// Python's is. By the owner's ruling `anthropic` keeps main's second report.
const HOST_CLIENT_TIMED_PROVIDER: &str = "azure_ai";
fn retry_reason(reason: &RetryReason) -> Value {
match reason {
RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}),
@ -504,6 +497,7 @@ impl LegacyLogging {
),
&stream.chunks,
error,
&stream.head,
),
);
match scheduled {
@ -675,8 +669,7 @@ impl LegacyLogging {
)?;
if self.operation == LoggingOperation::Messages
&& self.asynchronous
&& !(context.custom_llm_provider == HOST_CLIENT_TIMED_PROVIDER
&& self.host_client.load(Ordering::Acquire))
&& !self.host_client.load(Ordering::Acquire)
&& self.times_provider_exchange(py)?
{
self.api_started = Some(epoch_seconds());
@ -729,8 +722,7 @@ impl LegacyLogging {
) -> PyResult<HookStep<Self, ()>> {
use litellm_host::interceptors::{Cost, ResultSource};
self.stream_starts_on_first_read =
facts.provider.provider == STREAM_STARTS_ON_FIRST_READ_PROVIDER;
self.stream_starts_on_first_read = self.operation == LoggingOperation::Messages;
self.cache_key = match &facts.source {
ResultSource::Provider => None,
ResultSource::Cache { key } => Some(key.clone()),
@ -940,6 +932,7 @@ impl LegacyLogging {
),
)?;
self.stream = Some(DeliveredStream {
head: head.clone_ref(py),
chunks: PyList::empty(py).unbind(),
first_chunk: None,
});
@ -995,6 +988,7 @@ impl PythonOwned for LegacyLogging {
visit.call(&self.response)?;
visit.call(&self.error)?;
if let Some(stream) = &self.stream {
visit.call(&stream.head)?;
visit.call(&stream.chunks)?;
visit.call(&stream.first_chunk)?;
}

View file

@ -107,7 +107,7 @@ FAKES = {
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record(
'stream_success', list(chunks)
),
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error),
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error, head: logger.record('stream_failure', error),
}
assert FAKES.keys() == CONTRACT.keys(), sorted(FAKES.keys() ^ CONTRACT.keys())
for name, fake in FAKES.items():

View file

@ -1,6 +1,7 @@
use std::time::{SystemTime, UNIX_EPOCH};
use litellm_callbacks::Moment;
use litellm_python_compat::datetime::UtcDateTime;
use sha2::{Digest, Sha256};
use crate::settings::{Granularity, Settings};
@ -58,16 +59,18 @@ impl Stamp {
pub fn utc(time: SystemTime) -> Self {
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
let seconds = elapsed.as_secs();
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
let of_day = seconds % 86_400;
let moment = UtcDateTime::from_timestamp(
i64::try_from(seconds).unwrap_or(i64::MAX),
elapsed.subsec_micros(),
);
Self {
year,
month,
day,
hour: (of_day / 3600) as u32,
minute: (of_day / 60 % 60) as u32,
second: (of_day % 60) as u32,
micros: elapsed.subsec_micros(),
year: moment.year,
month: moment.month,
day: moment.day,
hour: moment.hour,
minute: moment.minute,
second: moment.second,
micros: moment.microsecond,
}
}
@ -84,24 +87,6 @@ impl Stamp {
}
}
fn civil_from_days(days: i64) -> (i64, u32, u32) {
let shifted = days + 719_468;
let era = shifted.div_euclid(146_097);
let day_of_era = shifted.rem_euclid(146_097);
let year_of_era =
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
let month_index = (5 * day_of_year + 2) / 153;
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
let month = if month_index < 10 {
month_index + 3
} else {
month_index - 9
} as u32;
let year = year_of_era + era * 400 + i64::from(month <= 2);
(year, month, day)
}
pub fn logging_id(start: &Stamp, id: &str) -> String {
format!("time-{}_{id}", start.clock_micros())
}
@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str {
}
pub fn quote(key: &str) -> String {
let mut encoded = String::with_capacity(key.len());
for byte in key.bytes() {
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
encoded.push(byte as char);
} else {
encoded.push_str(&format!("%{byte:02X}"));
}
}
encoded
litellm_python_compat::urllib::quote(key, "/")
}
pub fn dns_suffix(region: Option<&str>) -> &'static str {

Some files were not shown because too many files have changed in this diff Show more