mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-16 09:22:22 +00:00
Merge remote-tracking branch 'origin/main' into litellm_messages_drop_unsigned_thinking
# Conflicts: # litellm/rust/crates/inference-messages/src/handler.rs
This commit is contained in:
commit
e130ab8ca8
451 changed files with 44348 additions and 12031 deletions
3
.github/linters/ruff-strict.toml
vendored
3
.github/linters/ruff-strict.toml
vendored
|
|
@ -2,7 +2,7 @@ extend = "../../pyproject.toml"
|
|||
|
||||
[lint]
|
||||
preview = true
|
||||
select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"]
|
||||
select = ["ANN", "ASYNC230", "B004", "B006", "B008", "B009", "B010", "B018", "B019", "B021", "B026", "B033", "BLE", "C401", "C404", "C405", "C408", "C414", "C419", "C901", "D202", "D403", "D417", "D419", "DTZ001", "DTZ003", "DTZ005", "DTZ006", "DTZ007", "DTZ011", "EXE001", "EXE002", "F401", "FURB136", "FURB168", "FURB188", "I001", "LOG015", "N999", "PERF102", "PERF401", "PERF402", "PERF403", "PIE790", "PIE800", "PIE804", "PIE810", "PLC0206", "PLC0208", "PLC0414", "PLR0124", "PLR0206", "PLR0402", "PLR0916", "PLR1704", "PLR1711", "PLR1714", "PLR1730", "PLR2044", "PLW0127", "PLW0133", "PLW0602", "PLW0603", "PLW1508", "PLW1510", "PYI030", "PYI036", "PYI041", "PYI064", "RET501", "RET504", "RUF010", "RUF012", "RUF015", "RUF019", "RUF022", "RUF023", "RUF046", "RUF051", "RUF059", "RUF100", "S110", "S112", "SIM101", "SIM102", "SIM103", "SIM113", "SIM114", "SIM115", "SIM117", "SIM118", "SIM201", "SIM210", "SIM211", "SIM222", "SIM401", "TC004", "TC005", "TID251", "TRY002", "TRY004", "TRY201", "TRY203", "TRY300", "UP006", "UP007", "UP008", "UP012", "UP018", "UP024", "UP028", "UP031", "UP032", "UP034", "UP035", "UP036", "UP037", "UP045"]
|
||||
extend-select = []
|
||||
# Overrides the inherited list: rules this gate enforces itself must NOT be external here,
|
||||
# so this config's RUF100 flags their stale `# noqa` directives. What remains external is
|
||||
|
|
@ -44,6 +44,7 @@ convention = "google"
|
|||
|
||||
[lint.pylint]
|
||||
max-args = 5
|
||||
max-bool-expr = 3
|
||||
|
||||
[lint.flake8-tidy-imports.banned-api]
|
||||
"typing.Any".msg = "Use a concrete type. Frozen slots=True dataclass (preferred) / NamedTuple / ReadOnly TypedDict for payloads."
|
||||
|
|
|
|||
4
.github/workflows/test-merge-smoke.yml
vendored
4
.github/workflows/test-merge-smoke.yml
vendored
|
|
@ -178,6 +178,7 @@ jobs:
|
|||
|
||||
windows-sdk:
|
||||
name: windows-sdk
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: windows-2025
|
||||
timeout-minutes: 30
|
||||
env:
|
||||
|
|
@ -213,6 +214,7 @@ jobs:
|
|||
|
||||
proxy-build-from-pip:
|
||||
name: proxy-build-from-pip
|
||||
if: github.event_name != 'pull_request'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 10
|
||||
services:
|
||||
|
|
@ -352,7 +354,7 @@ jobs:
|
|||
|
||||
smoke-passed:
|
||||
name: smoke passed
|
||||
needs: [dashboard-build, core-checks, all-extras, windows-sdk, proxy-build-from-pip]
|
||||
needs: [dashboard-build, core-checks, all-extras]
|
||||
if: always()
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 2
|
||||
|
|
|
|||
2
.github/workflows/test-rust-db.yml
vendored
2
.github/workflows/test-rust-db.yml
vendored
|
|
@ -91,6 +91,8 @@ jobs:
|
|||
|
||||
- name: Install Python dependencies
|
||||
working-directory: .
|
||||
env:
|
||||
SQLX_OFFLINE: "true"
|
||||
run: uv sync --frozen --extra extra-proxy
|
||||
|
||||
- name: Deploy Prisma migrations
|
||||
|
|
|
|||
14
.github/workflows/test-unit.yml
vendored
14
.github/workflows/test-unit.yml
vendored
|
|
@ -126,7 +126,7 @@ jobs:
|
|||
name: ${{ matrix.shard }}
|
||||
needs: [rust-bridge, assert-shard-coverage]
|
||||
if: ${{ !cancelled() }}
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ${{ matrix.runner || 'ubuntu-latest' }}
|
||||
timeout-minutes: ${{ matrix.job-timeout-minutes }}
|
||||
permissions:
|
||||
contents: read
|
||||
|
|
@ -220,6 +220,7 @@ jobs:
|
|||
dist: loadscope
|
||||
|
||||
- shard: llms-providers
|
||||
runner: ubuntu-latest-16-cores
|
||||
test-path: |-
|
||||
tests/unit/llms/test_api_base_trailing_slash_routes.py
|
||||
tests/unit/llms/test_cache_control_and_reasoning.py
|
||||
|
|
@ -380,7 +381,7 @@ jobs:
|
|||
tests/unit/llms/zai
|
||||
--ignore=tests/unit/llms/base_llm/batches/base_batches_config_test.py
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
workers: 8
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
|
|
@ -398,10 +399,11 @@ jobs:
|
|||
dist: loadscope
|
||||
|
||||
- shard: root
|
||||
runner: ubuntu-latest-16-cores
|
||||
test-path: >-
|
||||
tests/unit/test_*.py
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
workers: 8
|
||||
reruns: 2
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
|
|
@ -522,6 +524,7 @@ jobs:
|
|||
dist: loadscope
|
||||
|
||||
- shard: proxy-endpoints
|
||||
runner: ubuntu-latest-16-cores
|
||||
test-path: |-
|
||||
tests/unit/proxy/management_endpoints
|
||||
tests/unit/proxy/management_helpers
|
||||
|
|
@ -539,13 +542,14 @@ jobs:
|
|||
--ignore=tests/unit/proxy/management_endpoints/test_roi_calculator_endpoints.py
|
||||
--ignore=tests/unit/proxy/management_helpers/test_audit_logs_proxy.py
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
workers: 8
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
dist: loadscope
|
||||
|
||||
- shard: proxy-feature-endpoints
|
||||
runner: ubuntu-latest-16-cores
|
||||
test-path: |-
|
||||
tests/unit/proxy/guardrails
|
||||
--ignore=tests/unit/proxy/google_endpoints/test_gemini_agents_endpoints.py
|
||||
|
|
@ -578,7 +582,7 @@ jobs:
|
|||
tests/unit/proxy/config_resolvers
|
||||
tests/unit/proxy/utils
|
||||
python-version: "3.12"
|
||||
workers: 4
|
||||
workers: 8
|
||||
reruns: 0
|
||||
timeout-minutes: 20
|
||||
job-timeout-minutes: 60
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
|
||||
BatchFileProvider: TypeAlias = Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"]
|
||||
|
||||
|
|
@ -242,7 +242,7 @@ async def _deliver(
|
|||
|
||||
|
||||
def _resolve_logger(callback: "litellm.CALLBACK_TYPES | S3Logger") -> "CustomLogger | S3Logger | None":
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
|
||||
match callback:
|
||||
case CustomLogger() | S3Logger():
|
||||
|
|
|
|||
|
|
@ -169,7 +169,7 @@ class DualCache(BaseCache):
|
|||
key,
|
||||
parent_otel_span: Span | None = None,
|
||||
local_only: bool = False,
|
||||
**kwargs,
|
||||
**kwargs: object,
|
||||
):
|
||||
# Try to fetch from in-memory cache first
|
||||
try:
|
||||
|
|
@ -190,7 +190,7 @@ class DualCache(BaseCache):
|
|||
|
||||
result = redis_result
|
||||
|
||||
print_verbose(f"get cache: cache result: {result}")
|
||||
print_verbose("get cache: cache result: %s", result)
|
||||
return result
|
||||
except Exception as e:
|
||||
log_redis_failure(
|
||||
|
|
@ -245,16 +245,16 @@ class DualCache(BaseCache):
|
|||
key,
|
||||
parent_otel_span: Span | None = None,
|
||||
local_only: bool = False,
|
||||
**kwargs,
|
||||
**kwargs: object,
|
||||
):
|
||||
# Try to fetch from in-memory cache first
|
||||
try:
|
||||
print_verbose(f"async get cache: cache key: {key}; local_only: {local_only}")
|
||||
print_verbose("async get cache: cache key: %s; local_only: %s", key, local_only)
|
||||
result = None
|
||||
if self.in_memory_cache is not None:
|
||||
in_memory_result: Final = await self.in_memory_cache.async_get_cache(key, **kwargs)
|
||||
|
||||
print_verbose(f"in_memory_result: {in_memory_result}")
|
||||
print_verbose("in_memory_result: %s", in_memory_result)
|
||||
if in_memory_result is not None:
|
||||
result = in_memory_result
|
||||
|
||||
|
|
@ -268,7 +268,7 @@ class DualCache(BaseCache):
|
|||
|
||||
result = redis_result
|
||||
|
||||
print_verbose(f"get cache: cache result: {result}")
|
||||
print_verbose("get cache: cache result: %s", result)
|
||||
return result
|
||||
except Exception as e:
|
||||
log_redis_failure(
|
||||
|
|
@ -451,8 +451,8 @@ class DualCache(BaseCache):
|
|||
with_traceback=True,
|
||||
)
|
||||
|
||||
async def async_set_cache(self, key, value, local_only: bool = False, **kwargs):
|
||||
print_verbose(f"async set cache: cache key: {key}; local_only: {local_only}; value: {value}")
|
||||
async def async_set_cache(self, key, value, local_only: bool = False, **kwargs: object):
|
||||
print_verbose("async set cache: cache key: %s; local_only: %s; value: %s", key, local_only, value)
|
||||
try:
|
||||
if self.in_memory_cache is not None:
|
||||
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
|
||||
|
|
@ -473,7 +473,7 @@ class DualCache(BaseCache):
|
|||
"""
|
||||
Batch write values to the cache
|
||||
"""
|
||||
print_verbose(f"async batch set cache: cache keys: {cache_list}; local_only: {local_only}")
|
||||
print_verbose("async batch set cache: cache keys: %s; local_only: %s", cache_list, local_only)
|
||||
try:
|
||||
if self.in_memory_cache is not None:
|
||||
if "ttl" not in kwargs and self.default_in_memory_ttl is not None:
|
||||
|
|
|
|||
|
|
@ -2362,6 +2362,7 @@ MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY: Final = "model_batch_enqueued_tok
|
|||
MAX_BATCH_FILE_RECORDS_KEY: Final = "max_batch_file_records"
|
||||
MAX_BATCH_FILE_UPLOADS_PER_DAY_KEY: Final = "max_batch_file_uploads_per_day"
|
||||
MAX_FILE_DOWNLOADS_PER_MINUTE_KEY: Final = "max_file_downloads_per_minute"
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: Final = "model_limit_max_borrow_multiple"
|
||||
ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS: Final = (
|
||||
BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
|
||||
MAX_BATCH_FILE_RECORDS_KEY,
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-enterprise"
|
||||
version = "0.1.85"
|
||||
version = "0.1.86"
|
||||
description = "Package for LiteLLM Enterprise features"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -26,7 +26,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.1.85"
|
||||
version = "0.1.86"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../../pyproject.toml:litellm-enterprise==",
|
||||
|
|
|
|||
|
|
@ -35,7 +35,6 @@ FileDeleteProvider = Literal[
|
|||
]
|
||||
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"]
|
||||
import litellm
|
||||
from litellm import get_secret_str
|
||||
from litellm.files.streaming import FileContentStreamingResponse
|
||||
from litellm.files.types import FileContentProvider, FileContentStreamingResult, FileRetrieveProvider
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
|
|
@ -47,7 +46,6 @@ from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
|||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.llms.openai.common_utils import get_openai_credentials
|
||||
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
|
||||
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
|
||||
from litellm.llms.xai.batches.handler import XAIBatchesHandler
|
||||
from litellm.llms.xai.batches.transformation import is_xai_batch_results_id
|
||||
from litellm.types.llms.openai import (
|
||||
|
|
@ -102,7 +100,6 @@ def _file_content_logging_obj(kwargs: dict[str, object], _is_async: bool) -> Lit
|
|||
|
||||
openai_files_instance: Final = OpenAIFilesAPI()
|
||||
azure_files_instance: Final = AzureOpenAIFilesAPI()
|
||||
vertex_ai_files_instance: Final = VertexAIFilesHandler()
|
||||
bedrock_files_instance: Final = BedrockFilesHandler()
|
||||
xai_batch_results_instance: Final = XAIBatchesHandler()
|
||||
#################################################
|
||||
|
|
@ -986,27 +983,6 @@ def file_content(
|
|||
client=client,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif custom_llm_provider == "vertex_ai":
|
||||
api_base: Final = optional_params.api_base or ""
|
||||
vertex_ai_project: Final = (
|
||||
optional_params.vertex_project or litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT")
|
||||
)
|
||||
vertex_ai_location: Final = (
|
||||
optional_params.vertex_location or litellm.vertex_location or get_secret_str("VERTEXAI_LOCATION")
|
||||
)
|
||||
vertex_credentials: Final = optional_params.vertex_credentials or get_secret_str("VERTEXAI_CREDENTIALS")
|
||||
|
||||
response = vertex_ai_files_instance.file_content(
|
||||
_is_async=_is_async,
|
||||
file_content_request=_file_content_request,
|
||||
api_base=api_base,
|
||||
vertex_credentials=vertex_credentials,
|
||||
vertex_project=vertex_ai_project,
|
||||
vertex_location=vertex_ai_location,
|
||||
timeout=timeout,
|
||||
max_retries=optional_params.max_retries,
|
||||
litellm_params=litellm_params_dict,
|
||||
)
|
||||
elif custom_llm_provider == "bedrock":
|
||||
response = bedrock_files_instance.file_content(
|
||||
_is_async=_is_async,
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
from collections.abc import Mapping
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
from functools import cache, lru_cache
|
||||
from typing import Annotated, Final
|
||||
|
||||
from pydantic import AliasChoices, ConfigDict, Field, TypeAdapter, ValidationError, field_validator, model_validator
|
||||
|
|
@ -218,7 +218,8 @@ class OpenTelemetryV2Config(BaseSettings):
|
|||
description=(
|
||||
"``llm_only`` keeps just the model-call spans on the operator's own Langfuse "
|
||||
"exporter (the spec whose owner is ``langfuse_otel``). Other exporters and "
|
||||
"key/team destinations are not affected."
|
||||
"key/team destinations are not affected. Destinations default to "
|
||||
"``langfuse_destination_span_scope`` instead."
|
||||
),
|
||||
)
|
||||
excluded_services: Annotated[frozenset[str], NoDecode] = Field(
|
||||
|
|
@ -317,9 +318,7 @@ class OpenTelemetryV2Config(BaseSettings):
|
|||
@field_validator("langfuse_span_scope", mode="before")
|
||||
@classmethod
|
||||
def _normalize_langfuse_span_scope(cls, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower()
|
||||
return value
|
||||
return _normalized_span_scope(value)
|
||||
|
||||
@field_validator(
|
||||
"baggage_promoted_keys",
|
||||
|
|
@ -417,6 +416,44 @@ def capture_message_content_from_env() -> str:
|
|||
return _CaptureMessageContentSettings().capture_message_content
|
||||
|
||||
|
||||
def _normalized_span_scope(value: object) -> object:
|
||||
return value.strip().lower() if isinstance(value, str) else value
|
||||
|
||||
|
||||
class _LangfuseDestinationSpanScopeSettings(BaseSettings):
|
||||
model_config = SettingsConfigDict(extra="ignore", frozen=True)
|
||||
|
||||
langfuse_destination_span_scope: OtelSpanScope = Field(
|
||||
default="full",
|
||||
validation_alias=AliasChoices("LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE"),
|
||||
)
|
||||
|
||||
@field_validator("langfuse_destination_span_scope", mode="before")
|
||||
@classmethod
|
||||
def _normalize_scope(cls, value: object) -> object:
|
||||
return _normalized_span_scope(value)
|
||||
|
||||
|
||||
def langfuse_destination_span_scope(configured: object = None) -> OtelSpanScope:
|
||||
return _cached_langfuse_destination_span_scope(None if configured is None else str(configured))
|
||||
|
||||
|
||||
@cache
|
||||
def _cached_langfuse_destination_span_scope(configured: str | None) -> OtelSpanScope:
|
||||
try:
|
||||
return _LangfuseDestinationSpanScopeSettings.model_validate(
|
||||
{"LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE": configured} if configured is not None else {}
|
||||
).langfuse_destination_span_scope
|
||||
except ValidationError as error:
|
||||
verbose_logger.warning(
|
||||
"Invalid LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE or "
|
||||
"callback_settings.otel.langfuse_destination_span_scope. "
|
||||
"Allowed values are 'full' and 'llm_only'. Falling back to 'full'. %s",
|
||||
error,
|
||||
)
|
||||
return "full"
|
||||
|
||||
|
||||
_EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...])
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -161,10 +161,12 @@ def _resource_defaults(callback_name: str) -> Mapping[str, str]:
|
|||
return backend() if backend is not None else _NO_ATTRS
|
||||
|
||||
|
||||
def _span_scope(callback_name: str, params: StandardCallbackDynamicParams) -> OtelSpanScope:
|
||||
def _span_scope(
|
||||
callback_name: str, params: StandardCallbackDynamicParams, langfuse_destination_span_scope: OtelSpanScope
|
||||
) -> OtelSpanScope:
|
||||
if callback_name != "langfuse_otel":
|
||||
return "full"
|
||||
return params.get("langfuse_span_scope") or "full"
|
||||
return params.get("langfuse_span_scope") or langfuse_destination_span_scope
|
||||
|
||||
|
||||
def destination_capable_backends() -> frozenset[str]:
|
||||
|
|
@ -199,6 +201,7 @@ def destination_for(
|
|||
capture_message_content: str | None = None,
|
||||
excluded_db_systems: frozenset[str] | None = None,
|
||||
metadata_keys: tuple[str, ...] | None = None,
|
||||
langfuse_destination_span_scope: OtelSpanScope = "full",
|
||||
) -> OtelDestination | None:
|
||||
"""The destination ``params`` names for ``callback_name``, or ``None``.
|
||||
|
||||
|
|
@ -228,7 +231,7 @@ def destination_for(
|
|||
resource_defaults=_resource_defaults(callback_name),
|
||||
callback_name=callback_name,
|
||||
protocol=protocol,
|
||||
span_scope=_span_scope(callback_name, params),
|
||||
span_scope=_span_scope(callback_name, params, langfuse_destination_span_scope),
|
||||
capture_message_content=capture_message_content,
|
||||
excluded_db_systems=excluded_db_systems,
|
||||
metadata_keys=metadata_keys or (),
|
||||
|
|
|
|||
|
|
@ -181,6 +181,18 @@ class _CustomerBudgetRow(Protocol):
|
|||
def litellm_budget_table(self) -> _JoinedBudgetRow | None: ...
|
||||
|
||||
|
||||
def _dynamic_limiter_holder_labels(
|
||||
kind: Literal["team", "key", "customer_org"], holder_id: str, holder_alias: str | None
|
||||
) -> Mapping[str, str | None]:
|
||||
match kind:
|
||||
case "team":
|
||||
return {"team": holder_id, "team_alias": holder_alias}
|
||||
case "key":
|
||||
return {"hashed_api_key": holder_id, "api_key_alias": holder_alias}
|
||||
case "customer_org":
|
||||
return {"customer_org_id": holder_id, "customer_org_alias": holder_alias}
|
||||
|
||||
|
||||
def _customer_budget_metrics_enabled() -> bool:
|
||||
return litellm.enable_end_user_cost_tracking_prometheus_only is True and not litellm.disable_end_user_cost_tracking
|
||||
|
||||
|
|
@ -4201,14 +4213,17 @@ class PrometheusLogger(CustomLogger):
|
|||
def set_dynamic_limiter_share(
|
||||
self,
|
||||
model_group: str,
|
||||
team: str,
|
||||
team_alias: str | None,
|
||||
holder_kind: Literal["team", "key", "customer_org"],
|
||||
holder_id: str,
|
||||
holder_alias: str | None,
|
||||
priority: str,
|
||||
share_tokens: int,
|
||||
borrowed_tokens: int,
|
||||
) -> None:
|
||||
enum_values: Final = UserAPIKeyLabelValues(
|
||||
model_group=model_group, team=team, team_alias=team_alias, priority=priority
|
||||
model_group=model_group,
|
||||
priority=priority,
|
||||
**_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias),
|
||||
)
|
||||
self._set_labeled_gauge(
|
||||
self.litellm_dynamic_limiter_share_tokens, "litellm_dynamic_limiter_share_tokens", enum_values, share_tokens
|
||||
|
|
@ -4239,8 +4254,9 @@ class PrometheusLogger(CustomLogger):
|
|||
def increment_dynamic_limiter_throttled(
|
||||
self,
|
||||
model_group: str,
|
||||
team: str,
|
||||
team_alias: str | None,
|
||||
holder_kind: Literal["team", "key", "customer_org"],
|
||||
holder_id: str,
|
||||
holder_alias: str | None,
|
||||
priority: str,
|
||||
rate_limit_type: str,
|
||||
reason: str,
|
||||
|
|
@ -4250,11 +4266,10 @@ class PrometheusLogger(CustomLogger):
|
|||
"litellm_dynamic_limiter_throttled_total",
|
||||
UserAPIKeyLabelValues(
|
||||
model_group=model_group,
|
||||
team=team,
|
||||
team_alias=team_alias,
|
||||
priority=priority,
|
||||
rate_limit_type=rate_limit_type,
|
||||
reason=reason,
|
||||
**_dynamic_limiter_holder_labels(holder_kind, holder_id, holder_alias),
|
||||
),
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,3 +1,481 @@
|
|||
from litellm.integrations.s3_v3 import S3BatchUploadError, S3Logger
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Collection, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
__all__ = ["S3BatchUploadError", "S3Logger"]
|
||||
from typing_extensions import Unpack
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
DEFAULT_S3_BATCH_SIZE,
|
||||
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.call import CallContext
|
||||
from litellm.integrations.custom_logger_v2.derivations import (
|
||||
cold_storage_object_key,
|
||||
log_id,
|
||||
logged_key_identity,
|
||||
mapping,
|
||||
payload_copy,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
|
||||
from litellm.integrations.custom_logger_v2.rust import RustLogger
|
||||
from litellm.integrations.custom_logger_v2.selection import Selection
|
||||
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
|
||||
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
|
||||
from litellm.secret_managers.main import get_secret, get_secret_str
|
||||
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
|
||||
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
CALLBACK_NAME: Final = "s3_v2"
|
||||
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
|
||||
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
|
||||
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
|
||||
|
||||
|
||||
class S3BatchUploadError(Exception):
|
||||
def __init__(self, failed: int, total: int) -> None:
|
||||
self.failed: Final = failed
|
||||
self.total: Final = total
|
||||
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
|
||||
|
||||
|
||||
def _raise_batch_upload_error(failed: int, total: int) -> None:
|
||||
raise S3BatchUploadError(failed=failed, total=total)
|
||||
|
||||
|
||||
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
|
||||
if failure is None:
|
||||
return
|
||||
queued, failed, total = failure
|
||||
try:
|
||||
_raise_batch_upload_error(failed, total)
|
||||
except S3BatchUploadError:
|
||||
verbose_logger.exception(
|
||||
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
|
||||
)
|
||||
|
||||
|
||||
def _cold_storage_is_s3_v2() -> bool:
|
||||
return litellm.cold_storage_custom_logger == "s3_v2"
|
||||
|
||||
|
||||
def _region_from_environment() -> str | None:
|
||||
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
|
||||
region: Final = get_secret_str("AWS_REGION", None)
|
||||
return region if region is not None else region_name
|
||||
|
||||
|
||||
class _Host:
|
||||
def __init__(self) -> None:
|
||||
self.failures: Final = CustomLogger()
|
||||
|
||||
def callback_failed(self) -> None:
|
||||
self.failures.handle_callback_failure(callback_name="S3Logger")
|
||||
|
||||
def web_identity_token(self, reference: str) -> str:
|
||||
if reference.startswith("os.environ/") or reference in os.environ:
|
||||
raise ValueError("Invalid web identity token reference.")
|
||||
match _read_secret(get_secret, reference):
|
||||
case None:
|
||||
raise ValueError("OIDC token could not be retrieved from secret manager.")
|
||||
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
|
||||
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
|
||||
)
|
||||
case str() as token:
|
||||
return token
|
||||
case other:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
|
||||
f"value: {other}, type: {type(other)}, valid types: {str}"
|
||||
)
|
||||
|
||||
|
||||
def _read_secret(read: Callable[[str], object], reference: str) -> object:
|
||||
return read(reference)
|
||||
|
||||
|
||||
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
|
||||
source: Final[Mapping[str, object] | Mapping[object, object]] = (
|
||||
params_source if params_source is not None else (configured or {})
|
||||
)
|
||||
return {
|
||||
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
|
||||
for key, value in source.items()
|
||||
if isinstance(key, str)
|
||||
}
|
||||
|
||||
|
||||
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
|
||||
return {
|
||||
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
|
||||
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
|
||||
}
|
||||
|
||||
|
||||
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
|
||||
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
|
||||
|
||||
|
||||
_CONFIGURED_FIRST: Final = frozenset(
|
||||
{
|
||||
"s3_bucket_name",
|
||||
"s3_path",
|
||||
"s3_region_name",
|
||||
"s3_api_version",
|
||||
"s3_endpoint_url",
|
||||
"s3_aws_access_key_id",
|
||||
"s3_aws_secret_access_key",
|
||||
"s3_aws_session_token",
|
||||
"s3_aws_session_name",
|
||||
"s3_aws_profile_name",
|
||||
"s3_aws_role_name",
|
||||
"s3_aws_web_identity_token",
|
||||
"s3_aws_sts_endpoint",
|
||||
"s3_config",
|
||||
"s3_server_side_encryption",
|
||||
"s3_sse_kms_key_id",
|
||||
}
|
||||
)
|
||||
_CONSTRUCTOR_ENABLES: Final = frozenset(
|
||||
{
|
||||
"s3_use_team_prefix",
|
||||
"s3_strip_base64_files",
|
||||
"s3_use_key_prefix",
|
||||
"s3_use_virtual_hosted_style",
|
||||
"s3_adaptive_concurrency",
|
||||
"s3_batch_file_upload",
|
||||
}
|
||||
)
|
||||
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
|
||||
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
|
||||
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
|
||||
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
|
||||
"s3_use_ssl": True,
|
||||
"s3_drop_on_terminal_error": True,
|
||||
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
|
||||
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
|
||||
}
|
||||
|
||||
|
||||
def _blank(value: object) -> bool:
|
||||
return value is None or value == ""
|
||||
|
||||
|
||||
def _constructor_wins(key: str, configured: object, value: object) -> bool:
|
||||
return (
|
||||
(key in _CONFIGURED_FIRST and not configured)
|
||||
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
|
||||
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
|
||||
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
|
||||
)
|
||||
|
||||
|
||||
_KEEP_CONFIGURED: Final = object()
|
||||
|
||||
|
||||
def _constructor_value(key: str, configured: object, value: object) -> object:
|
||||
if _constructor_wins(key, configured, value):
|
||||
return value
|
||||
if key in _CONSTRUCTOR_ENABLES and value:
|
||||
return True
|
||||
if key == "s3_max_queue_size" and value is not None:
|
||||
return resolve_s3_max_queue_size(
|
||||
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
|
||||
)
|
||||
if key == "s3_max_retry_age_seconds":
|
||||
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
|
||||
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
|
||||
return 0 if resolved is None else resolved
|
||||
return _KEEP_CONFIGURED
|
||||
|
||||
|
||||
def _constructed_params(
|
||||
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
|
||||
) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
params: Final = _resolved_params(params_source)
|
||||
overrides: Final = {
|
||||
key: replaced
|
||||
for key, value in settings.items()
|
||||
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
|
||||
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
|
||||
}
|
||||
return {**params, **overrides}
|
||||
|
||||
|
||||
class _Base64Stripper(CustomLogger):
|
||||
def strip(self, payload: StandardLoggingPayload) -> None:
|
||||
_ = self._strip_base64_from_messages_sync(payload)
|
||||
|
||||
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
|
||||
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
|
||||
|
||||
|
||||
def _shallow_copy(message: object) -> object:
|
||||
entries: Final = mapping(message)
|
||||
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
|
||||
|
||||
|
||||
_STRIPPER: Final = _Base64Stripper()
|
||||
|
||||
|
||||
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
if litellm.global_disable_no_log_param:
|
||||
return True
|
||||
litellm_params: Final = kwargs.get("litellm_params", {})
|
||||
lookup: Final = getattr(litellm_params, "get", None)
|
||||
if callable(lookup) and lookup("no-log", False) is True:
|
||||
return False
|
||||
return not litellm_logging.callback_disabled_dynamically(
|
||||
callback=CALLBACK_NAME,
|
||||
litellm_params=litellm_params,
|
||||
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
|
||||
)
|
||||
|
||||
|
||||
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
|
||||
raw: Final = payload.get("messages")
|
||||
return [*raw] if isinstance(raw, list) else []
|
||||
|
||||
|
||||
class S3StripBase64Files(CustomLoggerV2):
|
||||
selection = Selection(name=STRIP_CALLBACK_NAME)
|
||||
|
||||
def __init__(self, s3_logger_key: str) -> None:
|
||||
self.s3_logger_key: Final = s3_logger_key
|
||||
|
||||
@on(
|
||||
Hook.log_prepare,
|
||||
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
|
||||
writes={Field.standard_logging_payload},
|
||||
)
|
||||
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
|
||||
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
|
||||
return None
|
||||
messages: Final = _payload_messages(payload)
|
||||
if not messages:
|
||||
return None
|
||||
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
|
||||
|
||||
|
||||
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
batch_upload_failed(await sink.flush())
|
||||
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
|
||||
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
|
||||
|
||||
|
||||
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
|
||||
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
details: Final = mapping(kwargs)
|
||||
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
|
||||
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
|
||||
return None
|
||||
if logged.get("status") == "failure":
|
||||
return None
|
||||
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
|
||||
|
||||
|
||||
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
|
||||
source: Final = mapping(kwargs) if details is None else details
|
||||
return None if source is None else source.get("standard_logging_object")
|
||||
|
||||
|
||||
def _legacy_kwargs(view: object) -> object:
|
||||
return getattr(view, Field.deprecated_kwargs.value, None)
|
||||
|
||||
|
||||
_read_attribute: Final[Callable[[object, str], object]] = getattr
|
||||
|
||||
|
||||
class _ExcludedView:
|
||||
def __init__(self, view: object, details: Mapping[str, object]) -> None:
|
||||
args: Final = HookArgs(request=details)
|
||||
self.view: Final = view
|
||||
self.fields: Final[Mapping[str, object]] = {
|
||||
Field.deprecated_kwargs.value: details,
|
||||
Field.standard_logging_payload_ref.value: payload_copy(args),
|
||||
Field.log_id.value: log_id(args),
|
||||
Field.cold_storage_object_key.value: cold_storage_object_key(args),
|
||||
Field.logged_key_identity.value: logged_key_identity(args),
|
||||
}
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
if name not in self.fields:
|
||||
return _read_attribute(self.view, name)
|
||||
value: Final = self.fields[name]
|
||||
if value is None:
|
||||
raise AttributeError(name)
|
||||
return value
|
||||
|
||||
|
||||
_ATTEMPT_ENDED_READS: Final = frozenset(
|
||||
{
|
||||
Field.deprecated_kwargs,
|
||||
Field.attempt_timing,
|
||||
Field.log_id,
|
||||
Field.cold_storage_object_key,
|
||||
Field.standard_logging_payload_ref,
|
||||
Field.logged_key_identity,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class S3Logger(RustLogger):
|
||||
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
s3_callback_params_override: Mapping[str, object] | None = None,
|
||||
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
|
||||
) -> None:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
host: Final = _Host()
|
||||
params: Final = _constructed_params(s3_callback_params_override, settings)
|
||||
sink: Final = NativeS3Log(
|
||||
params,
|
||||
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
|
||||
host,
|
||||
_cold_storage_is_s3_v2(),
|
||||
None if params.get("s3_region_name") else _region_from_environment(),
|
||||
)
|
||||
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
|
||||
self.sink: Final = sink
|
||||
self.s3_host: Final = host
|
||||
self.periodic_flush_task: Final = asyncio.create_task(
|
||||
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
|
||||
)
|
||||
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
|
||||
self.v2_companion: Final = (
|
||||
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
|
||||
if sink.strip_base64_files
|
||||
else None
|
||||
)
|
||||
|
||||
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
|
||||
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
|
||||
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
|
||||
|
||||
async def run_rust(
|
||||
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
|
||||
) -> Patch | Abort | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
|
||||
return None
|
||||
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
|
||||
if payload is not None and not isinstance(payload, Mapping):
|
||||
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
|
||||
try:
|
||||
raw: Final = _legacy_kwargs(view)
|
||||
details: Final = _excluding_fields(raw)
|
||||
logged: Final = _logged_payload(raw, details)
|
||||
if self.sink.strip_base64_files and isinstance(logged, Mapping):
|
||||
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
|
||||
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
|
||||
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
|
||||
return self._failed(e)
|
||||
|
||||
def _failed(self, error: Exception) -> None:
|
||||
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
|
||||
self.s3_host.callback_failed()
|
||||
|
||||
@property
|
||||
def s3_path(self) -> str | None:
|
||||
return self.sink.s3_path
|
||||
|
||||
@property
|
||||
def s3_bucket_name(self) -> str | None:
|
||||
return self.sink.s3_bucket_name
|
||||
|
||||
def resolve_partition_granularity(self) -> S3PartitionGranularity:
|
||||
return "hour" if self.sink.partition_granularity() == "hour" else "day"
|
||||
|
||||
async def flush_queue(self) -> None:
|
||||
batch_upload_failed(await self.sink.flush())
|
||||
|
||||
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
|
||||
try:
|
||||
audit_log_id: Final = audit_log.get("id", "unknown")
|
||||
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
|
||||
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
|
||||
verbose_logger.exception("S3 audit log error: %s", e)
|
||||
|
||||
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
|
||||
object_name: Final = batch_line_items_object_name(payload)
|
||||
s3_path: Final = self.s3_path
|
||||
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
|
||||
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
|
||||
delivered: Final = await self.sink.upload_line_items(
|
||||
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
|
||||
)
|
||||
if not delivered:
|
||||
verbose_logger.error(
|
||||
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
|
||||
len(payload["lines"]),
|
||||
payload["part"],
|
||||
payload["batch_id"],
|
||||
s3_object_key,
|
||||
)
|
||||
|
||||
async def _download_object_from_s3(self, s3_object_key: str) -> object:
|
||||
try:
|
||||
downloaded: Final = await self.sink.download(s3_object_key)
|
||||
return None if downloaded is None else json.loads(downloaded)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error downloading from S3: %s", e)
|
||||
return None
|
||||
|
||||
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
|
||||
try:
|
||||
return await self._download_object_from_s3(object_key)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"S3BatchUploadError",
|
||||
"S3Logger",
|
||||
"S3StripBase64Files",
|
||||
"legacy_should_log",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,481 +0,0 @@
|
|||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Callable, Collection, Mapping
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from typing_extensions import Unpack
|
||||
|
||||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import (
|
||||
COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
DEFAULT_S3_BATCH_SIZE,
|
||||
DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
)
|
||||
from litellm.integrations.batch_line_item_objects import batch_line_items_ndjson, batch_line_items_object_name
|
||||
from litellm.integrations.custom_batch_logger import CustomBatchLogger
|
||||
from litellm.integrations.custom_logger import CustomLogger
|
||||
from litellm.integrations.custom_logger_v2.call import CallContext
|
||||
from litellm.integrations.custom_logger_v2.derivations import (
|
||||
cold_storage_object_key,
|
||||
log_id,
|
||||
logged_key_identity,
|
||||
mapping,
|
||||
payload_copy,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.enums import Field, Hook
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
|
||||
from litellm.integrations.custom_logger_v2.logger import CustomLoggerV2, on
|
||||
from litellm.integrations.custom_logger_v2.outcomes import Abort, Patch
|
||||
from litellm.integrations.custom_logger_v2.rust import RustLogger
|
||||
from litellm.integrations.custom_logger_v2.selection import Selection
|
||||
from litellm.integrations.s3 import resolve_s3_max_queue_size, resolve_s3_max_retry_age_seconds
|
||||
from litellm.llms.bedrock.web_identity_session_policy import S3_LOGGING_SESSION_POLICY, session_policy_document
|
||||
from litellm.secret_managers.main import get_secret, get_secret_str
|
||||
from litellm.types.integrations.s3_v2 import S3LoggerSettings, S3PartitionGranularity
|
||||
from litellm.types.utils import StandardAuditLogPayload, StandardBatchLineItemsPayload, StandardLoggingPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
CALLBACK_NAME: Final = "s3_v2"
|
||||
STRIP_CALLBACK_NAME: Final = "s3_v2_strip_base64_files"
|
||||
_WEB_IDENTITY_TOKEN_MIN_LENGTH: Final = 4
|
||||
_DEFAULT_MAX_RETRY_AGE_SECONDS: Final = 3600
|
||||
|
||||
|
||||
class S3BatchUploadError(Exception):
|
||||
def __init__(self, failed: int, total: int) -> None:
|
||||
self.failed: Final = failed
|
||||
self.total: Final = total
|
||||
super().__init__(f"{failed} of {total} S3 uploads failed; transient failures kept in queue for the next flush")
|
||||
|
||||
|
||||
def _raise_batch_upload_error(failed: int, total: int) -> None:
|
||||
raise S3BatchUploadError(failed=failed, total=total)
|
||||
|
||||
|
||||
def batch_upload_failed(failure: tuple[int, int, int] | None) -> None:
|
||||
if failure is None:
|
||||
return
|
||||
queued, failed, total = failure
|
||||
try:
|
||||
_raise_batch_upload_error(failed, total)
|
||||
except S3BatchUploadError:
|
||||
verbose_logger.exception(
|
||||
"CustomLogger: async_send_batch raised; preserving %s events in queue for retry", queued
|
||||
)
|
||||
|
||||
|
||||
def _cold_storage_is_s3_v2() -> bool:
|
||||
return litellm.cold_storage_custom_logger == "s3_v2"
|
||||
|
||||
|
||||
def _region_from_environment() -> str | None:
|
||||
region_name: Final = get_secret_str("AWS_REGION_NAME", None)
|
||||
region: Final = get_secret_str("AWS_REGION", None)
|
||||
return region if region is not None else region_name
|
||||
|
||||
|
||||
class _Host:
|
||||
def __init__(self) -> None:
|
||||
self.failures: Final = CustomLogger()
|
||||
|
||||
def callback_failed(self) -> None:
|
||||
self.failures.handle_callback_failure(callback_name="S3Logger")
|
||||
|
||||
def web_identity_token(self, reference: str) -> str:
|
||||
if reference.startswith("os.environ/") or reference in os.environ:
|
||||
raise ValueError("Invalid web identity token reference.")
|
||||
match _read_secret(get_secret, reference):
|
||||
case None:
|
||||
raise ValueError("OIDC token could not be retrieved from secret manager.")
|
||||
case str() as token if len(token) < _WEB_IDENTITY_TOKEN_MIN_LENGTH:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid length for parameter WebIdentityToken, "
|
||||
f"value: {len(token)}, valid min length: {_WEB_IDENTITY_TOKEN_MIN_LENGTH}"
|
||||
)
|
||||
case str() as token:
|
||||
return token
|
||||
case other:
|
||||
raise ValueError(
|
||||
"Parameter validation failed:\nInvalid type for parameter WebIdentityToken, "
|
||||
f"value: {other}, type: {type(other)}, valid types: {str}"
|
||||
)
|
||||
|
||||
|
||||
def _read_secret(read: Callable[[str], object], reference: str) -> object:
|
||||
return read(reference)
|
||||
|
||||
|
||||
def _resolved_params(params_source: Mapping[str, object] | None) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
configured: Final[Mapping[object, object] | None] = getattr(litellm, "s3_callback_params", None)
|
||||
source: Final[Mapping[str, object] | Mapping[object, object]] = (
|
||||
params_source if params_source is not None else (configured or {})
|
||||
)
|
||||
return {
|
||||
key: (litellm.get_secret(value) if isinstance(value, str) and value.startswith("os.environ/") else value)
|
||||
for key, value in source.items()
|
||||
if isinstance(key, str)
|
||||
}
|
||||
|
||||
|
||||
def _constants(batch_size: int | None = DEFAULT_S3_BATCH_SIZE) -> Mapping[str, object]:
|
||||
return {
|
||||
"batch_size": batch_size or litellm.DEFAULT_BATCH_SIZE,
|
||||
"max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"max_adaptive_concurrency": DEFAULT_S3_MAX_ADAPTIVE_CONCURRENCY,
|
||||
"upload_attempt_timeout_seconds": S3_UPLOAD_ATTEMPT_TIMEOUT_SECONDS,
|
||||
"download_read_timeout_seconds": COMPLETION_HTTP_FALLBACK_SECONDS,
|
||||
"download_connect_timeout_seconds": HTTP_HANDLER_CONNECT_TIMEOUT_SECONDS,
|
||||
"web_identity_session_policy": session_policy_document(S3_LOGGING_SESSION_POLICY),
|
||||
}
|
||||
|
||||
|
||||
def _flush_interval(flush_interval: float | None = DEFAULT_S3_FLUSH_INTERVAL_SECONDS) -> float:
|
||||
return flush_interval or litellm.DEFAULT_FLUSH_INTERVAL_SECONDS
|
||||
|
||||
|
||||
_CONFIGURED_FIRST: Final = frozenset(
|
||||
{
|
||||
"s3_bucket_name",
|
||||
"s3_path",
|
||||
"s3_region_name",
|
||||
"s3_api_version",
|
||||
"s3_endpoint_url",
|
||||
"s3_aws_access_key_id",
|
||||
"s3_aws_secret_access_key",
|
||||
"s3_aws_session_token",
|
||||
"s3_aws_session_name",
|
||||
"s3_aws_profile_name",
|
||||
"s3_aws_role_name",
|
||||
"s3_aws_web_identity_token",
|
||||
"s3_aws_sts_endpoint",
|
||||
"s3_config",
|
||||
"s3_server_side_encryption",
|
||||
"s3_sse_kms_key_id",
|
||||
}
|
||||
)
|
||||
_CONSTRUCTOR_ENABLES: Final = frozenset(
|
||||
{
|
||||
"s3_use_team_prefix",
|
||||
"s3_strip_base64_files",
|
||||
"s3_use_key_prefix",
|
||||
"s3_use_virtual_hosted_style",
|
||||
"s3_adaptive_concurrency",
|
||||
"s3_batch_file_upload",
|
||||
}
|
||||
)
|
||||
_CONFIGURED_UNLESS_NONE: Final = frozenset({"s3_use_ssl", "s3_verify", "s3_drop_on_terminal_error"})
|
||||
_CONSTRUCTOR_UNLESS_NONE: Final = frozenset({"s3_log_prompts_only", "s3_partition_granularity"})
|
||||
_CONFIGURED_UNLESS_BLANK: Final = frozenset({"s3_max_concurrent_uploads", "s3_max_adaptive_concurrency"})
|
||||
_CONSTRUCTOR_DEFAULTS: Final[Mapping[str, object]] = {
|
||||
"s3_use_ssl": True,
|
||||
"s3_drop_on_terminal_error": True,
|
||||
"s3_max_concurrent_uploads": DEFAULT_S3_MAX_CONCURRENT_UPLOADS,
|
||||
"s3_max_retry_age_seconds": _DEFAULT_MAX_RETRY_AGE_SECONDS,
|
||||
"s3_flush_interval": DEFAULT_S3_FLUSH_INTERVAL_SECONDS,
|
||||
"s3_batch_size": DEFAULT_S3_BATCH_SIZE,
|
||||
}
|
||||
|
||||
|
||||
def _blank(value: object) -> bool:
|
||||
return value is None or value == ""
|
||||
|
||||
|
||||
def _constructor_wins(key: str, configured: object, value: object) -> bool:
|
||||
return (
|
||||
(key in _CONFIGURED_FIRST and not configured)
|
||||
or (key in _CONFIGURED_UNLESS_NONE and configured is None)
|
||||
or (key in _CONFIGURED_UNLESS_BLANK and _blank(configured))
|
||||
or (key in _CONSTRUCTOR_UNLESS_NONE and value is not None)
|
||||
)
|
||||
|
||||
|
||||
_KEEP_CONFIGURED: Final = object()
|
||||
|
||||
|
||||
def _constructor_value(key: str, configured: object, value: object) -> object:
|
||||
if _constructor_wins(key, configured, value):
|
||||
return value
|
||||
if key in _CONSTRUCTOR_ENABLES and value:
|
||||
return True
|
||||
if key == "s3_max_queue_size" and value is not None:
|
||||
return resolve_s3_max_queue_size(
|
||||
configured, resolve_s3_max_queue_size(value, CustomBatchLogger.DEFAULT_MAX_QUEUE_SIZE)
|
||||
)
|
||||
if key == "s3_max_retry_age_seconds":
|
||||
fallback: Final = resolve_s3_max_retry_age_seconds(value, _DEFAULT_MAX_RETRY_AGE_SECONDS)
|
||||
resolved: Final = fallback if _blank(configured) else resolve_s3_max_retry_age_seconds(configured, fallback)
|
||||
return 0 if resolved is None else resolved
|
||||
return _KEEP_CONFIGURED
|
||||
|
||||
|
||||
def _constructed_params(
|
||||
params_source: Mapping[str, object] | None, settings: S3LoggerSettings
|
||||
) -> dict[str, object]: # mutable-ok: handed to Rust
|
||||
params: Final = _resolved_params(params_source)
|
||||
overrides: Final = {
|
||||
key: replaced
|
||||
for key, value in settings.items()
|
||||
if value != _CONSTRUCTOR_DEFAULTS.get(key, False if key in _CONSTRUCTOR_ENABLES else None)
|
||||
and (replaced := _constructor_value(key, params.get(key), value)) is not _KEEP_CONFIGURED
|
||||
}
|
||||
return {**params, **overrides}
|
||||
|
||||
|
||||
class _Base64Stripper(CustomLogger):
|
||||
def strip(self, payload: StandardLoggingPayload) -> None:
|
||||
_ = self._strip_base64_from_messages_sync(payload)
|
||||
|
||||
def stripped(self, messages: list[object]) -> list[dict[str, object]]:
|
||||
return self._process_messages(messages=[_shallow_copy(message) for message in messages])
|
||||
|
||||
|
||||
def _shallow_copy(message: object) -> object:
|
||||
entries: Final = mapping(message)
|
||||
return copy.copy(entries) if entries is not None and isinstance(message, dict) else message
|
||||
|
||||
|
||||
_STRIPPER: Final = _Base64Stripper()
|
||||
|
||||
|
||||
def legacy_should_log(kwargs: Mapping[str, object]) -> bool:
|
||||
from litellm.litellm_core_utils import litellm_logging
|
||||
|
||||
if litellm.global_disable_no_log_param:
|
||||
return True
|
||||
litellm_params: Final = kwargs.get("litellm_params", {})
|
||||
lookup: Final = getattr(litellm_params, "get", None)
|
||||
if callable(lookup) and lookup("no-log", False) is True:
|
||||
return False
|
||||
return not litellm_logging.callback_disabled_dynamically(
|
||||
callback=CALLBACK_NAME,
|
||||
litellm_params=litellm_params,
|
||||
standard_callback_dynamic_params=kwargs.get("standard_callback_dynamic_params") or {}, # pyright: ignore[reportArgumentType] # the dynamic params dict the legacy path hands the same check
|
||||
)
|
||||
|
||||
|
||||
def _payload_messages(payload: Mapping[str, object]) -> list[object]:
|
||||
raw: Final = payload.get("messages")
|
||||
return [*raw] if isinstance(raw, list) else []
|
||||
|
||||
|
||||
class S3StripBase64Files(CustomLoggerV2):
|
||||
selection = Selection(name=STRIP_CALLBACK_NAME)
|
||||
|
||||
def __init__(self, s3_logger_key: str) -> None:
|
||||
self.s3_logger_key: Final = s3_logger_key
|
||||
|
||||
@on(
|
||||
Hook.log_prepare,
|
||||
reads={Field.deprecated_kwargs, Field.standard_logging_payload},
|
||||
writes={Field.standard_logging_payload},
|
||||
)
|
||||
async def on_log_prepare(self, view: object, ctx: CallContext[object]) -> Patch | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = mapping(getattr(view, Field.deprecated_kwargs.value, None))
|
||||
payload: Final = mapping(getattr(view, Field.standard_logging_payload.value, None))
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or kwargs is None or payload is None or not legacy_should_log(kwargs):
|
||||
return None
|
||||
messages: Final = _payload_messages(payload)
|
||||
if not messages:
|
||||
return None
|
||||
return Patch().merge(Field.standard_logging_payload, {"messages": _STRIPPER.stripped(messages)})
|
||||
|
||||
|
||||
async def _periodic_flush(sink: "NativeS3Log", interval: float) -> None:
|
||||
while True:
|
||||
await asyncio.sleep(interval)
|
||||
try:
|
||||
batch_upload_failed(await sink.flush())
|
||||
except (asyncio.CancelledError, KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except BaseException as e: # noqa: BLE001 # a failed flush keeps the queue and the next interval tries again, as flush_queue does
|
||||
verbose_logger.exception("CustomLogger: periodic flush failed: %s", e)
|
||||
|
||||
|
||||
def _excluding_fields(kwargs: object) -> Mapping[str, object] | None:
|
||||
excluded: Final[object] = getattr(litellm, "standard_logging_payload_excluded_fields", None)
|
||||
details: Final = mapping(kwargs)
|
||||
logged: Final = None if details is None else mapping(details.get("standard_logging_object"))
|
||||
if details is None or logged is None or not isinstance(excluded, Collection) or not excluded:
|
||||
return None
|
||||
if logged.get("status") == "failure":
|
||||
return None
|
||||
return {**details, "standard_logging_object": {key: value for key, value in logged.items() if key not in excluded}}
|
||||
|
||||
|
||||
def _logged_payload(kwargs: object, details: Mapping[str, object] | None) -> object:
|
||||
source: Final = mapping(kwargs) if details is None else details
|
||||
return None if source is None else source.get("standard_logging_object")
|
||||
|
||||
|
||||
def _legacy_kwargs(view: object) -> object:
|
||||
return getattr(view, Field.deprecated_kwargs.value, None)
|
||||
|
||||
|
||||
_read_attribute: Final[Callable[[object, str], object]] = getattr
|
||||
|
||||
|
||||
class _ExcludedView:
|
||||
def __init__(self, view: object, details: Mapping[str, object]) -> None:
|
||||
args: Final = HookArgs(request=details)
|
||||
self.view: Final = view
|
||||
self.fields: Final[Mapping[str, object]] = {
|
||||
Field.deprecated_kwargs.value: details,
|
||||
Field.standard_logging_payload_ref.value: payload_copy(args),
|
||||
Field.log_id.value: log_id(args),
|
||||
Field.cold_storage_object_key.value: cold_storage_object_key(args),
|
||||
Field.logged_key_identity.value: logged_key_identity(args),
|
||||
}
|
||||
|
||||
def __getattr__(self, name: str) -> object:
|
||||
if name not in self.fields:
|
||||
return _read_attribute(self.view, name)
|
||||
value: Final = self.fields[name]
|
||||
if value is None:
|
||||
raise AttributeError(name)
|
||||
return value
|
||||
|
||||
|
||||
_ATTEMPT_ENDED_READS: Final = frozenset(
|
||||
{
|
||||
Field.deprecated_kwargs,
|
||||
Field.attempt_timing,
|
||||
Field.log_id,
|
||||
Field.cold_storage_object_key,
|
||||
Field.standard_logging_payload_ref,
|
||||
Field.logged_key_identity,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class S3Logger(RustLogger):
|
||||
selection = Selection(name=CALLBACK_NAME, honors_no_log=True)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
s3_callback_params_override: Mapping[str, object] | None = None,
|
||||
**settings: Unpack[S3LoggerSettings], # kwargs-ok: the S3Logger constructor keywords
|
||||
) -> None:
|
||||
from litellm.rust_bridge._native import NativeS3Log
|
||||
|
||||
host: Final = _Host()
|
||||
params: Final = _constructed_params(s3_callback_params_override, settings)
|
||||
sink: Final = NativeS3Log(
|
||||
params,
|
||||
_constants(settings.get("s3_batch_size", DEFAULT_S3_BATCH_SIZE)),
|
||||
host,
|
||||
_cold_storage_is_s3_v2(),
|
||||
None if params.get("s3_region_name") else _region_from_environment(),
|
||||
)
|
||||
self.s3_region_name: Final[object] = params.get("s3_region_name") or settings.get("s3_region_name")
|
||||
self.sink: Final = sink
|
||||
self.s3_host: Final = host
|
||||
self.periodic_flush_task: Final = asyncio.create_task(
|
||||
_periodic_flush(sink, _flush_interval(settings.get("s3_flush_interval", DEFAULT_S3_FLUSH_INTERVAL_SECONDS)))
|
||||
)
|
||||
super().__init__(sink.callback(), {Field.deprecated_kwargs: lambda _kwargs: None})
|
||||
self.v2_companion: Final = (
|
||||
S3StripBase64Files(litellm.logging_callback_manager.get_custom_logger_key(self))
|
||||
if sink.strip_base64_files
|
||||
else None
|
||||
)
|
||||
|
||||
@on(Hook.attempt_ended, reads=_ATTEMPT_ENDED_READS, awaited=True)
|
||||
async def on_attempt_ended(self, view: object, ctx: CallContext[object]) -> Patch | Abort | None:
|
||||
return await self.run_rust(Hook.attempt_ended, _ATTEMPT_ENDED_READS, view, ctx)
|
||||
|
||||
async def run_rust(
|
||||
self, hook: Hook, reads: frozenset[Field], view: object, ctx: CallContext[object]
|
||||
) -> Patch | Abort | None:
|
||||
from litellm.litellm_core_utils.litellm_logging import V2_ASYNC_LOGGING_LINE
|
||||
|
||||
kwargs: Final = getattr(view, Field.deprecated_kwargs.value, None)
|
||||
if not V2_ASYNC_LOGGING_LINE.get() or not isinstance(kwargs, Mapping) or not legacy_should_log(kwargs): # pyright: ignore[reportUnknownArgumentType] # the legacy kwargs dict
|
||||
return None
|
||||
payload: Final[object] = kwargs.get("standard_logging_object") # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # the legacy kwargs dict
|
||||
if payload is not None and not isinstance(payload, Mapping):
|
||||
return self._failed(AttributeError(f"'{type(payload).__name__}' object has no attribute 'get'")) # pyright: ignore[reportUnknownArgumentType] # any non-mapping payload
|
||||
try:
|
||||
raw: Final = _legacy_kwargs(view)
|
||||
details: Final = _excluding_fields(raw)
|
||||
logged: Final = _logged_payload(raw, details)
|
||||
if self.sink.strip_base64_files and isinstance(logged, Mapping):
|
||||
_STRIPPER.strip(logged) # pyright: ignore[reportArgumentType] # the shared StandardLoggingPayload, stripped in place as S3Logger does
|
||||
return await super().run_rust(hook, reads, view if details is None else _ExcludedView(view, details), ctx)
|
||||
except Exception as e: # noqa: BLE001 # S3Logger counts a failure to build the element and moves on
|
||||
return self._failed(e)
|
||||
|
||||
def _failed(self, error: Exception) -> None:
|
||||
verbose_logger.error("s3 Layer Error - %s", error, exc_info=error)
|
||||
self.s3_host.callback_failed()
|
||||
|
||||
@property
|
||||
def s3_path(self) -> str | None:
|
||||
return self.sink.s3_path
|
||||
|
||||
@property
|
||||
def s3_bucket_name(self) -> str | None:
|
||||
return self.sink.s3_bucket_name
|
||||
|
||||
def resolve_partition_granularity(self) -> S3PartitionGranularity:
|
||||
return "hour" if self.sink.partition_granularity() == "hour" else "day"
|
||||
|
||||
async def flush_queue(self) -> None:
|
||||
batch_upload_failed(await self.sink.flush())
|
||||
|
||||
async def async_log_audit_log_event(self, audit_log: StandardAuditLogPayload) -> None:
|
||||
try:
|
||||
audit_log_id: Final = audit_log.get("id", "unknown")
|
||||
batch_upload_failed(await self.sink.submit_audit(dict(audit_log), f"{audit_log_id}"))
|
||||
except Exception as e: # noqa: BLE001 # an audit log failure is logged and swallowed, as in S3Logger
|
||||
verbose_logger.exception("S3 audit log error: %s", e)
|
||||
|
||||
async def async_log_batch_line_items(self, payload: StandardBatchLineItemsPayload) -> None:
|
||||
object_name: Final = batch_line_items_object_name(payload)
|
||||
s3_path: Final = self.s3_path
|
||||
s3_object_key: Final = (f"{s3_path.rstrip('/')}/" if s3_path else "") + object_name
|
||||
body: Final = await asyncio.to_thread(batch_line_items_ndjson, payload)
|
||||
delivered: Final = await self.sink.upload_line_items(
|
||||
s3_object_key, body, "-".join(object_name.rsplit("/", 2)[-2:])
|
||||
)
|
||||
if not delivered:
|
||||
verbose_logger.error(
|
||||
"s3 logging: the %d line items of part %d of batch %s were not stored at %s",
|
||||
len(payload["lines"]),
|
||||
payload["part"],
|
||||
payload["batch_id"],
|
||||
s3_object_key,
|
||||
)
|
||||
|
||||
async def _download_object_from_s3(self, s3_object_key: str) -> object:
|
||||
try:
|
||||
downloaded: Final = await self.sink.download(s3_object_key)
|
||||
return None if downloaded is None else json.loads(downloaded)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error downloading from S3: %s", e)
|
||||
return None
|
||||
|
||||
async def get_proxy_server_request_from_cold_storage_with_object_key(self, object_key: str) -> object:
|
||||
try:
|
||||
return await self._download_object_from_s3(object_key)
|
||||
except Exception as e: # noqa: BLE001 # a failed read is None, as in S3Logger
|
||||
verbose_logger.exception("Error retrieving object %s from cold storage: %s", object_key, e)
|
||||
return None
|
||||
|
||||
|
||||
__all__ = [
|
||||
"S3BatchUploadError",
|
||||
"S3Logger",
|
||||
"S3StripBase64Files",
|
||||
"legacy_should_log",
|
||||
]
|
||||
|
|
@ -46,7 +46,7 @@ from litellm.integrations.opik.opik import OpikLogger
|
|||
from litellm.integrations.pointfive import PointFiveLogger
|
||||
from litellm.integrations.posthog import PostHogLogger
|
||||
from litellm.integrations.prometheus import PrometheusLogger
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
from litellm.integrations.sqs import SQSLogger
|
||||
from litellm.integrations.ternary.ternary_logger import TernaryLogger
|
||||
from litellm.integrations.vantage.vantage_logger import VantageLogger
|
||||
|
|
|
|||
|
|
@ -171,8 +171,12 @@ def is_annual_budget_duration(duration: str) -> bool:
|
|||
return _parse_duration(normalize_duration(duration)) == (12, "mo")
|
||||
|
||||
|
||||
def normalize_budget_window_start(moment: datetime) -> datetime:
|
||||
return moment.replace(microsecond=moment.microsecond // 1000 * 1000)
|
||||
|
||||
|
||||
def get_annual_budget_window_start(reset_at: datetime, timezone_str: str) -> datetime:
|
||||
boundary: Final = _setup_timezone(reset_at, timezone_str)[0]
|
||||
boundary: Final = _setup_timezone(normalize_budget_window_start(reset_at), timezone_str)[0]
|
||||
return _shift_months(boundary, -12).astimezone(timezone.utc)
|
||||
|
||||
|
||||
|
|
@ -180,24 +184,25 @@ def get_budget_window_start(duration: str, reset_at: datetime) -> datetime:
|
|||
"""Start of the budget period that ends at `reset_at`, under the same rules
|
||||
`get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar
|
||||
months and a duration it does not recognize resets at the next midnight."""
|
||||
boundary: Final = normalize_budget_window_start(reset_at)
|
||||
value, unit = _parse_duration(normalize_duration(duration))
|
||||
if value is None:
|
||||
return reset_at - timedelta(days=1)
|
||||
return boundary - timedelta(days=1)
|
||||
match unit:
|
||||
case "mo":
|
||||
return _shift_months(reset_at, -value)
|
||||
return _shift_months(boundary, -value)
|
||||
case "d":
|
||||
return _shift_months(reset_at, -1) if value == 30 else reset_at - timedelta(days=value)
|
||||
return _shift_months(boundary, -1) if value == 30 else boundary - timedelta(days=value)
|
||||
case "w":
|
||||
return reset_at - timedelta(weeks=value)
|
||||
return boundary - timedelta(weeks=value)
|
||||
case "h":
|
||||
return reset_at - timedelta(hours=value)
|
||||
return boundary - timedelta(hours=value)
|
||||
case "m":
|
||||
return reset_at - timedelta(minutes=value)
|
||||
return boundary - timedelta(minutes=value)
|
||||
case "s":
|
||||
return reset_at - timedelta(seconds=value)
|
||||
return boundary - timedelta(seconds=value)
|
||||
case _:
|
||||
return reset_at - timedelta(days=1)
|
||||
return boundary - timedelta(days=1)
|
||||
|
||||
|
||||
def _setup_timezone(current_time: datetime, timezone_str: str = "UTC") -> tuple[datetime, tzinfo]:
|
||||
|
|
|
|||
|
|
@ -6536,7 +6536,7 @@ def build_custom_logger_compatible_class(
|
|||
_in_memory_loggers.append(_gcs_bucket_logger)
|
||||
return _gcs_bucket_logger
|
||||
elif logging_integration == "s3_v2":
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, S3V2Logger):
|
||||
|
|
@ -7360,7 +7360,7 @@ def get_custom_logger_compatible_class(
|
|||
if isinstance(callback, GCSBucketLogger):
|
||||
return callback
|
||||
elif logging_integration == "s3_v2":
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
|
||||
for callback in _in_memory_loggers:
|
||||
if isinstance(callback, S3V2Logger):
|
||||
|
|
@ -8067,7 +8067,7 @@ class StandardLoggingPayloadSetup:
|
|||
)
|
||||
if custom_logger and hasattr(custom_logger, "s3_path") and getattr(custom_logger, "s3_path"):
|
||||
s3_path = getattr(custom_logger, "s3_path")
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
|
||||
if isinstance(custom_logger, S3V2Logger):
|
||||
partition_granularity = custom_logger.resolve_partition_granularity()
|
||||
|
|
|
|||
|
|
@ -1008,9 +1008,8 @@ class ChunkProcessor:
|
|||
cache_creation_token_details: CacheCreationTokenDetails | None = None
|
||||
cost: float | None = None
|
||||
|
||||
for chunk in chunks:
|
||||
usage_chunk = self.extract_usage_chunk(chunk)
|
||||
|
||||
usage_chunks: Final = tuple(self.extract_usage_chunk(chunk) for chunk in chunks)
|
||||
for usage_chunk in usage_chunks:
|
||||
if usage_chunk is not None:
|
||||
usage_chunk_dict = self._usage_chunk_calculation_helper(usage_chunk)
|
||||
if usage_chunk_dict["prompt_tokens"] is not None and (
|
||||
|
|
@ -1091,15 +1090,12 @@ class ChunkProcessor:
|
|||
completion_tokens_details=None if cursor_was_reset else completion_tokens_details,
|
||||
prompt_tokens_details=prompt_tokens_details,
|
||||
cost=cost,
|
||||
inference_geo=self._last_provider_pricing_field(chunks, "inference_geo"),
|
||||
speed=self._last_provider_pricing_field(chunks, "speed"),
|
||||
inference_geo=self._last_provider_pricing_field(usage_chunks, "inference_geo"),
|
||||
speed=self._last_provider_pricing_field(usage_chunks, "speed"),
|
||||
)
|
||||
|
||||
def _last_provider_pricing_field(
|
||||
self,
|
||||
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
|
||||
field: str,
|
||||
) -> str | None:
|
||||
@staticmethod
|
||||
def _last_provider_pricing_field(usage_chunks: Sequence[Usage | None], field: str) -> str | None:
|
||||
"""
|
||||
Last value of a provider-specific usage field that changes pricing but is not a
|
||||
declared ``Usage`` field, e.g. Anthropic's ``speed`` (fast mode multiplies
|
||||
|
|
@ -1107,9 +1103,8 @@ class ChunkProcessor:
|
|||
"""
|
||||
values: Final = [
|
||||
value
|
||||
for chunk in chunks
|
||||
if (usage_chunk := self.extract_usage_chunk(chunk)) is not None
|
||||
and isinstance(value := getattr(usage_chunk, field, None), str)
|
||||
for usage_chunk in usage_chunks
|
||||
if usage_chunk is not None and isinstance(value := getattr(usage_chunk, field, None), str)
|
||||
]
|
||||
return values[-1] if values else None
|
||||
|
||||
|
|
|
|||
|
|
@ -1884,9 +1884,11 @@ def _count_anthropic_content(
|
|||
except Exception as e:
|
||||
if default_token_count is not None:
|
||||
return default_token_count
|
||||
raise (UnsupportedContentTypeError if isinstance(e, UnsupportedContentTypeError) else ValueError)(
|
||||
f"Error counting field '{field_name}': {e}"
|
||||
)
|
||||
raise (
|
||||
UnsupportedContentTypeError
|
||||
if server_fields is not None or isinstance(e, UnsupportedContentTypeError)
|
||||
else ValueError
|
||||
)(f"Error counting field '{field_name}': {e}")
|
||||
return tokens
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -79,6 +79,18 @@ def _without_region_path(model: str) -> str:
|
|||
return model if region is None else model_id
|
||||
|
||||
|
||||
_ASYNC_INVOKE_REQUEST_FIELDS: Final = frozenset({"clientRequestToken", "tags"})
|
||||
|
||||
|
||||
def _split_async_invoke_extra_body(
|
||||
extra_body: Mapping[str, object],
|
||||
) -> tuple[Mapping[str, object], Mapping[str, object]]:
|
||||
return (
|
||||
{key: value for key, value in extra_body.items() if key in _ASYNC_INVOKE_REQUEST_FIELDS},
|
||||
{key: value for key, value in extra_body.items() if key not in _ASYNC_INVOKE_REQUEST_FIELDS},
|
||||
)
|
||||
|
||||
|
||||
class BedrockEmbedding(BaseAWSLLM):
|
||||
_web_identity_session_policy: ClassVar[WebIdentitySessionPolicy] = BEDROCK_EMBEDDING_SESSION_POLICY
|
||||
|
||||
|
|
@ -296,7 +308,13 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
|
||||
for data in (
|
||||
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
|
||||
if "modelInput" in item
|
||||
else {**extra_body, **item}
|
||||
for item in batch_data
|
||||
):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
@ -366,7 +384,13 @@ class BedrockEmbedding(BaseAWSLLM):
|
|||
extra_body: Mapping[str, object] = MappingProxyType({}),
|
||||
):
|
||||
responses: Final[list[dict]] = []
|
||||
for data in ({**extra_body, **item} for item in batch_data):
|
||||
request_fields, model_fields = _split_async_invoke_extra_body(extra_body)
|
||||
for data in (
|
||||
{**request_fields, **item, "modelInput": {**model_fields, **item["modelInput"]}}
|
||||
if "modelInput" in item
|
||||
else {**extra_body, **item}
|
||||
for item in batch_data
|
||||
):
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if extra_headers is not None:
|
||||
headers = {"Content-Type": "application/json", **extra_headers}
|
||||
|
|
|
|||
|
|
@ -92,6 +92,7 @@ class FalAIBytedanceSeedreamV3Config(FalAIBytedanceBaseConfig):
|
|||
"""
|
||||
|
||||
IMAGE_GENERATION_ENDPOINT: str = "fal-ai/bytedance/seedream/v3/text-to-image"
|
||||
MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/bytedance/seedream/", "bytedance/seedream/")
|
||||
|
||||
|
||||
class FalAIBytedanceDreaminaV31Config(FalAIBytedanceBaseConfig):
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class FalAIFluxProV11UltraConfig(FalAIBaseConfig):
|
|||
"""
|
||||
|
||||
IMAGE_GENERATION_ENDPOINT: str = "fal-ai/flux-pro/v1.1-ultra"
|
||||
MODEL_FAMILY_ROOTS: tuple[str, ...] = ("fal-ai/flux-pro/",)
|
||||
|
||||
def get_supported_openai_params(self, model: str) -> list[OpenAIImageGenerationOptionalParams]:
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import httpx
|
|||
from pydantic import ConfigDict, TypeAdapter
|
||||
from typing_extensions import ReadOnly, TypedDict
|
||||
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.litellm_core_utils.url_utils import encode_url_path_segments, join_api_base
|
||||
from litellm.llms.base_llm.image_generation.transformation import (
|
||||
BaseImageGenerationConfig,
|
||||
)
|
||||
|
|
@ -71,6 +71,7 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
|
|||
|
||||
DEFAULT_BASE_URL: str = "https://fal.run"
|
||||
IMAGE_GENERATION_ENDPOINT: str = ""
|
||||
MODEL_FAMILY_ROOTS: tuple[str, ...] = ()
|
||||
|
||||
def get_complete_url(
|
||||
self,
|
||||
|
|
@ -89,7 +90,16 @@ class FalAIBaseConfig(BaseImageGenerationConfig):
|
|||
base_url: Final = api_base or get_secret_str("FAL_AI_API_BASE") or self.DEFAULT_BASE_URL
|
||||
if not self.IMAGE_GENERATION_ENDPOINT:
|
||||
return base_url.rstrip("/")
|
||||
return join_api_base(base_url, self.IMAGE_GENERATION_ENDPOINT)
|
||||
return join_api_base(base_url, self._endpoint_for_model(model))
|
||||
|
||||
def _endpoint_for_model(self, model: str) -> str:
|
||||
path: Final = encode_url_path_segments(model.strip("/"), field_name="model")
|
||||
names_default_endpoint: Final = path.lower().removeprefix(
|
||||
"fal-ai/"
|
||||
) == self.IMAGE_GENERATION_ENDPOINT.removeprefix("fal-ai/")
|
||||
if names_default_endpoint or not path.lower().startswith(self.MODEL_FAMILY_ROOTS):
|
||||
return self.IMAGE_GENERATION_ENDPOINT
|
||||
return path
|
||||
|
||||
def validate_environment(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ class _ModelProfile:
|
|||
_SEEDANCE_PROFILE: Final[_ModelProfile] = _ModelProfile(
|
||||
resolutions=frozenset({"480p", "720p", "1080p", "4k"}),
|
||||
resolution_tiers=((480, "480p"), (720, "720p"), (1080, "1080p"), (sys.maxsize, "4k")),
|
||||
default_resolution="720p",
|
||||
default_resolution=None,
|
||||
integer_duration=False,
|
||||
reference_key="image_url",
|
||||
reference_as_list=False,
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
"""
|
||||
Supports writing files to Google AI Studio Files API.
|
||||
|
||||
For vertex ai, check out the vertex_ai/files/handler.py file.
|
||||
For vertex ai, check out the vertex_ai/files/transformation.py file.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ Helper util for handling perplexity-specific cost calculation
|
|||
- e.g.: citation tokens, search queries
|
||||
"""
|
||||
|
||||
import math
|
||||
from datetime import datetime
|
||||
from typing import Final
|
||||
|
||||
|
|
@ -52,13 +53,14 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
except (ValueError, TypeError):
|
||||
return default
|
||||
|
||||
cache_read_cost_value: Final = model_info.get("cache_read_input_token_cost")
|
||||
rates: Final = apply_off_peak_pricing(
|
||||
model_info,
|
||||
current_time,
|
||||
TokenRates(
|
||||
input_rate=_safe_float_cast(model_info.get("input_cost_per_token")),
|
||||
output_rate=_safe_float_cast(model_info.get("output_cost_per_token")),
|
||||
cache_read_rate=0.0,
|
||||
cache_read_rate=math.nan if cache_read_cost_value is None else _safe_float_cast(cache_read_cost_value),
|
||||
cache_creation_rate=0.0,
|
||||
reasoning_rate=None,
|
||||
),
|
||||
|
|
@ -67,7 +69,15 @@ def cost_per_token(model: str, usage: Usage, current_time: datetime | None = Non
|
|||
output_cost_per_token: Final = rates.output_rate
|
||||
|
||||
## CALCULATE INPUT COST
|
||||
prompt_cost: float = (usage.prompt_tokens or 0) * input_cost_per_token
|
||||
prompt_tokens: Final = usage.prompt_tokens or 0
|
||||
reported_cached_tokens: Final = getattr(usage.prompt_tokens_details, "cached_tokens", 0) or 0
|
||||
cached_tokens: Final = min(prompt_tokens, reported_cached_tokens)
|
||||
cache_read_cost_per_token: Final = (
|
||||
input_cost_per_token if math.isnan(rates.cache_read_rate) else rates.cache_read_rate
|
||||
)
|
||||
prompt_cost: float = (
|
||||
prompt_tokens - cached_tokens
|
||||
) * input_cost_per_token + cached_tokens * cache_read_cost_per_token
|
||||
|
||||
## ADD CITATION TOKENS COST (if present)
|
||||
citation_tokens: Final = getattr(usage, "citation_tokens", 0) or 0
|
||||
|
|
|
|||
|
|
@ -77,13 +77,19 @@ def _profile_for_model(model: str) -> _ModelProfile:
|
|||
return _MINIMAX_PROFILE if replicate_model_owner(model) == "minimax" else _GENERIC_PROFILE
|
||||
|
||||
|
||||
def _invalid_seconds() -> ReplicateError:
|
||||
return ReplicateError(status_code=400, message="Replicate seconds must be a finite number", headers={})
|
||||
|
||||
|
||||
def _duration_value(seconds: object) -> int | float:
|
||||
if isinstance(seconds, bool) or not isinstance(seconds, (int, float, str)):
|
||||
raise TypeError("Replicate seconds must be a numeric value")
|
||||
raise _invalid_seconds()
|
||||
try:
|
||||
duration: Final = float(seconds)
|
||||
except ValueError as error:
|
||||
raise ValueError("Replicate seconds must be a numeric value") from error
|
||||
except (ValueError, OverflowError) as error:
|
||||
raise _invalid_seconds() from error
|
||||
if not math.isfinite(duration):
|
||||
raise _invalid_seconds()
|
||||
return int(duration) if duration.is_integer() else duration
|
||||
|
||||
|
||||
|
|
@ -99,7 +105,7 @@ def _size_params(size: object, profile: _ModelProfile) -> Mapping[str, object]:
|
|||
if normalized.count("x") != 1:
|
||||
return MappingProxyType({})
|
||||
width_text, height_text = normalized.split("x")
|
||||
if not (width_text.isdigit() and height_text.isdigit()) or int(width_text) <= 0 or int(height_text) <= 0:
|
||||
if not (width_text.isdecimal() and height_text.isdecimal()) or int(width_text) <= 0 or int(height_text) <= 0:
|
||||
return MappingProxyType({})
|
||||
width: Final = int(width_text)
|
||||
height: Final = int(height_text)
|
||||
|
|
|
|||
|
|
@ -1,239 +0,0 @@
|
|||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Coroutine, Mapping
|
||||
from typing import Final
|
||||
from urllib.parse import unquote
|
||||
|
||||
import httpx
|
||||
|
||||
from litellm import LlmProviders
|
||||
from litellm.integrations.gcs_bucket.gcs_bucket_base import (
|
||||
GCSBucketBase,
|
||||
GCSLoggingConfig,
|
||||
)
|
||||
from litellm.litellm_core_utils.cloud_storage_security import (
|
||||
VERTEX_AI_MANAGED_GCS_PREFIX,
|
||||
should_allow_legacy_cloud_file_ids,
|
||||
validate_managed_cloud_file_id,
|
||||
)
|
||||
from litellm.litellm_core_utils.litellm_logging import Logging
|
||||
from litellm.llms.custom_httpx.http_handler import get_async_httpx_client
|
||||
from litellm.types.llms.openai import (
|
||||
FileContentRequest,
|
||||
HttpxBinaryResponseContent,
|
||||
)
|
||||
from litellm.types.llms.vertex_ai import VERTEX_CREDENTIALS_TYPES
|
||||
from litellm.types.utils import StandardCallbackDynamicParams
|
||||
|
||||
from .transformation import VertexAIFilesConfig
|
||||
|
||||
|
||||
class VertexAIFilesHandler(GCSBucketBase):
|
||||
"""
|
||||
Handles Calling VertexAI in OpenAI Files API format v1/files/*
|
||||
|
||||
This implementation uploads files on GCS Buckets
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.async_httpx_client = get_async_httpx_client(
|
||||
llm_provider=LlmProviders.VERTEX_AI,
|
||||
)
|
||||
|
||||
def _resolve_read_gcs_config(
|
||||
self,
|
||||
litellm_params: Mapping[str, object] | None,
|
||||
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""
|
||||
Resolve the GCS bucket and service-account credentials for the read/content path.
|
||||
|
||||
Sources them from the deployment's ``litellm_params`` (``gcs_bucket_name`` /
|
||||
``bucket_name`` and ``vertex_credentials``), mirroring the write path in
|
||||
``VertexAIFilesConfig._get_configured_bucket_name``, and falls back to the
|
||||
``GCS_BATCH_BUCKET_NAME`` then ``GCS_BUCKET_NAME`` / ``GCS_PATH_SERVICE_ACCOUNT``
|
||||
env vars. This lets Vertex batch run entirely at the model-group level, so output
|
||||
written to a per-model bucket is readable without setting the global env vars.
|
||||
"""
|
||||
params: Final[Mapping[str, object]] = litellm_params or {}
|
||||
bucket_candidate: Final = params.get("gcs_bucket_name") or params.get("bucket_name")
|
||||
configured_bucket_name = (
|
||||
bucket_candidate
|
||||
if isinstance(bucket_candidate, str)
|
||||
else os.getenv("GCS_BATCH_BUCKET_NAME") or os.getenv("GCS_BUCKET_NAME")
|
||||
)
|
||||
|
||||
credentials: Final = params.get("vertex_credentials") or vertex_credentials
|
||||
if isinstance(credentials, dict):
|
||||
path_service_account: str | None = json.dumps(credentials)
|
||||
elif isinstance(credentials, str):
|
||||
path_service_account = credentials
|
||||
else:
|
||||
path_service_account = os.getenv("GCS_PATH_SERVICE_ACCOUNT")
|
||||
|
||||
return configured_bucket_name, path_service_account
|
||||
|
||||
def _extract_bucket_and_object_from_file_id(
|
||||
self,
|
||||
file_id: str,
|
||||
configured_bucket_name: str,
|
||||
litellm_params: dict | None = None,
|
||||
) -> tuple[str, str]:
|
||||
"""
|
||||
Validate and extract bucket name and object path from file_id.
|
||||
|
||||
Expected format: gs://bucket-name/litellm-vertex-files/path/to/file
|
||||
|
||||
Returns:
|
||||
tuple: (bucket_name, object_path)
|
||||
- bucket_name: "bucket-name"
|
||||
- object_path: "litellm-vertex-files/path/to/file"
|
||||
"""
|
||||
return validate_managed_cloud_file_id(
|
||||
file_id=file_id,
|
||||
scheme="gs://",
|
||||
configured_bucket_name=configured_bucket_name,
|
||||
allowed_object_prefixes=(VERTEX_AI_MANAGED_GCS_PREFIX,),
|
||||
allow_legacy_cloud_file_ids=should_allow_legacy_cloud_file_ids(litellm_params),
|
||||
)
|
||||
|
||||
async def afile_content(
|
||||
self,
|
||||
file_content_request: FileContentRequest,
|
||||
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
timeout: float | httpx.Timeout,
|
||||
max_retries: int | None,
|
||||
litellm_params: dict | None = None,
|
||||
) -> HttpxBinaryResponseContent:
|
||||
"""
|
||||
Download file content from GCS bucket for VertexAI files.
|
||||
|
||||
Args:
|
||||
file_content_request: Contains file_id (URL-encoded GCS path)
|
||||
vertex_credentials: VertexAI credentials
|
||||
vertex_project: VertexAI project ID
|
||||
vertex_location: VertexAI location
|
||||
timeout: Request timeout
|
||||
max_retries: Max retry attempts
|
||||
|
||||
Returns:
|
||||
HttpxBinaryResponseContent: Binary content wrapped in compatible response format
|
||||
"""
|
||||
file_id: Final = file_content_request.get("file_id")
|
||||
if not file_id:
|
||||
raise ValueError("file_id is required in file_content_request")
|
||||
|
||||
configured_bucket_name, path_service_account = self._resolve_read_gcs_config(
|
||||
litellm_params=litellm_params,
|
||||
vertex_credentials=vertex_credentials,
|
||||
)
|
||||
dynamic_params: Final = StandardCallbackDynamicParams(
|
||||
gcs_bucket_name=configured_bucket_name,
|
||||
gcs_path_service_account=path_service_account,
|
||||
)
|
||||
gcs_logging_config: Final[GCSLoggingConfig] = await self.get_gcs_logging_config(
|
||||
kwargs={"standard_callback_dynamic_params": dynamic_params}
|
||||
)
|
||||
bucket_name, object_path = self._extract_bucket_and_object_from_file_id(
|
||||
file_id=file_id,
|
||||
configured_bucket_name=gcs_logging_config["bucket_name"],
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
|
||||
download_kwargs: Final = {
|
||||
"standard_callback_dynamic_params": {
|
||||
"gcs_bucket_name": bucket_name,
|
||||
"gcs_path_service_account": gcs_logging_config["path_service_account"],
|
||||
}
|
||||
}
|
||||
|
||||
file_content: Final = await self.download_gcs_object(object_name=object_path, **download_kwargs)
|
||||
decoded_file_id: Final = unquote(file_id)
|
||||
|
||||
if file_content is None:
|
||||
raise ValueError(f"Failed to download file from GCS: {decoded_file_id}")
|
||||
|
||||
mock_response: Final = httpx.Response(
|
||||
status_code=200,
|
||||
content=file_content,
|
||||
headers={
|
||||
"content-type": "application/octet-stream",
|
||||
"content-length": str(len(file_content)),
|
||||
},
|
||||
request=httpx.Request(method="GET", url=decoded_file_id),
|
||||
)
|
||||
|
||||
# Apply transformation to convert Vertex AI batch outputs to OpenAI format
|
||||
config: Final = VertexAIFilesConfig()
|
||||
|
||||
# Create a logging object for transformation
|
||||
logging_obj: Final = Logging(
|
||||
model="",
|
||||
messages=[],
|
||||
stream=False,
|
||||
call_type="afile_content",
|
||||
start_time=time.time(),
|
||||
litellm_call_id="",
|
||||
function_id="",
|
||||
)
|
||||
|
||||
return config.transform_file_content_response(
|
||||
raw_response=mock_response, logging_obj=logging_obj, litellm_params={}
|
||||
)
|
||||
|
||||
def file_content(
|
||||
self,
|
||||
_is_async: bool,
|
||||
file_content_request: FileContentRequest,
|
||||
api_base: str | None,
|
||||
vertex_credentials: VERTEX_CREDENTIALS_TYPES | None,
|
||||
vertex_project: str | None,
|
||||
vertex_location: str | None,
|
||||
timeout: float | httpx.Timeout,
|
||||
max_retries: int | None,
|
||||
litellm_params: dict | None = None,
|
||||
) -> HttpxBinaryResponseContent | Coroutine[object, object, HttpxBinaryResponseContent]:
|
||||
"""
|
||||
Download file content from GCS bucket for VertexAI files.
|
||||
Supports both sync and async operations.
|
||||
|
||||
Args:
|
||||
_is_async: Whether to run asynchronously
|
||||
file_content_request: Contains file_id (URL-encoded GCS path)
|
||||
api_base: API base (unused for GCS operations)
|
||||
vertex_credentials: VertexAI credentials
|
||||
vertex_project: VertexAI project ID
|
||||
vertex_location: VertexAI location
|
||||
timeout: Request timeout
|
||||
max_retries: Max retry attempts
|
||||
|
||||
Returns:
|
||||
HttpxBinaryResponseContent or Coroutine: Binary content wrapped in compatible response format
|
||||
"""
|
||||
if _is_async:
|
||||
return self.afile_content(
|
||||
file_content_request=file_content_request,
|
||||
vertex_credentials=vertex_credentials,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
else:
|
||||
return asyncio.run(
|
||||
self.afile_content(
|
||||
file_content_request=file_content_request,
|
||||
vertex_credentials=vertex_credentials,
|
||||
vertex_project=vertex_project,
|
||||
vertex_location=vertex_location,
|
||||
timeout=timeout,
|
||||
max_retries=max_retries,
|
||||
litellm_params=litellm_params,
|
||||
)
|
||||
)
|
||||
|
|
@ -10,8 +10,9 @@ Model name handling:
|
|||
- This module focuses on request/response transformation only
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
from typing import Final, cast # noqa: TID251 # extra_body carries raw fields the TypedDict lacks
|
||||
|
||||
from litellm.llms.base_llm.embedding.transformation import split_extra_body
|
||||
from litellm.types.utils import EmbeddingResponse, Usage
|
||||
|
||||
from .types import (
|
||||
|
|
@ -84,7 +85,12 @@ class VertexBGEConfig:
|
|||
vertex_text_embedding_input_list.append(embedding_input)
|
||||
|
||||
vertex_request["instances"] = vertex_text_embedding_input_list
|
||||
vertex_request["parameters"] = EmbeddingParameters(**optional_params)
|
||||
body_params, extra_body = split_extra_body(optional_params)
|
||||
vertex_request["parameters"] = (
|
||||
cast( # cast-ok: extra_body carries raw Vertex fields the TypedDict does not list
|
||||
EmbeddingParameters, {**extra_body, **body_params}
|
||||
)
|
||||
)
|
||||
|
||||
return vertex_request
|
||||
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ from typing import Final, TypeAlias, cast # noqa: TID251 # native binding sele
|
|||
from litellm.exceptions import BadRequestError, UnsupportedParamsError
|
||||
from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider
|
||||
from litellm.llms.anthropic.messages.native import handler as main
|
||||
from litellm.llms.bedrock.messages.invoke_transformations.anthropic_claude_transformation import (
|
||||
AmazonAnthropicClaudeMessagesConfig,
|
||||
)
|
||||
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
|
||||
from litellm.rust_bridge import catalog, detectors
|
||||
from litellm.rust_bridge.catalog import Feature, Route, RouteContext, Target
|
||||
|
|
@ -105,8 +108,11 @@ def _route_target(
|
|||
api_base: str | None,
|
||||
compacting: bool,
|
||||
) -> Target:
|
||||
if main._provider_messages_config(model, provider, api_base) is not None: # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
|
||||
return Target.MESSAGES
|
||||
provider_config: Final = main._provider_messages_config(model, provider, api_base) # pyright: ignore[reportPrivateUsage] # the handler's own config lookup
|
||||
if provider_config is not None:
|
||||
return (
|
||||
Target.BEDROCK_INVOKE if type(provider_config) is AmazonAnthropicClaudeMessagesConfig else Target.MESSAGES
|
||||
)
|
||||
if main._deployment_passes_through_anthropic_messages(model_info): # pyright: ignore[reportPrivateUsage] # the handler's own passthrough opt-in
|
||||
return Target.MESSAGES_PASSTHROUGH
|
||||
if not compacting and main._should_route_to_responses_api(provider, requested_model, model, model_info): # pyright: ignore[reportPrivateUsage] # the handler's own translation choice
|
||||
|
|
|
|||
|
|
@ -101932,5 +101932,31 @@
|
|||
"supports_parallel_function_calling": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
"fal_ai/bytedance/seedream/v5/pro/us/edit": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K with one input image; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135; the first input image is free and each additional input image adds $0.0045 per output, which is not modeled; fal calls this pricing tentative"},
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.135,
|
||||
"output_cost_per_image_1K": 0.0675,
|
||||
"output_cost_per_image_2K": 0.135,
|
||||
"source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/edit",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/edits",
|
||||
"/v1/images/generations"
|
||||
],
|
||||
"supports_vision": true
|
||||
},
|
||||
"fal_ai/bytedance/seedream/v5/pro/us/text-to-image": {
|
||||
"litellm_provider": "fal_ai",
|
||||
"metadata": {"comment":"per output image by total pixel area at the default image_size auto_2K; outputs up to and including 1536x1536 pixels are $0.0675 and outputs above that up to 2048x2048 are $0.135, so 1.5K has no catalog field"},
|
||||
"mode": "image_generation",
|
||||
"output_cost_per_image": 0.135,
|
||||
"output_cost_per_image_1K": 0.0675,
|
||||
"output_cost_per_image_2K": 0.135,
|
||||
"source": "https://fal.ai/models/bytedance/seedream/v5/pro/us/text-to-image",
|
||||
"supported_endpoints": [
|
||||
"/v1/images/generations"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy"
|
||||
version = "1.109.0"
|
||||
version = "1.111.0"
|
||||
description = "Opt-in Rust HTTP forwarding server for the LiteLLM Python proxy"
|
||||
requires-python = ">=3.10, <3.16"
|
||||
license = "MIT"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
CREATE TABLE IF NOT EXISTS "LiteLLM_SpendLogDelivery" (
|
||||
"delivery_id" TEXT PRIMARY KEY DEFAULT gen_random_uuid()::text,
|
||||
"delivery_id" TEXT PRIMARY KEY,
|
||||
"request_id" TEXT NOT NULL,
|
||||
"state" "AccountingDeliveryState" NOT NULL DEFAULT 'pending',
|
||||
"attempts" INTEGER NOT NULL DEFAULT 0,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,2 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
ALTER TABLE "LiteLLM_SpendLogDelivery" ALTER COLUMN "delivery_id" DROP DEFAULT;
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
SET LOCAL lock_timeout = '10s';
|
||||
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_parent_team_ids_idx" ON "LiteLLM_CustomerMemberTable" USING GIN ("parent_team_ids");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerMemberTable_budget_override_id_idx" ON "LiteLLM_CustomerMemberTable"("budget_override_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_id");
|
||||
CREATE INDEX IF NOT EXISTS "LiteLLM_CustomerTeamTable_budget_override_id_idx" ON "LiteLLM_CustomerTeamTable"("budget_override_id");
|
||||
|
|
@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable {
|
|||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String
|
||||
@@index([parent_org_id])
|
||||
@@index([budget_id])
|
||||
@@index([budget_override_id])
|
||||
@@index([spend])
|
||||
}
|
||||
|
||||
|
|
@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable {
|
|||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String
|
||||
@@index([parent_org_id])
|
||||
@@index([parent_team_ids], type: Gin)
|
||||
@@index([budget_id])
|
||||
@@index([budget_override_id])
|
||||
@@index([spend])
|
||||
}
|
||||
|
||||
|
|
@ -2308,7 +2313,7 @@ enum AccountingDeliveryState {
|
|||
}
|
||||
|
||||
model LiteLLM_SpendLogDelivery {
|
||||
delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text"))
|
||||
delivery_id String @id @default(uuid())
|
||||
request_id String
|
||||
state AccountingDeliveryState @default(pending)
|
||||
accounting_payload Json
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[project]
|
||||
name = "litellm-proxy-extras"
|
||||
version = "0.4.116"
|
||||
version = "0.4.117"
|
||||
description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package."
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.9"
|
||||
|
|
@ -30,7 +30,7 @@ required-version = ">=0.10.9"
|
|||
module-root = ""
|
||||
|
||||
[tool.commitizen]
|
||||
version = "0.4.116"
|
||||
version = "0.4.117"
|
||||
version_files = [
|
||||
"pyproject.toml:^version",
|
||||
"../../pyproject.toml:litellm-proxy-extras==",
|
||||
|
|
|
|||
|
|
@ -206,6 +206,8 @@ from .auth_utils import (
|
|||
if TYPE_CHECKING:
|
||||
from opentelemetry.trace import Span as _Span
|
||||
|
||||
from litellm.proxy.spend_tracking.native_budgets import BudgetDecision
|
||||
|
||||
Span = _Span
|
||||
else:
|
||||
Span = Any
|
||||
|
|
@ -948,12 +950,11 @@ def _global_proxy_budget_check(global_proxy_spend: float | None, skip_budget_che
|
|||
and RouteChecks.runs_inference_checks(route=route)
|
||||
and route not in MODEL_LIST_ROUTES
|
||||
):
|
||||
if math.isfinite(litellm.max_budget) and global_proxy_spend > litellm.max_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=global_proxy_spend,
|
||||
max_budget=litellm.max_budget,
|
||||
entity_type=Litellm_EntityType.PROXY.value,
|
||||
)
|
||||
native: Final = _native_budget_decision(
|
||||
{"check": "proxy", "spend": global_proxy_spend, "max_budget": litellm.max_budget}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
_GUARDRAIL_MODIFICATION_KEYS: Final[tuple] = (
|
||||
|
|
@ -1488,14 +1489,16 @@ async def common_checks(
|
|||
user_info=call_info,
|
||||
)
|
||||
)
|
||||
if math.isfinite(user_budget) and user_spend >= user_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=user_spend,
|
||||
max_budget=user_budget,
|
||||
message=f"ExceededBudget: User={user_object.user_id} over budget. Spend={user_spend}, Budget={user_budget}",
|
||||
entity_type=Litellm_EntityType.USER.value,
|
||||
entity_id=user_object.user_id,
|
||||
)
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "user",
|
||||
"spend": user_spend,
|
||||
"max_budget": user_budget,
|
||||
"user_id": user_object.user_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
# Each scope reads a distinct counter key with no cross-scope ordering
|
||||
# dependency, so the per-scope Redis-first reads run concurrently instead
|
||||
|
|
@ -2032,14 +2035,16 @@ async def check_end_user_budget(
|
|||
max_budget=end_user_budget,
|
||||
fallback_authoritative=True,
|
||||
)
|
||||
if end_user_spend > end_user_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=end_user_spend,
|
||||
max_budget=end_user_budget,
|
||||
message=f"ExceededBudget: End User={end_user_obj.user_id} over budget. Spend={end_user_spend}, Budget={end_user_budget}",
|
||||
entity_type=Litellm_EntityType.END_USER.value,
|
||||
entity_id=end_user_obj.user_id,
|
||||
)
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "end_user",
|
||||
"spend": end_user_spend,
|
||||
"max_budget": end_user_budget,
|
||||
"end_user_id": end_user_obj.user_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
_check_end_user_budget: Final = check_end_user_budget
|
||||
|
|
@ -6957,13 +6962,6 @@ async def is_valid_fallback_model(
|
|||
return True
|
||||
|
||||
|
||||
# The shape abbreviate_api_key writes into LiteLLM_VerificationToken.key_name. The
|
||||
# last four characters are only barred from being whitespace or a control code,
|
||||
# because a custom key's can be anything else, punctuation and non-ASCII included;
|
||||
# a real key is at least MINIMUM_CUSTOM_KEY_LENGTH long, so it never fullmatches.
|
||||
_MASKED_KEY_NAME_RE: Final = re.compile(r"sk-\.\.\.(?:[^\s\x00-\x1f\x7f-\x9f]{4})?")
|
||||
|
||||
|
||||
def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
|
||||
"""
|
||||
Throttle an over-budget key instead of blocking it, when the key opted in
|
||||
|
|
@ -6982,6 +6980,17 @@ def _apply_budget_exceeded_throttle(valid_token: UserAPIKeyAuth) -> bool:
|
|||
return True
|
||||
|
||||
|
||||
def _native_budget_decision(check: Mapping[str, object]) -> "BudgetDecision":
|
||||
from litellm.proxy.spend_tracking.native_budgets import native_budget_decision
|
||||
|
||||
return native_budget_decision(check)
|
||||
|
||||
|
||||
def _budget_exceeded_throttle_pct(valid_token: UserAPIKeyAuth) -> float | None:
|
||||
pct: Final = budget_throttle_percentage()
|
||||
return pct if pct is not None and should_throttle_budget_exceeded(valid_token) else None
|
||||
|
||||
|
||||
async def virtual_key_max_budget_check(
|
||||
valid_token: UserAPIKeyAuth,
|
||||
proxy_logging_obj: ProxyLogging,
|
||||
|
|
@ -7041,25 +7050,21 @@ async def virtual_key_max_budget_check(
|
|||
# Defense-in-depth (GHSA-2rv4-xv66-fpjg): spend >= NaN is always False,
|
||||
# so a NaN max_budget would silently disable enforcement. Treat a
|
||||
# non-finite max_budget as "no configured limit" rather than as a bypass.
|
||||
if math.isfinite(valid_token.max_budget) and spend >= valid_token.max_budget:
|
||||
if _apply_budget_exceeded_throttle(valid_token):
|
||||
return
|
||||
# This message is returned to the caller, and key_name has no enforced
|
||||
# shape (a direct DB write bypasses abbreviate_api_key), so echo it only
|
||||
# when it still looks masked and fall back to the alias otherwise.
|
||||
key_label: Final = valid_token.key_alias or "key"
|
||||
key_descriptor: Final = (
|
||||
f"{key_label} ({valid_token.key_name})"
|
||||
if valid_token.key_name and _MASKED_KEY_NAME_RE.fullmatch(valid_token.key_name)
|
||||
else key_label
|
||||
)
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=spend,
|
||||
max_budget=valid_token.max_budget,
|
||||
message=f"Budget has been exceeded! Key={key_descriptor} Current cost: {spend}, Max budget: {valid_token.max_budget}",
|
||||
entity_type=Litellm_EntityType.KEY.value,
|
||||
entity_id=valid_token.token,
|
||||
)
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "key_max",
|
||||
"spend": spend,
|
||||
"max_budget": valid_token.max_budget,
|
||||
"token": valid_token.token,
|
||||
"key_alias": valid_token.key_alias,
|
||||
"key_name": valid_token.key_name,
|
||||
"throttle_pct": _budget_exceeded_throttle_pct(valid_token),
|
||||
}
|
||||
)
|
||||
if native.decision == "throttle" and _apply_budget_exceeded_throttle(valid_token):
|
||||
return
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
_virtual_key_max_budget_check: Final = virtual_key_max_budget_check
|
||||
|
|
@ -7096,17 +7101,17 @@ async def _virtual_key_multi_budget_check(
|
|||
window_duration=str(w["budget_duration"]),
|
||||
window_start=get_budget_window_start(w),
|
||||
)
|
||||
if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=window_spend,
|
||||
max_budget=w["max_budget"],
|
||||
message=(
|
||||
f"ExceededBudget: Key over {w['budget_duration']} budget. "
|
||||
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
|
||||
),
|
||||
entity_type=Litellm_EntityType.KEY.value,
|
||||
entity_id=valid_token.token,
|
||||
)
|
||||
native = _native_budget_decision(
|
||||
{
|
||||
"check": "key_window",
|
||||
"spend": window_spend,
|
||||
"max_budget": w["max_budget"],
|
||||
"budget_duration": str(w["budget_duration"]),
|
||||
"token": valid_token.token,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def virtual_key_soft_budget_check(
|
||||
|
|
@ -7119,7 +7124,10 @@ async def virtual_key_soft_budget_check(
|
|||
|
||||
"""
|
||||
|
||||
if valid_token.soft_budget and valid_token.spend >= valid_token.soft_budget:
|
||||
native: Final = _native_budget_decision(
|
||||
{"check": "key_soft", "spend": valid_token.spend, "soft_budget": valid_token.soft_budget}
|
||||
)
|
||||
if native.alerts:
|
||||
verbose_proxy_logger.debug(
|
||||
"Crossed Soft Budget for token %s, spend %s, soft_budget %s",
|
||||
valid_token.token,
|
||||
|
|
@ -7422,7 +7430,16 @@ async def _check_team_member_budget(
|
|||
max_budget=team_member_budget,
|
||||
)
|
||||
|
||||
if not math.isfinite(team_member_budget):
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "team_member",
|
||||
"spend": team_member_spend,
|
||||
"max_budget": team_member_budget,
|
||||
"user_id": valid_token.user_id,
|
||||
"team_id": team_object.team_id,
|
||||
}
|
||||
)
|
||||
if native.skipped:
|
||||
return
|
||||
|
||||
team_member_max_budget_alert_check(
|
||||
|
|
@ -7437,14 +7454,8 @@ async def _check_team_member_budget(
|
|||
max_budget=team_member_budget,
|
||||
)
|
||||
|
||||
if team_member_spend >= team_member_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=team_member_spend,
|
||||
max_budget=team_member_budget,
|
||||
message=f"Budget has been exceeded! User={valid_token.user_id} in Team={team_object.team_id} Current cost: {team_member_spend}, Max budget: {team_member_budget}",
|
||||
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
|
||||
entity_id=f"{valid_token.user_id}:{team_object.team_id}",
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def check_team_member_model_access(
|
||||
|
|
@ -7534,7 +7545,15 @@ async def team_max_budget_check(
|
|||
max_budget=team_object.max_budget,
|
||||
)
|
||||
|
||||
if math.isfinite(team_object.max_budget) and spend > team_object.max_budget:
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "team_max",
|
||||
"spend": spend,
|
||||
"max_budget": team_object.max_budget,
|
||||
"team_id": team_object.team_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
if valid_token:
|
||||
call_info: Final = CallInfo(
|
||||
token=valid_token.token,
|
||||
|
|
@ -7553,13 +7572,7 @@ async def team_max_budget_check(
|
|||
)
|
||||
)
|
||||
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=spend,
|
||||
max_budget=team_object.max_budget,
|
||||
message=f"Budget has been exceeded! Team={team_object.team_id} Current cost: {spend}, Max budget: {team_object.max_budget}",
|
||||
entity_type=Litellm_EntityType.TEAM.value,
|
||||
entity_id=team_object.team_id,
|
||||
)
|
||||
raise native.error()
|
||||
|
||||
|
||||
_team_max_budget_check: Final = team_max_budget_check
|
||||
|
|
@ -7571,29 +7584,26 @@ async def _user_multi_budget_check(user_object: LiteLLM_UserTable | None) -> Non
|
|||
from litellm.proxy.proxy_server import get_current_spend
|
||||
|
||||
for window in user_object.budget_limits:
|
||||
if (
|
||||
spend := await get_current_spend(
|
||||
counter_key=user_window_counter(user_object.user_id, window.budget_duration),
|
||||
fallback_spend=0.0,
|
||||
max_budget=window.max_budget,
|
||||
window_entity_type="User",
|
||||
window_entity_id=user_object.user_id,
|
||||
window_duration=window.budget_duration,
|
||||
window_start=get_budget_window_start(window),
|
||||
)
|
||||
) < window.max_budget:
|
||||
continue
|
||||
if math.isfinite(window.max_budget) and spend >= window.max_budget:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=spend,
|
||||
max_budget=window.max_budget,
|
||||
message=(
|
||||
f"ExceededBudget: User={user_object.user_id} over {window.budget_duration} budget. "
|
||||
f"Spend=${spend:.4f}, Limit=${window.max_budget:.2f}"
|
||||
),
|
||||
entity_type=Litellm_EntityType.USER.value,
|
||||
entity_id=user_object.user_id,
|
||||
)
|
||||
spend = await get_current_spend(
|
||||
counter_key=user_window_counter(user_object.user_id, window.budget_duration),
|
||||
fallback_spend=0.0,
|
||||
max_budget=window.max_budget,
|
||||
window_entity_type="User",
|
||||
window_entity_id=user_object.user_id,
|
||||
window_duration=window.budget_duration,
|
||||
window_start=get_budget_window_start(window),
|
||||
)
|
||||
native = _native_budget_decision(
|
||||
{
|
||||
"check": "user_window",
|
||||
"spend": spend,
|
||||
"max_budget": window.max_budget,
|
||||
"budget_duration": window.budget_duration,
|
||||
"user_id": user_object.user_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def _team_multi_budget_check(
|
||||
|
|
@ -7623,17 +7633,17 @@ async def _team_multi_budget_check(
|
|||
window_duration=str(w["budget_duration"]),
|
||||
window_start=get_budget_window_start(w),
|
||||
)
|
||||
if math.isfinite(w["max_budget"]) and window_spend >= w["max_budget"]:
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=window_spend,
|
||||
max_budget=w["max_budget"],
|
||||
message=(
|
||||
f"ExceededBudget: Team={team_object.team_id} over {w['budget_duration']} budget. "
|
||||
f"Spend=${window_spend:.4f}, Limit=${w['max_budget']:.2f}"
|
||||
),
|
||||
entity_type=Litellm_EntityType.TEAM.value,
|
||||
entity_id=team_object.team_id,
|
||||
)
|
||||
native = _native_budget_decision(
|
||||
{
|
||||
"check": "team_window",
|
||||
"spend": window_spend,
|
||||
"max_budget": w["max_budget"],
|
||||
"budget_duration": str(w["budget_duration"]),
|
||||
"team_id": team_object.team_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def _team_soft_budget_check(
|
||||
|
|
@ -7644,12 +7654,12 @@ async def _team_soft_budget_check(
|
|||
"""
|
||||
Triggers a budget alert if the team is over it's soft budget.
|
||||
"""
|
||||
if (
|
||||
team_object is not None
|
||||
and team_object.soft_budget is not None
|
||||
and team_object.spend is not None
|
||||
and team_object.spend >= team_object.soft_budget
|
||||
):
|
||||
if team_object is None:
|
||||
return
|
||||
native: Final = _native_budget_decision(
|
||||
{"check": "team_soft", "spend": team_object.spend, "soft_budget": team_object.soft_budget}
|
||||
)
|
||||
if native.alerts and team_object.spend is not None:
|
||||
verbose_proxy_logger.debug(
|
||||
"Crossed Soft Budget for team %s, spend %s, soft_budget %s",
|
||||
team_object.team_id,
|
||||
|
|
@ -7737,7 +7747,15 @@ async def _project_max_budget_check(
|
|||
max_budget=max_budget,
|
||||
)
|
||||
|
||||
if project_spend >= max_budget:
|
||||
native: Final = _native_budget_decision(
|
||||
{
|
||||
"check": "project_max",
|
||||
"spend": project_spend,
|
||||
"max_budget": max_budget,
|
||||
"project_id": project_object.project_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
if valid_token:
|
||||
call_info: Final = CallInfo(
|
||||
token=valid_token.token,
|
||||
|
|
@ -7756,13 +7774,7 @@ async def _project_max_budget_check(
|
|||
)
|
||||
)
|
||||
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=project_spend,
|
||||
max_budget=max_budget,
|
||||
message=f"Budget has been exceeded! Project={project_object.project_id} Current cost: {project_spend}, Max budget: {max_budget}",
|
||||
entity_type=Litellm_EntityType.PROJECT.value,
|
||||
entity_id=project_object.project_id,
|
||||
)
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def _project_soft_budget_check(
|
||||
|
|
@ -7782,7 +7794,10 @@ async def _project_soft_budget_check(
|
|||
if project_object.litellm_budget_table is not None:
|
||||
soft_budget = project_object.litellm_budget_table.soft_budget
|
||||
|
||||
if soft_budget is not None and project_object.spend is not None and project_object.spend >= soft_budget:
|
||||
native: Final = _native_budget_decision(
|
||||
{"check": "project_soft", "spend": project_object.spend, "soft_budget": soft_budget}
|
||||
)
|
||||
if native.alerts:
|
||||
verbose_proxy_logger.debug(
|
||||
"Crossed Soft Budget for project %s, spend %s, soft_budget %s",
|
||||
project_object.project_id,
|
||||
|
|
@ -7945,7 +7960,10 @@ async def organization_max_budget_check(
|
|||
)
|
||||
|
||||
# Check if organization spend exceeds max budget
|
||||
if math.isfinite(org_max_budget) and org_spend >= org_max_budget:
|
||||
native: Final = _native_budget_decision(
|
||||
{"check": "organization", "spend": org_spend, "max_budget": org_max_budget, "org_id": org_id}
|
||||
)
|
||||
if native.refused:
|
||||
# Trigger budget alert
|
||||
call_info: Final = CallInfo(
|
||||
token=valid_token.token,
|
||||
|
|
@ -7964,13 +7982,7 @@ async def organization_max_budget_check(
|
|||
)
|
||||
)
|
||||
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=org_spend,
|
||||
max_budget=org_max_budget,
|
||||
message=f"Budget has been exceeded! Organization={org_id} Current cost: {org_spend}, Max budget: {org_max_budget}",
|
||||
entity_type=Litellm_EntityType.ORGANIZATION.value,
|
||||
entity_id=org_id,
|
||||
)
|
||||
raise native.error()
|
||||
|
||||
|
||||
_organization_max_budget_check: Final = organization_max_budget_check
|
||||
|
|
@ -8034,15 +8046,16 @@ async def tag_max_budget_check_for_tags(
|
|||
max_budget=tag_object.litellm_budget_table.max_budget,
|
||||
fallback_authoritative=True,
|
||||
)
|
||||
if tag_spend <= tag_object.litellm_budget_table.max_budget:
|
||||
continue
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=tag_spend,
|
||||
max_budget=tag_object.litellm_budget_table.max_budget,
|
||||
message=f"Budget has been exceeded! Tag={tag_name} Current cost: {tag_spend}, Max budget: {tag_object.litellm_budget_table.max_budget}",
|
||||
entity_type=Litellm_EntityType.TAG.value,
|
||||
entity_id=tag_name,
|
||||
native = _native_budget_decision(
|
||||
{
|
||||
"check": "tag",
|
||||
"spend": tag_spend,
|
||||
"max_budget": tag_object.litellm_budget_table.max_budget,
|
||||
"tag": tag_name,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
async def _model_access_group_max_budget_check(
|
||||
|
|
@ -8089,15 +8102,11 @@ async def _model_access_group_max_budget_check(
|
|||
max_budget=budget.max_budget,
|
||||
fallback_authoritative=True,
|
||||
)
|
||||
if group_spend < budget.max_budget:
|
||||
continue
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=group_spend,
|
||||
max_budget=budget.max_budget,
|
||||
message=f"Budget has been exceeded! Model access group={group} Current cost: {group_spend}, Max budget: {budget.max_budget}",
|
||||
entity_type=Litellm_EntityType.MODEL_ACCESS_GROUP.value,
|
||||
entity_id=group,
|
||||
native = _native_budget_decision(
|
||||
{"check": "model_access_group", "spend": group_spend, "max_budget": budget.max_budget, "group": group}
|
||||
)
|
||||
if native.refused:
|
||||
raise native.error()
|
||||
|
||||
|
||||
def is_model_allowed_by_pattern(model: str, allowed_model_pattern: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -22,6 +22,7 @@ from litellm.constants import (
|
|||
INVALID_VIRTUAL_KEY_ERROR_MARKER,
|
||||
MINIMUM_CUSTOM_KEY_LENGTH,
|
||||
MODEL_BATCH_ENQUEUED_TOKEN_LIMIT_METADATA_KEY,
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
|
||||
STANDARD_CUSTOMER_ID_HEADERS,
|
||||
)
|
||||
from litellm.litellm_core_utils.clientside_credential_handler import (
|
||||
|
|
@ -1922,7 +1923,21 @@ class BatchLimitRequest(Protocol):
|
|||
def model_fields_set(self) -> Collection[str]: ...
|
||||
|
||||
|
||||
_ADMIN_ONLY_METADATA_KEYS: Final = (*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS, TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY)
|
||||
_ADMIN_ONLY_METADATA_KEYS: Final = (
|
||||
*ADMIN_ONLY_BATCH_LIMIT_METADATA_KEYS,
|
||||
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY,
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
|
||||
)
|
||||
_ADMIN_ONLY_METADATA_REASONS: Final[Mapping[str, str]] = MappingProxyType(
|
||||
{
|
||||
TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY: (
|
||||
"It lets the holder read and write those buckets with the proxy's AWS credentials."
|
||||
),
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY: (
|
||||
"It caps how far the holder borrows idle model capacity, so the holder cannot raise or remove it."
|
||||
),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def enforce_batch_limits_are_admin_only(
|
||||
|
|
@ -1955,10 +1970,8 @@ def enforce_batch_limits_are_admin_only(
|
|||
)
|
||||
if changed is None:
|
||||
return
|
||||
reason: Final = (
|
||||
"It lets the holder read and write those buckets with the proxy's AWS credentials."
|
||||
if changed == TEAM_S3_BUCKET_ALLOWLIST_METADATA_KEY
|
||||
else "It limits what the holder can do with batches, so the holder cannot raise it."
|
||||
reason: Final = _ADMIN_ONLY_METADATA_REASONS.get(
|
||||
changed, "It limits what the holder can do with batches, so the holder cannot raise it."
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=403,
|
||||
|
|
|
|||
|
|
@ -553,7 +553,7 @@ async def _register_organization(
|
|||
if isinstance(registered, Failure):
|
||||
if registered.code is ErrorCode.FREE_TIER_LIMIT:
|
||||
verbose_proxy_logger.warning(
|
||||
"CUSTOMER TRACKING OFF for organization %s because the free customer portal limit was reached. "
|
||||
"CUSTOMER TRACKING OFF for organization %s because the Customer Management free tier limit was reached. "
|
||||
"Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise",
|
||||
customer_org_alias,
|
||||
)
|
||||
|
|
@ -620,7 +620,7 @@ async def _register_member(
|
|||
if isinstance(registered, Failure):
|
||||
if registered.code is ErrorCode.FREE_TIER_LIMIT:
|
||||
verbose_proxy_logger.warning(
|
||||
"CUSTOMER TRACKING OFF for member %s because the free customer portal limit was reached. "
|
||||
"CUSTOMER TRACKING OFF for member %s because the Customer Management free tier limit was reached. "
|
||||
"Upgrade to Enterprise to remove the limits: https://www.litellm.ai/enterprise",
|
||||
customer_member_id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -198,6 +198,7 @@ from litellm.proxy.spend_tracking.committed_spend import (
|
|||
committed_spend_tracked,
|
||||
read_admission_committed_spend,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.native_budgets import native_budget_decision
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import (
|
||||
bind_admission_counter_keys,
|
||||
release_spend_counter_batch,
|
||||
|
|
@ -2322,12 +2323,22 @@ async def _small_preflight_failure_json(request: Request) -> Mapping[str, object
|
|||
}
|
||||
|
||||
|
||||
def _is_bodyless_get(request: Request) -> bool:
|
||||
return (
|
||||
request.scope.get("method") == "GET"
|
||||
and request.headers.get("content-length", "0") == "0"
|
||||
and request.headers.get("transfer-encoding") is None
|
||||
)
|
||||
|
||||
|
||||
async def _reject_from_headers_before_reading_body(
|
||||
request: Request, api_key: str | None, custom_litellm_key_header: str | None, route: str
|
||||
) -> None:
|
||||
"""Refuse a request its headers already condemn, a blocked key or an exhausted key limit, before reading its body.
|
||||
|
||||
Only a key already in the cache is considered, so a miss changes nothing and the full auth path still runs.
|
||||
The limit is only checked where a body is declared and the route is metered, because model listing, token
|
||||
counting and other GET discovery routes never reach the limiter and must keep answering an exhausted key.
|
||||
A retry flood of large payloads against a limited key used to buffer every payload just to answer 429.
|
||||
"""
|
||||
if not RouteChecks.is_llm_api_route(route=route):
|
||||
|
|
@ -2377,6 +2388,8 @@ async def _reject_from_headers_before_reading_body(
|
|||
dispatched_call_type=str(request_data["call_type"]) if request_data["call_type"] is not None else None,
|
||||
)
|
||||
return
|
||||
if RouteChecks.is_unbilled_route(route) or _is_bodyless_get(request):
|
||||
return
|
||||
limiter: Final = proxy_logging_obj.get_proxy_hook("parallel_request_limiter")
|
||||
if not isinstance(limiter, PROXY_MaxParallelRequestsHandler_v3):
|
||||
return
|
||||
|
|
@ -3119,7 +3132,7 @@ async def user_api_key_auth_builder(
|
|||
|
||||
for k, v in team_obj_dict.items():
|
||||
field_name = f"team_{k}"
|
||||
if field_name in valid_token.__fields__:
|
||||
if field_name in type(valid_token).model_fields:
|
||||
setattr(valid_token, field_name, v)
|
||||
except Exception as e:
|
||||
verbose_logger.debug(e) # moving from .warning to .debug as it spams logs when team missing from cache.
|
||||
|
|
@ -3420,7 +3433,16 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
fallback_spend=team_member_spend,
|
||||
max_budget=team_member_budget,
|
||||
)
|
||||
if team_member_spend >= team_member_budget:
|
||||
native: Final = native_budget_decision(
|
||||
{
|
||||
"check": "team_member_positive",
|
||||
"spend": team_member_spend,
|
||||
"max_budget": team_member_budget,
|
||||
"user_id": valid_token.user_id,
|
||||
"team_id": valid_token.team_id,
|
||||
}
|
||||
)
|
||||
if native.refused:
|
||||
# common_checks sends this alert on requests that get past here, so only the
|
||||
# request rejected here sends it from the builder.
|
||||
team_member_max_budget_alert_check(
|
||||
|
|
@ -3434,17 +3456,7 @@ async def validate_resolved_virtual_key( # noqa: C901 # Preserve ordering of e
|
|||
spend=team_member_spend,
|
||||
max_budget=team_member_budget,
|
||||
)
|
||||
_entity_id: Final = f"{valid_token.user_id}:{valid_token.team_id}"
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=team_member_spend,
|
||||
max_budget=team_member_budget,
|
||||
message=(
|
||||
f"Budget has been exceeded! TeamMember={_entity_id} "
|
||||
f"Current cost: {team_member_spend}, Max budget: {team_member_budget}"
|
||||
),
|
||||
entity_type=Litellm_EntityType.TEAM_MEMBER.value,
|
||||
entity_id=_entity_id,
|
||||
)
|
||||
raise native.error()
|
||||
|
||||
# Check 3. If token is expired
|
||||
if valid_token.expires is not None:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import json
|
|||
import logging
|
||||
import math
|
||||
import time
|
||||
import weakref
|
||||
from collections.abc import (
|
||||
AsyncGenerator,
|
||||
AsyncIterator,
|
||||
|
|
@ -4461,6 +4462,9 @@ class ProxyBaseLLMRequestProcessing:
|
|||
),
|
||||
)
|
||||
|
||||
def _request_fields(self) -> Mapping[str, object]:
|
||||
return self.data
|
||||
|
||||
def _arm_detached_stream_failure_hook(
|
||||
self,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
|
|
@ -4475,17 +4479,24 @@ class ProxyBaseLLMRequestProcessing:
|
|||
so the failed spend is still written and the budget reservation released;
|
||||
a replacement error the hook raises has no client left to reach.
|
||||
"""
|
||||
request_data: Final = self.data
|
||||
# The closure lives on the logging object until the logging object dies, so it must not
|
||||
# hold the logging object, or a request stays alive until a full GC pass (#51277)
|
||||
logging_obj_ref: Final = weakref.ref(logging_obj)
|
||||
request_data: Final = {
|
||||
key: value for key, value in self._request_fields().items() if key != "litellm_logging_obj"
|
||||
}
|
||||
route_type: Final = self.route_type
|
||||
|
||||
async def _on_detached_stream_failure(exc: Exception) -> None:
|
||||
if not logging_obj.claim_stream_failure_report():
|
||||
live_logging_obj: Final = logging_obj_ref()
|
||||
if live_logging_obj is None or not live_logging_obj.claim_stream_failure_report():
|
||||
return
|
||||
try:
|
||||
await proxy_logging_obj.post_call_failure_hook(
|
||||
user_api_key_dict=user_api_key_dict,
|
||||
original_exception=exc,
|
||||
request_data=request_data,
|
||||
dispatched_call_type=self.route_type,
|
||||
request_data={**request_data, "litellm_logging_obj": live_logging_obj},
|
||||
dispatched_call_type=route_type,
|
||||
)
|
||||
except HTTPException:
|
||||
return
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import json
|
|||
import math
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable, Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from enum import Enum
|
||||
from itertools import chain
|
||||
|
|
@ -36,6 +36,7 @@ from litellm.litellm_core_utils.duration_parser import (
|
|||
duration_in_seconds,
|
||||
get_annual_budget_window_start,
|
||||
is_annual_budget_duration,
|
||||
normalize_budget_window_start,
|
||||
)
|
||||
from litellm.proxy._types import (
|
||||
DB_RETRY_SAFE_ERROR_TYPES,
|
||||
|
|
@ -378,9 +379,15 @@ def _customer_reset_key(
|
|||
def _customer_ids_by_reset(
|
||||
rows: Sequence[_CustomerScopedRow], ids: Sequence[str], now: datetime, windows: Mapping[str, str]
|
||||
) -> Mapping[tuple[float | None, str | None], tuple[str, ...]]:
|
||||
keys: Final = tuple(_customer_reset_key(row, now, windows) for row in rows)
|
||||
capped: Final = tuple(
|
||||
(row_id, row) for row_id, row in zip(ids, rows) if _customer_effective_cap(row, now) is not None
|
||||
)
|
||||
keys: Final = tuple(_customer_reset_key(row, now, windows) for _, row in capped)
|
||||
return MappingProxyType(
|
||||
{key: tuple(row_id for row_id, row_key in zip(ids, keys) if row_key == key) for key in dict.fromkeys(keys)}
|
||||
{
|
||||
key: tuple(row_id for (row_id, _), row_key in zip(capped, keys) if row_key == key)
|
||||
for key in dict.fromkeys(keys)
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -506,6 +513,10 @@ class _BudgetCascade:
|
|||
customer_ids: Mapping[CustomerSpendEntity, tuple[str, ...]] = field(default_factory=lambda: MappingProxyType({}))
|
||||
customer_org_alias_keys: tuple[str, ...] = ()
|
||||
|
||||
@property
|
||||
def customer_budget_ids(self) -> tuple[str, ...]:
|
||||
return tuple(b.budget_id for b in self.budgets if b.max_budget is not None and b.budget_id in self.budget_ids)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _EndUserWalk:
|
||||
|
|
@ -1280,7 +1291,11 @@ class ResetBudgetJob:
|
|||
_queue_budget_linked_resets(uow.projects, cascade, extra=_SPENT_ROWS_WHERE)
|
||||
_queue_budget_linked_resets(uow.tags, cascade, extra=_SPENT_ROWS_WHERE)
|
||||
_queue_budget_linked_resets(uow.model_access_groups, cascade, extra=_SPENT_ROWS_WHERE)
|
||||
_queue_budget_linked_resets(uow.customer_organizations, cascade, extra=_SPENT_ROWS_WHERE)
|
||||
_queue_budget_linked_resets(
|
||||
uow.customer_organizations,
|
||||
replace(cascade, budget_ids=cascade.customer_budget_ids),
|
||||
extra=_SPENT_ROWS_WHERE,
|
||||
)
|
||||
for budget_id, budget_reset_at in cascade.budget_resets:
|
||||
uow.budgets.queue_window_advance(budget_id=budget_id, budget_reset_at=budget_reset_at)
|
||||
|
||||
|
|
@ -1932,7 +1947,7 @@ class ResetBudgetJob:
|
|||
entity_type=entity_type.value,
|
||||
entity_id=entity_id,
|
||||
window_duration=budget_duration,
|
||||
new_window_start=window_start,
|
||||
new_window_start=normalize_budget_window_start(window_start),
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 # the row is best effort; counter resets must still land
|
||||
verbose_proxy_logger.warning(
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import re
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, Final, TypeAlias, TypeVar, cast, overload
|
||||
from weakref import WeakValueDictionary
|
||||
|
||||
|
|
@ -48,6 +49,13 @@ def _ttl(kwargs: Mapping[str, object]) -> float | None:
|
|||
return _TTL.validate_python(kwargs.get("ttl"))
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _DecodedModel:
|
||||
payload: object
|
||||
model_type: type[BaseModel]
|
||||
model: BaseModel
|
||||
|
||||
|
||||
class CacheLease:
|
||||
__slots__ = ("__weakref__", "key", "shared")
|
||||
|
||||
|
|
@ -114,6 +122,7 @@ class UserApiKeyCache(DualCache):
|
|||
self._redis_connection: RedisCache | None = None
|
||||
self._shared: AuthCacheStore | None = None
|
||||
self._leases: Final[WeakValueDictionary[str, CacheLease]] = WeakValueDictionary()
|
||||
self._decoded_models: Final[dict[str, _DecodedModel]] = {} # mutable-ok: per-key memo of validated reads
|
||||
self.attach_redis_cache(redis_cache, default_redis_ttl=default_redis_ttl)
|
||||
|
||||
@property
|
||||
|
|
@ -175,6 +184,7 @@ class UserApiKeyCache(DualCache):
|
|||
|
||||
def set_local(self, key: str, value: object, ttl: float | None) -> None:
|
||||
"""A memory write alone, with ``default_in_memory_ttl`` when ``ttl`` is None."""
|
||||
self._drop_decoded((key,))
|
||||
if self._shared is not None:
|
||||
self._shared.set_local(key, value, ttl)
|
||||
return
|
||||
|
|
@ -199,6 +209,7 @@ class UserApiKeyCache(DualCache):
|
|||
"""Caches ``value`` unless the lease's key was evicted since it was taken; False when refused."""
|
||||
if self._leases.get(lease.key) is not lease:
|
||||
return False
|
||||
self._drop_decoded((lease.key,))
|
||||
shared: Final = self._shared
|
||||
if shared is None:
|
||||
await self.async_set_cache(key=lease.key, value=value, ttl=ttl)
|
||||
|
|
@ -214,9 +225,35 @@ class UserApiKeyCache(DualCache):
|
|||
return True
|
||||
|
||||
def _void_leases(self, keys: Sequence[str]) -> None:
|
||||
self._drop_decoded(keys)
|
||||
for key in keys:
|
||||
_ = self._leases.pop(key, None)
|
||||
|
||||
def _drop_decoded(self, keys: Sequence[str]) -> None:
|
||||
for key in keys:
|
||||
_ = self._decoded_models.pop(key, None)
|
||||
|
||||
def _decoded(self, cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
|
||||
"""In Python memory, a read of the same payload object a previous read validated returns a shallow copy of
|
||||
that model instead of validating again, so a caller that sets a field cannot change what the next reader
|
||||
gets. The shared store hands back a new payload on every read, so it is validated each time."""
|
||||
if model_type is None or cached is None:
|
||||
return cached
|
||||
memo: Final = self._decoded_models.get(key)
|
||||
if memo is not None and memo.payload is cached and memo.model_type is model_type:
|
||||
return memo.model.model_copy()
|
||||
decoded: Final = _validated(cached, key, model_type, method)
|
||||
if decoded is None or decoded is cached or self._shared is not None:
|
||||
return decoded
|
||||
_ = self._decoded_models.pop(key, None)
|
||||
self._decoded_models[key] = _DecodedModel(payload=cached, model_type=model_type, model=decoded)
|
||||
if len(self._decoded_models) > self._decoded_capacity():
|
||||
_ = self._decoded_models.pop(next(iter(self._decoded_models)))
|
||||
return decoded.model_copy()
|
||||
|
||||
def _decoded_capacity(self) -> int:
|
||||
return self.in_memory_cache.max_size_in_memory + self.key_object_cache.in_memory_cache.max_size_in_memory
|
||||
|
||||
def delete_local(self, key: str) -> None:
|
||||
"""Drops this worker's memory copy alone, as a cross-worker eviction broadcast does."""
|
||||
self._void_leases((key,))
|
||||
|
|
@ -271,7 +308,7 @@ class UserApiKeyCache(DualCache):
|
|||
if is_user_key_cache_key(key)
|
||||
else super().get_cache(key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs)
|
||||
)
|
||||
return _decoded(cached, key, model_type, "get_cache")
|
||||
return self._decoded(cached, key, model_type, "get_cache")
|
||||
|
||||
@overload
|
||||
async def async_get_cache(
|
||||
|
|
@ -316,7 +353,7 @@ class UserApiKeyCache(DualCache):
|
|||
key=key, parent_otel_span=parent_otel_span, local_only=local_only, **kwargs
|
||||
)
|
||||
)
|
||||
return _decoded(cached, key, model_type, "async_get_cache")
|
||||
return self._decoded(cached, key, model_type, "async_get_cache")
|
||||
|
||||
@staticmethod
|
||||
def _get_shared_sync(shared: AuthCacheStore, key: str, local_only: bool) -> object:
|
||||
|
|
@ -337,6 +374,8 @@ class UserApiKeyCache(DualCache):
|
|||
def set_cache(self, key: str | None, value: CacheValue, local_only: bool = False, **kwargs: object):
|
||||
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
|
||||
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
|
||||
if key is not None:
|
||||
self._drop_decoded((key,))
|
||||
shared: Final = self._shared
|
||||
if shared is not None:
|
||||
if key is None:
|
||||
|
|
@ -358,6 +397,8 @@ class UserApiKeyCache(DualCache):
|
|||
else, or with options the pipeline does not carry, it goes to Redis directly as before."""
|
||||
model_type: Final = cast(type[BaseModel] | None, kwargs.pop("model_type", None))
|
||||
payload: Final[object] = CacheCodec.serialize(value, model_type=model_type)
|
||||
if key is not None:
|
||||
self._drop_decoded((key,))
|
||||
ttl: Final = kwargs.get("ttl")
|
||||
pipelined: Final = (
|
||||
key is not None
|
||||
|
|
@ -443,6 +484,7 @@ class UserApiKeyCache(DualCache):
|
|||
|
||||
def flush_cache(self) -> None:
|
||||
self._leases.clear()
|
||||
self._decoded_models.clear()
|
||||
self.budget_windows_absent_cache.flush_cache()
|
||||
if self._shared is not None:
|
||||
self._shared.flush_local()
|
||||
|
|
@ -458,6 +500,7 @@ class UserApiKeyCache(DualCache):
|
|||
``model_type``: ``BaseModel`` values become JSON-safe dicts; dicts/scalars unchanged.
|
||||
"""
|
||||
normalized: Final = tuple((key, CacheCodec.serialize(value, model_type=None)) for key, value in cache_list)
|
||||
self._drop_decoded(tuple(key for key, _ in normalized))
|
||||
shared: Final = self._shared
|
||||
if shared is not None:
|
||||
await self._set_many_shared(shared, normalized, local_only=local_only, ttl=_ttl(kwargs))
|
||||
|
|
@ -519,9 +562,7 @@ class UserApiKeyCache(DualCache):
|
|||
)
|
||||
|
||||
|
||||
def _decoded(cached: object, key: str, model_type: type[BaseModel] | None, method: str) -> object:
|
||||
if model_type is None or cached is None:
|
||||
return cached
|
||||
def _validated(cached: object, key: str, model_type: type[BaseModel], method: str) -> BaseModel | None:
|
||||
decoded: Final = CacheCodec.deserialize(cached, model_type=model_type)
|
||||
if decoded is None:
|
||||
verbose_proxy_logger.error(
|
||||
|
|
|
|||
56
litellm/proxy/db/connection_budget.py
Normal file
56
litellm/proxy/db/connection_budget.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
"""Split each Postgres pool's connection budget between Prisma and the Rust core's SQLx pool.
|
||||
|
||||
Migration scaffolding. Prisma and SQLx share a database only while the Python gateway still runs Prisma
|
||||
queries next to ported ones. These invariants hold until Prisma is gone:
|
||||
|
||||
- Operators set one `connection_limit` per pool (`database_connection_pool_limit`, a URL pin or
|
||||
`database_extra_connection_params`). Nothing an operator sets names Prisma, SQLx or Rust. The Rust share
|
||||
is carved out of that limit, never added on top, so a Rust pool does not grow the connection budget.
|
||||
- The Rust share is the hard-coded `RUST_RESERVED_CONNECTIONS`. It is 0 until a ported query runs on the
|
||||
native pool, so Prisma keeps its whole limit, untouched, and no native pool opens.
|
||||
- Only a positive integer limit is split. Any other value goes to Prisma as it is, which judges it the same
|
||||
way it did before the split existed.
|
||||
- The split is Python's. It is about Prisma and Prisma lives only in the Python gateway, so this module is
|
||||
deleted with Prisma.
|
||||
- The Rust core takes a plain `max_connections` and never learns about the split. That is already its final
|
||||
interface, since the Rust gateway will pass `database_connection_pool_limit` straight to SQLx.
|
||||
"""
|
||||
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
RUST_RESERVED_CONNECTIONS: Final = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ConnectionBudget:
|
||||
total: int
|
||||
python: int
|
||||
rust: int
|
||||
|
||||
@classmethod
|
||||
def split(cls, total: int, rust_reserved: int = RUST_RESERVED_CONNECTIONS) -> "ConnectionBudget":
|
||||
"""Prisma keeps at least one connection, so a total at or below the reserve ends up over it."""
|
||||
return cls(total=total, python=max(1, total - rust_reserved), rust=rust_reserved)
|
||||
|
||||
@classmethod
|
||||
def of(
|
||||
cls, connection_limit: str | float, rust_reserved: int = RUST_RESERVED_CONNECTIONS
|
||||
) -> "ConnectionBudget | None":
|
||||
"""The split of a positive integer limit, or None for any other value Prisma is left to judge."""
|
||||
digits: Final = str(connection_limit)
|
||||
if not (digits.isascii() and digits.isdigit()) or int(digits) == 0:
|
||||
return None
|
||||
return cls.split(int(digits), rust_reserved)
|
||||
|
||||
@property
|
||||
def prisma_limit(self) -> int | None:
|
||||
"""Prisma's share once the Rust pool takes some of the total, or None to leave Prisma's limit untouched."""
|
||||
return None if self.rust == 0 else self.python
|
||||
|
||||
def prisma_params(self, params: Mapping[str, str | int | float]) -> Mapping[str, str | int | float]:
|
||||
"""``params`` with Prisma's share as its ``connection_limit``, untouched while the Rust pool takes none."""
|
||||
limit: Final = self.prisma_limit
|
||||
return params if limit is None else MappingProxyType({**params, "connection_limit": limit})
|
||||
|
|
@ -32,7 +32,7 @@ from litellm.proxy.db.log_db_metrics import ( # noqa: F401 # legacy module exp
|
|||
_T = TypeVar("_T")
|
||||
|
||||
|
||||
def _service_logging() -> ServiceLogging | None:
|
||||
def proxy_service_logging() -> ServiceLogging | None:
|
||||
try:
|
||||
from litellm.proxy.proxy_server import proxy_logging_obj
|
||||
except ImportError:
|
||||
|
|
@ -40,7 +40,9 @@ def _service_logging() -> ServiceLogging | None:
|
|||
return proxy_logging_obj.service_logging_obj
|
||||
|
||||
|
||||
def _event_metadata(table: str | None, operation: str | None) -> dict[str, str]:
|
||||
def db_event_metadata(
|
||||
table: str | None, operation: str | None
|
||||
) -> dict[str, str]: # mutable-ok: ServiceLogging hooks take event_metadata as a dict
|
||||
pairs: Final = (("table_name", table), ("db_operation", operation))
|
||||
return {key: value for key, value in pairs if value is not None}
|
||||
|
||||
|
|
@ -73,9 +75,9 @@ async def db_span(call_type: str, table: str | None, operation: str | None = Non
|
|||
if db_io_claimed():
|
||||
yield
|
||||
return
|
||||
service_logging: Final = _service_logging()
|
||||
service_logging: Final = proxy_service_logging()
|
||||
start_time: Final = datetime.now()
|
||||
event_metadata: Final = _event_metadata(table, operation)
|
||||
event_metadata: Final = db_event_metadata(table, operation)
|
||||
with claim_db_io() as witness:
|
||||
try:
|
||||
yield
|
||||
|
|
|
|||
|
|
@ -254,12 +254,28 @@ def translate_libpq_ssl_params(url: str, resolve_root_cert: RootCertResolver = p
|
|||
return urllib.parse.urlunsplit(parsed._replace(query=query))
|
||||
|
||||
|
||||
def postgres_connection_budget_message(writer_limit: str, reader_limit: str | None, num_workers: str) -> str:
|
||||
PRISMA_APPLICATION_NAME: Final = "litellm-prisma"
|
||||
NATIVE_APPLICATION_NAME: Final = "litellm-native"
|
||||
|
||||
|
||||
def prisma_default_params(
|
||||
lifetime_params: Mapping[str, str | int | float], native_connections: int
|
||||
) -> Mapping[str, str | int | float]:
|
||||
"""Prisma's URL defaults, tagged ``PRISMA_APPLICATION_NAME`` only once a native pool shares the database."""
|
||||
if native_connections == 0:
|
||||
return lifetime_params
|
||||
return MappingProxyType({**lifetime_params, "application_name": PRISMA_APPLICATION_NAME})
|
||||
|
||||
|
||||
def postgres_connection_budget_message(
|
||||
writer_limit: str, rust_writer_limit: int, reader_limit: str | None, rust_reader_limit: int, num_workers: str
|
||||
) -> str:
|
||||
"""The startup line that states this pod's worst-case Postgres connection demand.
|
||||
|
||||
Prisma's ``connection_limit`` is per query engine, and every uvicorn worker
|
||||
owns one engine per configured database (writer, plus the reader when
|
||||
``DATABASE_URL_READ_REPLICA`` is set). The limits are read off the final URLs,
|
||||
``DATABASE_URL_READ_REPLICA`` is set) plus a Rust pool beside each whose
|
||||
share (``connection_budget``) is above 0. The limits are read off the final URLs,
|
||||
so a reader that pins its own ``connection_limit`` or an operator override in
|
||||
``database_extra_connection_params`` is counted at its real value. The
|
||||
server-side cap is shared by every pod, so the number an operator has to keep
|
||||
|
|
@ -272,16 +288,21 @@ def postgres_connection_budget_message(writer_limit: str, reader_limit: str | No
|
|||
reader: Final = int(reader_limit) if reader_limit is not None else 0
|
||||
except ValueError:
|
||||
return (
|
||||
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + reader "
|
||||
f"connection_limit) (workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
|
||||
"LiteLLM Proxy: Postgres connection budget per pod = workers x (writer connection_limit + "
|
||||
f"{'rust writer pool + ' if rust_writer_limit else ''}reader connection_limit"
|
||||
f"{' + rust reader pool' if rust_reader_limit else ''}) "
|
||||
f"(workers={num_workers!r}, writer={writer_limit!r}, reader={reader_limit!r}); "
|
||||
"keep pods x that figure under max_connections minus superuser_reserved_connections"
|
||||
)
|
||||
engines: Final = (
|
||||
f"(writer connection_limit {writer} + reader connection_limit {reader})"
|
||||
if reader_limit is not None
|
||||
else f"writer connection_limit {writer}"
|
||||
rust_reader: Final = rust_reader_limit if reader_limit is not None else 0
|
||||
terms: Final = (
|
||||
f"writer connection_limit {writer}",
|
||||
*((f"rust writer pool {rust_writer_limit}",) if rust_writer_limit else ()),
|
||||
*((f"reader connection_limit {reader}",) if reader_limit is not None else ()),
|
||||
*((f"rust reader pool {rust_reader}",) if rust_reader else ()),
|
||||
)
|
||||
per_pod: Final = workers * (writer + reader)
|
||||
engines: Final = terms[0] if len(terms) == 1 else f"({' + '.join(terms)})"
|
||||
per_pod: Final = workers * (writer + rust_writer_limit + reader + rust_reader)
|
||||
return (
|
||||
f"LiteLLM Proxy: Postgres connection budget per pod = {workers} worker(s) x {engines} = up to {per_pod} "
|
||||
"connections; keep pods x that figure under max_connections minus superuser_reserved_connections"
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from litellm.proxy._types import (
|
|||
ProxyException,
|
||||
)
|
||||
from litellm.proxy.db.db_lookup_gate import DBLookupDeadlineExceeded
|
||||
from litellm.rust_bridge._native import NativeDatabaseFailure
|
||||
from litellm.secret_managers.main import str_to_bool
|
||||
|
||||
# Bounds the __cause__/__context__ walk in find_database_service_unavailable_error_in_chain.
|
||||
|
|
@ -175,6 +176,8 @@ class PrismaDBExceptionHandler:
|
|||
"""
|
||||
import prisma.engine.errors
|
||||
|
||||
if isinstance(e, NativeDatabaseFailure):
|
||||
return e.unavailable
|
||||
if isinstance(e, (*DB_CONNECTION_ERROR_TYPES, DBLookupDeadlineExceeded)):
|
||||
return True
|
||||
if isinstance(e, _exception_types(prisma.engine.errors.EngineConnectionError)):
|
||||
|
|
@ -769,6 +772,8 @@ _DATA_REJECTED_SQLSTATE_CLASSES: Final = frozenset({"22", "23"})
|
|||
|
||||
def db_write_failure_is_requeue_safe(e: Exception) -> bool:
|
||||
"""Whether a failed write provably never applied, so its rows can be sent again without double counting."""
|
||||
if isinstance(e, NativeDatabaseFailure):
|
||||
return e.requeue_safe
|
||||
if isinstance(e, DB_CONNECTION_ERROR_TYPES):
|
||||
return isinstance(e, DB_RETRY_SAFE_ERROR_TYPES)
|
||||
sqlstate: Final = PrismaDBExceptionHandler.postgres_sqlstate(e)
|
||||
|
|
|
|||
369
litellm/proxy/db/native_database.py
Normal file
369
litellm/proxy/db/native_database.py
Normal file
|
|
@ -0,0 +1,369 @@
|
|||
"""Settings for the native (SQLx) pool, resolved from the same URL Prisma connects with.
|
||||
|
||||
``DATABASE_URL`` is final once ``db_url_settings`` and the CLI have assembled it: token
|
||||
minted, libpq TLS translated to Prisma's dialect, timeouts in ``options``, pool params
|
||||
added. Reading that URL keeps one resolver for both pools instead of a second pass over
|
||||
the ``DATABASE_*`` env vars.
|
||||
|
||||
Credentials follow the host. Each process runs one token refresh loop, owned by whatever
|
||||
hosts it, and feeds every client in that process: here the worker's loop re-mints Prisma's
|
||||
URL and pushes the same kind of token into the Rust pool through ``set_writer_password``.
|
||||
The Rust core only accepts a password and never mints one, so a Rust gateway process
|
||||
brings its own loop. Two processes minting at once is fine, since IAM and Entra tokens
|
||||
are independent. What must not exist twice is the logic that resolves region, endpoint
|
||||
and user, so when the Rust gateway needs it, it moves into the Rust core and this
|
||||
loop calls it.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import shlex
|
||||
import struct
|
||||
import urllib.parse
|
||||
from collections.abc import Awaitable, Callable, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final, Protocol
|
||||
|
||||
from cryptography.hazmat.primitives.serialization import Encoding, NoEncryption, PrivateFormat, pkcs12
|
||||
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm._service_logger import ServiceTypes
|
||||
from litellm.proxy.db.db_span import db_event_metadata
|
||||
from litellm.proxy.db.db_url_settings import NATIVE_APPLICATION_NAME, PRISMA_APPLICATION_NAME
|
||||
from litellm.proxy.db.pgbouncer import database_url_is_pooled, fold_libpq_option_tokens, libpq_option_setting
|
||||
from litellm.proxy.db.token_auth import (
|
||||
DEFAULT_POSTGRES_PORT,
|
||||
DatabaseTokenAuth,
|
||||
IAMEndpoint,
|
||||
mint_database_token,
|
||||
parse_database_token_expiration,
|
||||
parse_iam_endpoint_from_url,
|
||||
resolve_database_token_auth,
|
||||
)
|
||||
from litellm.rust_bridge.database import (
|
||||
NativeClientIdentity,
|
||||
NativeDatabaseEndpoint,
|
||||
NativePoolLimits,
|
||||
SslMode,
|
||||
)
|
||||
|
||||
TOKEN_REFRESH_BUFFER_SECONDS: Final = 180.0
|
||||
TOKEN_FALLBACK_REFRESH_SECONDS: Final = 600.0
|
||||
TOKEN_RETRY_SECONDS: Final = 30.0
|
||||
PRISMA_DEFAULT_SCHEMA: Final = "public"
|
||||
PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS: Final = 10.0
|
||||
PRISMA_DEFAULT_STATEMENT_CACHE_SIZE: Final = 100
|
||||
TLS_PROBE_TIMEOUT_SECONDS: Final = 5.0
|
||||
_POSTGRES_SSL_REQUEST: Final = struct.pack("!ii", 8, 80877103)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class NativeEndpointSettings:
|
||||
endpoint: NativeDatabaseEndpoint
|
||||
ignored_options: tuple[str, ...]
|
||||
|
||||
|
||||
def _params(url: str) -> Mapping[str, str]:
|
||||
return MappingProxyType(dict(urllib.parse.parse_qsl(urllib.parse.urlsplit(url).query)))
|
||||
|
||||
|
||||
def needs_tls_probe(url: str) -> bool:
|
||||
"""Prisma's ``sslmode=prefer`` with ``sslaccept=strict`` verifies TLS when the server offers it
|
||||
and falls back to plaintext when it does not; SQLx has no verified-or-plaintext mode, so the
|
||||
caller asks the server once and pins the answer."""
|
||||
params: Final = _params(url)
|
||||
return params.get("sslmode", "prefer") == "prefer" and params.get("sslaccept") == "strict" and "host" not in params
|
||||
|
||||
|
||||
async def server_offers_tls(url: str, timeout: float = TLS_PROBE_TIMEOUT_SECONDS) -> bool:
|
||||
"""Whether the server answers a Postgres ``SSLRequest`` with ``S``; an unreachable server counts as offering it."""
|
||||
parsed: Final = urllib.parse.urlsplit(url)
|
||||
try:
|
||||
reader, writer = await asyncio.wait_for(
|
||||
asyncio.open_connection(parsed.hostname, parsed.port or int(DEFAULT_POSTGRES_PORT)), timeout
|
||||
)
|
||||
except (OSError, asyncio.TimeoutError):
|
||||
return True
|
||||
try:
|
||||
writer.write(_POSTGRES_SSL_REQUEST)
|
||||
await writer.drain()
|
||||
answer: Final = await asyncio.wait_for(reader.read(1), timeout)
|
||||
except (OSError, asyncio.TimeoutError):
|
||||
return True
|
||||
finally:
|
||||
writer.close()
|
||||
return answer != b"N"
|
||||
|
||||
|
||||
def _tls(params: Mapping[str, str], tls_offered: bool) -> tuple[SslMode, str | None]:
|
||||
sslmode: Final = params.get("sslmode", "prefer")
|
||||
strict: Final = params.get("sslaccept") == "strict"
|
||||
if sslmode == "disable" or (sslmode == "prefer" and strict and not tls_offered):
|
||||
return "disable", None
|
||||
if strict:
|
||||
return "verify-full", params.get("sslcert")
|
||||
return ("require" if sslmode == "require" else "prefer"), None
|
||||
|
||||
|
||||
def _client_identity(
|
||||
params: Mapping[str, str], ssl_mode: SslMode, read_file: Callable[[Path], bytes]
|
||||
) -> NativeClientIdentity | None:
|
||||
identity_path: Final = params.get("sslidentity")
|
||||
if identity_path is None or ssl_mode == "disable":
|
||||
return None
|
||||
password: Final = params.get("sslpassword")
|
||||
key, cert, chain = pkcs12.load_key_and_certificates(
|
||||
read_file(Path(identity_path)), None if password is None else password.encode()
|
||||
)
|
||||
if key is None or cert is None:
|
||||
raise ValueError(f"sslidentity {identity_path!r} must hold a client certificate and its private key")
|
||||
return NativeClientIdentity(
|
||||
cert_pem=b"".join(c.public_bytes(Encoding.PEM) for c in (cert, *chain)).decode(),
|
||||
key_pem=key.private_bytes(Encoding.PEM, PrivateFormat.PKCS8, NoEncryption()).decode(),
|
||||
)
|
||||
|
||||
|
||||
def _session_options(params: Mapping[str, str]) -> tuple[tuple[tuple[str, str], ...], tuple[str, ...]]:
|
||||
raw: Final = params.get("options", "")
|
||||
try:
|
||||
tokens: Final[Sequence[str]] = fold_libpq_option_tokens(tuple(shlex.split(raw)))
|
||||
except ValueError:
|
||||
return (), (raw,)
|
||||
readable: Final = tuple((token, libpq_option_setting(token)) for token in tokens)
|
||||
schema: Final = params.get("schema", PRISMA_DEFAULT_SCHEMA)
|
||||
search_path: Final = () if schema == PRISMA_DEFAULT_SCHEMA else (("search_path", f'"{schema}"'),)
|
||||
settings: Final = tuple(
|
||||
(name, value) for name, _, value in (setting.partition("=") for _, setting in readable if setting is not None)
|
||||
)
|
||||
return (*search_path, *settings), tuple(token for token, setting in readable if setting is None)
|
||||
|
||||
|
||||
def _seconds(params: Mapping[str, str], key: str, default: float | None = None) -> float | None:
|
||||
value: Final = params.get(key)
|
||||
if value is None:
|
||||
return default
|
||||
return None if float(value) == 0 else float(value)
|
||||
|
||||
|
||||
def _statement_cache_capacity(params: Mapping[str, str]) -> int:
|
||||
if params.get("pgbouncer") == "true":
|
||||
return 0
|
||||
return int(params.get("statement_cache_size", PRISMA_DEFAULT_STATEMENT_CACHE_SIZE))
|
||||
|
||||
|
||||
def _application_name(params: Mapping[str, str]) -> str:
|
||||
configured: Final = params.get("application_name", PRISMA_APPLICATION_NAME)
|
||||
return NATIVE_APPLICATION_NAME if configured == PRISMA_APPLICATION_NAME else configured
|
||||
|
||||
|
||||
def native_endpoint(
|
||||
url: str,
|
||||
*,
|
||||
tls_offered: bool,
|
||||
max_connections: int,
|
||||
read_file: Callable[[Path], bytes] = Path.read_bytes,
|
||||
) -> NativeEndpointSettings:
|
||||
"""The native pool's endpoint for the Prisma URL ``url``, read the way Prisma reads it.
|
||||
|
||||
``host`` is Prisma's Unix socket directory. The pool takes ``max_connections``, the
|
||||
Rust share of the budget (``connection_budget``), and tags its connections
|
||||
``NATIVE_APPLICATION_NAME``. An unset ``pool_timeout`` takes Prisma's default and
|
||||
0 waits forever.
|
||||
``sslidentity`` / ``sslpassword`` are a PKCS#12 client certificate, handed over as PEM.
|
||||
``options`` tokens that are not ``-c name=value`` settings cannot be sent by SQLx and
|
||||
come back in ``ignored_options`` instead of failing boot.
|
||||
"""
|
||||
parsed: Final = urllib.parse.urlsplit(url)
|
||||
params: Final = _params(url)
|
||||
socket: Final = params.get("host")
|
||||
if not (parsed.hostname or socket) or not parsed.username:
|
||||
raise ValueError("DATABASE_URL must name a host and a user for the native pool")
|
||||
ssl_mode, ssl_root_cert = _tls(params, tls_offered)
|
||||
options, ignored = _session_options(params)
|
||||
return NativeEndpointSettings(
|
||||
endpoint=NativeDatabaseEndpoint(
|
||||
host=parsed.hostname or "localhost",
|
||||
port=parsed.port or int(DEFAULT_POSTGRES_PORT),
|
||||
socket=socket,
|
||||
username=urllib.parse.unquote(parsed.username),
|
||||
password=urllib.parse.unquote(parsed.password) if parsed.password else None,
|
||||
database=urllib.parse.unquote(parsed.path.lstrip("/")) or "postgres",
|
||||
ssl_mode=ssl_mode,
|
||||
ssl_root_cert=ssl_root_cert,
|
||||
ssl_client=_client_identity(params, ssl_mode, read_file),
|
||||
application_name=_application_name(params),
|
||||
options=options,
|
||||
statement_cache_capacity=_statement_cache_capacity(params),
|
||||
pool=NativePoolLimits(
|
||||
max_connections=max_connections,
|
||||
acquire_timeout_seconds=_seconds(params, "pool_timeout", PRISMA_DEFAULT_POOL_TIMEOUT_SECONDS),
|
||||
idle_timeout_seconds=_seconds(params, "max_idle_connection_lifetime"),
|
||||
socket_timeout_seconds=_seconds(params, "socket_timeout"),
|
||||
),
|
||||
),
|
||||
ignored_options=ignored,
|
||||
)
|
||||
|
||||
|
||||
async def resolve_native_endpoint(url: str, max_connections: int) -> NativeEndpointSettings:
|
||||
tls_offered: Final = await server_offers_tls(url) if needs_tls_probe(url) else True
|
||||
return native_endpoint(url, tls_offered=tls_offered, max_connections=max_connections)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TokenRotation:
|
||||
auth: DatabaseTokenAuth
|
||||
endpoint: IAMEndpoint
|
||||
mint_token: Callable[[DatabaseTokenAuth, IAMEndpoint], str] = mint_database_token
|
||||
|
||||
def mint(self) -> str:
|
||||
return urllib.parse.unquote(self.mint_token(self.auth, self.endpoint))
|
||||
|
||||
def seconds_until_refresh(self, token: str, now: datetime) -> float:
|
||||
expires_at: Final = parse_database_token_expiration(self.auth, token)
|
||||
if expires_at is None:
|
||||
return TOKEN_FALLBACK_REFRESH_SECONDS
|
||||
return max(TOKEN_RETRY_SECONDS, (expires_at - now).total_seconds() - TOKEN_REFRESH_BUFFER_SECONDS)
|
||||
|
||||
|
||||
def writer_token_rotation(url: str) -> TokenRotation | None:
|
||||
"""The token rotation the writer needs, or None for a static password.
|
||||
|
||||
A URL pointed at the in-container PgBouncer carries PgBouncer's own credential,
|
||||
and PgBouncer renews the upstream token itself.
|
||||
"""
|
||||
if database_url_is_pooled():
|
||||
return None
|
||||
auth: Final = resolve_database_token_auth()
|
||||
if auth is None:
|
||||
return None
|
||||
return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url))
|
||||
|
||||
|
||||
def reader_token_rotation(url: str) -> TokenRotation | None:
|
||||
"""The token rotation the read replica needs, or None for a static password.
|
||||
|
||||
The reader never goes through the in-container PgBouncer, so only the token toggles
|
||||
decide, with the reader's own region, the same as its Prisma client.
|
||||
"""
|
||||
auth: Final = resolve_database_token_auth(read_replica=True)
|
||||
if auth is None:
|
||||
return None
|
||||
return TokenRotation(auth=auth, endpoint=parse_iam_endpoint_from_url(url))
|
||||
|
||||
|
||||
def _utcnow() -> datetime:
|
||||
return datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
|
||||
|
||||
async def _refresh(install: Callable[[str], None], rotation: TokenRotation, now: Callable[[], datetime]) -> float:
|
||||
try:
|
||||
token: Final = await asyncio.to_thread(rotation.mint)
|
||||
except Exception as error: # noqa: BLE001 # boto3 and azure-identity raise many types; any failure means retry
|
||||
verbose_proxy_logger.error(
|
||||
"Native database pool keeps its current %s (%r); retrying in %.0fs.",
|
||||
rotation.auth.label,
|
||||
error,
|
||||
TOKEN_RETRY_SECONDS,
|
||||
)
|
||||
return TOKEN_RETRY_SECONDS
|
||||
install(token)
|
||||
return rotation.seconds_until_refresh(token, now())
|
||||
|
||||
|
||||
async def keep_token_fresh(
|
||||
install: Callable[[str], None],
|
||||
rotation: TokenRotation,
|
||||
current_token: str,
|
||||
now: Callable[[], datetime] = _utcnow,
|
||||
sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
|
||||
) -> None:
|
||||
"""Hand the native pool a fresh token before the current one expires.
|
||||
|
||||
Existing connections stay authenticated; only connections opened after a refresh
|
||||
use the new token, the same as a re-minted Prisma URL.
|
||||
"""
|
||||
delay = rotation.seconds_until_refresh(current_token, now()) # rebind-ok: each refresh schedules the next
|
||||
while True:
|
||||
await sleep(delay)
|
||||
delay = await _refresh(install, rotation, now)
|
||||
|
||||
|
||||
class DatabaseServiceHooks(Protocol):
|
||||
async def async_service_success_hook(
|
||||
self,
|
||||
service: ServiceTypes,
|
||||
call_type: str,
|
||||
duration: float,
|
||||
parent_otel_span: None = None,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature
|
||||
) -> None: ...
|
||||
|
||||
async def async_service_failure_hook(
|
||||
self,
|
||||
service: ServiceTypes,
|
||||
duration: float,
|
||||
error: Exception,
|
||||
call_type: str,
|
||||
parent_otel_span: None = None,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
event_metadata: dict[str, str] | None = None, # mutable-ok: matches the ServiceLogging hook signature
|
||||
) -> None: ...
|
||||
|
||||
|
||||
class NativeDatabaseError(Exception):
|
||||
"""A native (SQLx) query failure, reported to service logging like a Prisma one."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ServiceLoggingEvents:
|
||||
"""Turns native round trips into the ``ServiceTypes.DB`` events ``db_span`` emits.
|
||||
|
||||
Rust calls ``record`` from its own threads once per round trip, so the hook is
|
||||
handed to the proxy's event loop rather than awaited here.
|
||||
"""
|
||||
|
||||
loop: asyncio.AbstractEventLoop
|
||||
service_logging: Callable[[], DatabaseServiceHooks | None]
|
||||
|
||||
def record(
|
||||
self,
|
||||
call_type: str,
|
||||
table: str | None,
|
||||
operation: str | None,
|
||||
started_at: float,
|
||||
duration_seconds: float,
|
||||
failure: str | None,
|
||||
) -> None:
|
||||
service_logging: Final = self.service_logging()
|
||||
if service_logging is None:
|
||||
return
|
||||
start_time: Final = datetime.fromtimestamp(started_at, tz=timezone.utc).astimezone().replace(tzinfo=None)
|
||||
hook: Final = (
|
||||
service_logging.async_service_success_hook(
|
||||
service=ServiceTypes.DB,
|
||||
call_type=call_type,
|
||||
parent_otel_span=None,
|
||||
duration=duration_seconds,
|
||||
start_time=start_time,
|
||||
end_time=start_time + timedelta(seconds=duration_seconds),
|
||||
event_metadata=db_event_metadata(table, operation),
|
||||
)
|
||||
if failure is None
|
||||
else service_logging.async_service_failure_hook(
|
||||
error=NativeDatabaseError(failure),
|
||||
service=ServiceTypes.DB,
|
||||
call_type=call_type,
|
||||
parent_otel_span=None,
|
||||
duration=duration_seconds,
|
||||
start_time=start_time,
|
||||
end_time=start_time + timedelta(seconds=duration_seconds),
|
||||
event_metadata=db_event_metadata(table, operation),
|
||||
)
|
||||
)
|
||||
_ = self.loop.call_soon_threadsafe(self.loop.create_task, hook)
|
||||
|
|
@ -140,19 +140,27 @@ def _userlist_quote(value: str) -> str:
|
|||
return '"' + value.replace('"', '""') + '"'
|
||||
|
||||
|
||||
def _option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None:
|
||||
"""The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else.
|
||||
|
||||
Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``; a
|
||||
detached ``-c`` is folded into the token that follows it first.
|
||||
"""
|
||||
folded: Final = tuple(
|
||||
def fold_libpq_option_tokens(tokens: Sequence[str]) -> tuple[str, ...]:
|
||||
"""The tokens of a libpq ``options`` string with each detached ``-c`` folded into the token after it."""
|
||||
return tuple(
|
||||
f"-c{tokens[index + 1]}" if token == "-c" and index + 1 < len(tokens) else token
|
||||
for index, token in enumerate(tokens)
|
||||
if index == 0 or tokens[index - 1] != "-c"
|
||||
)
|
||||
settings: Final = tuple(token[2:] for token in folded if token.startswith(("-c", "--")) and "=" in token[2:])
|
||||
return settings if len(settings) == len(folded) else None
|
||||
|
||||
|
||||
def libpq_option_setting(token: str) -> str | None:
|
||||
"""The ``name=value`` setting a folded ``-cname=value`` or ``--name=value`` token carries, or None."""
|
||||
return token[2:] if token.startswith(("-c", "--")) and "=" in token[2:] else None
|
||||
|
||||
|
||||
def libpq_option_settings(tokens: Sequence[str]) -> tuple[str, ...] | None:
|
||||
"""The ``name=value`` settings in a libpq ``options`` string, or None if it holds anything else.
|
||||
|
||||
Accepts ``-c name=value``, ``-cname=value`` and ``--name=value``.
|
||||
"""
|
||||
settings: Final = tuple(libpq_option_setting(token) for token in fold_libpq_option_tokens(tokens))
|
||||
return None if None in settings else tuple(setting for setting in settings if setting is not None)
|
||||
|
||||
|
||||
def _connect_query(options: str) -> str | PgBouncerError:
|
||||
|
|
@ -163,7 +171,7 @@ def _connect_query(options: str) -> str | PgBouncerError:
|
|||
the settings are applied to each new server connection instead. Every
|
||||
client shares them, which is what the single ``DATABASE_URL`` gave anyway.
|
||||
"""
|
||||
settings: Final = _option_settings(tuple(shlex.split(options)))
|
||||
settings: Final = libpq_option_settings(tuple(shlex.split(options)))
|
||||
if settings is None:
|
||||
return PgBouncerError(f"cannot translate the DATABASE_URL options {options!r} into PgBouncer settings")
|
||||
return "; ".join(
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -41,6 +41,7 @@ from litellm.caching.redis_cache import log_redis_failure
|
|||
from litellm.constants import (
|
||||
DYNAMIC_RATE_LIMIT_ERROR_THRESHOLD_PER_MINUTE,
|
||||
INTERNAL_CALL_ORIGIN_METADATA_KEY,
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY,
|
||||
NO_MAX_TOKENS_TPM_RESERVATION_FRACTION,
|
||||
RESPONSES_WEBSOCKET_CALL_ID_METADATA_KEY,
|
||||
)
|
||||
|
|
@ -118,6 +119,7 @@ from litellm.types.extraction import ExtractionResponse
|
|||
from litellm.types.llms.openai import BaseLiteLLMOpenAIResponseObject, ResponseAPIUsage
|
||||
from litellm.types.mcp_server.mcp_server_manager import MCPServer
|
||||
from litellm.types.passthrough_endpoints.pass_through_endpoints import EndpointType
|
||||
from litellm.types.proxy.auth.customer_hierarchy import CustomerScopeAuth
|
||||
from litellm.types.utils import (
|
||||
CallTypes,
|
||||
EmbeddingResponse,
|
||||
|
|
@ -200,7 +202,7 @@ def inherited_team_model_limit(
|
|||
return team_limit
|
||||
|
||||
|
||||
TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = "team_model_limit_max_borrow_multiple"
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE: Final = MODEL_LIMIT_MAX_BORROW_MULTIPLE_METADATA_KEY
|
||||
MAX_BORROW_MULTIPLE_CEILING: Final = 1_000_000
|
||||
_MAX_BORROW_MULTIPLE: Final[TypeAdapter[float]] = TypeAdapter(
|
||||
Annotated[float, Field(ge=1, le=MAX_BORROW_MULTIPLE_CEILING, allow_inf_nan=False)]
|
||||
|
|
@ -231,30 +233,48 @@ class BorrowingTeamModelLimits:
|
|||
NO_BORROWING: Final = BorrowingTeamModelLimits(tpm=None, rpm=None)
|
||||
|
||||
|
||||
def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None:
|
||||
raw: Final[object] = (user_api_key_dict.team_metadata or {}).get(TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE)
|
||||
BorrowerKind: TypeAlias = Literal["team", "key", "customer_org"]
|
||||
BORROWER_KINDS: Final[tuple[BorrowerKind, ...]] = ("team", "key", "customer_org")
|
||||
_METADATA: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ModelLimitBorrower:
|
||||
"""Who holds a per-model limit the dynamic rate limiter enforces at saturation, and that limit."""
|
||||
|
||||
kind: BorrowerKind
|
||||
entity_id: str
|
||||
alias: str | None
|
||||
limits: BorrowingTeamModelLimits
|
||||
|
||||
|
||||
def _max_borrow_multiple(raw: object, setting: str, owner: str) -> float | None:
|
||||
if raw is None:
|
||||
return None
|
||||
try:
|
||||
return _MAX_BORROW_MULTIPLE.validate_python(raw)
|
||||
except ValidationError:
|
||||
verbose_proxy_logger.warning(
|
||||
"Ignoring %s=%r on team %s: expected a number from 1 to %d, so the team borrows up to the model limit",
|
||||
TEAM_MODEL_LIMIT_MAX_BORROW_MULTIPLE,
|
||||
"Ignoring %s=%r on %s: expected a number from 1 to %d, so the value is not used",
|
||||
setting,
|
||||
raw,
|
||||
user_api_key_dict.team_id,
|
||||
owner,
|
||||
MAX_BORROW_MULTIPLE_CEILING,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def team_model_limit_max_borrow_multiple(user_api_key_dict: UserAPIKeyAuth) -> float | None:
|
||||
return _max_borrow_multiple(
|
||||
_METADATA.validate_python(user_api_key_dict.team_metadata or {}).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE),
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE,
|
||||
f"team {user_api_key_dict.team_id}",
|
||||
)
|
||||
|
||||
|
||||
def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits:
|
||||
settings: Final = litellm.priority_reservation_settings
|
||||
if (
|
||||
settings is None
|
||||
or not settings.team_model_limit_borrows_idle_capacity_from_other_teams
|
||||
or user_api_key_dict.team_id is None
|
||||
):
|
||||
if settings is None or "team" not in settings.model_limit_borrowing_entities or user_api_key_dict.team_id is None:
|
||||
return NO_BORROWING
|
||||
limits: Final = BorrowingTeamModelLimits(
|
||||
tpm=inherited_team_model_limit(user_api_key_dict, model, "model_tpm_limit"),
|
||||
|
|
@ -263,10 +283,90 @@ def borrowing_team_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLi
|
|||
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
|
||||
return NO_BORROWING
|
||||
return BorrowingTeamModelLimits(
|
||||
tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict)
|
||||
tpm=limits.tpm,
|
||||
rpm=limits.rpm,
|
||||
max_borrow_multiple=team_model_limit_max_borrow_multiple(user_api_key_dict)
|
||||
or settings.model_limit_borrowing_config.max_borrow_multiple,
|
||||
)
|
||||
|
||||
|
||||
def borrowing_key_model_limits(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> BorrowingTeamModelLimits:
|
||||
"""The key's own per-model limits when keys are listed to borrow idle capacity."""
|
||||
settings: Final = litellm.priority_reservation_settings
|
||||
if settings is None or "key" not in settings.model_limit_borrowing_entities or user_api_key_dict.api_key is None:
|
||||
return NO_BORROWING
|
||||
limits: Final = BorrowingTeamModelLimits(
|
||||
tpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_tpm_limit")),
|
||||
rpm=model.limit_in(get_key_own_model_rate_limit(user_api_key_dict, "model_rpm_limit")),
|
||||
)
|
||||
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
|
||||
return NO_BORROWING
|
||||
return BorrowingTeamModelLimits(
|
||||
tpm=limits.tpm,
|
||||
rpm=limits.rpm,
|
||||
max_borrow_multiple=_max_borrow_multiple(
|
||||
_METADATA.validate_python(
|
||||
user_api_key_dict.metadata or {} # pyright: ignore[reportUnknownMemberType] # validates the untyped key metadata
|
||||
).get(MODEL_LIMIT_MAX_BORROW_MULTIPLE),
|
||||
MODEL_LIMIT_MAX_BORROW_MULTIPLE,
|
||||
"a key",
|
||||
)
|
||||
or settings.model_limit_borrowing_config.max_borrow_multiple,
|
||||
)
|
||||
|
||||
|
||||
def _customer_org_scope(user_api_key_dict: UserAPIKeyAuth) -> CustomerScopeAuth | None:
|
||||
hierarchy: Final = user_api_key_dict.customer_hierarchy
|
||||
if hierarchy is None:
|
||||
return None
|
||||
return next((scope for scope in hierarchy.scopes if scope.key == "customer_org"), None)
|
||||
|
||||
|
||||
def borrowing_customer_org_model_limits(
|
||||
user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel
|
||||
) -> BorrowingTeamModelLimits:
|
||||
"""The customer organization's per-model limits when customer organizations are listed to borrow idle
|
||||
capacity. The share is one counter per organization and model, across every key the customer calls."""
|
||||
settings: Final = litellm.priority_reservation_settings
|
||||
scope: Final = _customer_org_scope(user_api_key_dict)
|
||||
if settings is None or "customer_organization" not in settings.model_limit_borrowing_entities or scope is None:
|
||||
return NO_BORROWING
|
||||
limits: Final = BorrowingTeamModelLimits(
|
||||
tpm=model.limit_in(scope.model_tpm_limit), rpm=model.limit_in(scope.model_rpm_limit)
|
||||
)
|
||||
if limits == NO_BORROWING or not dynamic_rate_limiter_covers(model.group):
|
||||
return NO_BORROWING
|
||||
return BorrowingTeamModelLimits(
|
||||
tpm=limits.tpm, rpm=limits.rpm, max_borrow_multiple=settings.model_limit_borrowing_config.max_borrow_multiple
|
||||
)
|
||||
|
||||
|
||||
def model_limit_borrowers(user_api_key_dict: UserAPIKeyAuth, model: RateLimitedModel) -> tuple[ModelLimitBorrower, ...]:
|
||||
"""Every level of the request whose per-model limit on ``model`` is a share that borrows idle capacity."""
|
||||
org_scope: Final = _customer_org_scope(user_api_key_dict)
|
||||
candidates: Final = (
|
||||
ModelLimitBorrower(
|
||||
kind="team",
|
||||
entity_id=str(user_api_key_dict.team_id),
|
||||
alias=user_api_key_dict.team_alias,
|
||||
limits=borrowing_team_model_limits(user_api_key_dict, model),
|
||||
),
|
||||
ModelLimitBorrower(
|
||||
kind="key",
|
||||
entity_id=str(user_api_key_dict.api_key),
|
||||
alias=user_api_key_dict.key_alias,
|
||||
limits=borrowing_key_model_limits(user_api_key_dict, model),
|
||||
),
|
||||
ModelLimitBorrower(
|
||||
kind="customer_org",
|
||||
entity_id="" if org_scope is None else org_scope.entity_id,
|
||||
alias=None if org_scope is None else org_scope.alias,
|
||||
limits=borrowing_customer_org_model_limits(user_api_key_dict, model),
|
||||
),
|
||||
)
|
||||
return tuple(borrower for borrower in candidates if borrower.limits != NO_BORROWING)
|
||||
|
||||
|
||||
def _resolve_media_models(router: "Router | None", model: str, team_id: str | None) -> tuple[str, ...]:
|
||||
if router is None:
|
||||
return (model,)
|
||||
|
|
@ -587,6 +687,34 @@ class BorrowingTeamModelHold:
|
|||
share: RateLimitDescriptor | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class BorrowingShareState:
|
||||
"""One borrower's share on the admitting model for this request: what was enforced, what was held on it, and the
|
||||
holds taken on router fallback targets."""
|
||||
|
||||
limit: RateLimitDescriptor | None = None
|
||||
reserved_tokens: int = 0
|
||||
reserved_window: ReservationWindowIdentity | None = None
|
||||
share: RateLimitDescriptor | None = None
|
||||
cap: RateLimitDescriptor | None = None
|
||||
cap_reserved_tokens: int = 0
|
||||
cap_reserved_window: ReservationWindowIdentity | None = None
|
||||
fallback_limits: Mapping[str, BorrowingTeamModelHold] = field(default_factory=lambda: MappingProxyType({}))
|
||||
|
||||
def clear_reservation(self) -> None:
|
||||
self.reserved_tokens = 0
|
||||
self.cap_reserved_tokens = 0
|
||||
self.reserved_window = None
|
||||
self.cap_reserved_window = None
|
||||
|
||||
def hold_fallback(self, names: tuple[str, ...], hold: BorrowingTeamModelHold) -> None:
|
||||
self.fallback_limits = MappingProxyType({**self.fallback_limits, **dict.fromkeys(names, hold)})
|
||||
|
||||
def drop_fallback_hold(self, hold: BorrowingTeamModelHold | None) -> None:
|
||||
if hold is not None:
|
||||
self.fallback_limits = MappingProxyType({k: h for k, h in self.fallback_limits.items() if h is not hold})
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RequestRateLimiterStash:
|
||||
"""
|
||||
|
|
@ -631,17 +759,11 @@ class RequestRateLimiterStash:
|
|||
otpm_reserved_window_identities: frozenset[ReservationWindowIdentity] = field(default_factory=frozenset)
|
||||
batch_enqueued_reservation: BatchEnqueuedTokenReservation | None = None
|
||||
batch_tpd_refund_ops: tuple[ReservationOperation, ...] = ()
|
||||
borrowing_team_model_limit: RateLimitDescriptor | None = None
|
||||
borrowing_team_model_reserved_tokens: int = 0
|
||||
borrowing_team_model_reserved_window: ReservationWindowIdentity | None = None
|
||||
borrowing_team_model_share: RateLimitDescriptor | None = None
|
||||
borrowing_team_model_cap: RateLimitDescriptor | None = None
|
||||
borrowing_team_model_cap_reserved_tokens: int = 0
|
||||
borrowing_team_model_cap_reserved_window: ReservationWindowIdentity | None = None
|
||||
borrowing_team_model_fallback_limits: Mapping[str, BorrowingTeamModelHold] = field(
|
||||
default_factory=lambda: MappingProxyType({})
|
||||
borrowing_shares: Mapping[BorrowerKind, BorrowingShareState] = field(
|
||||
default_factory=lambda: MappingProxyType({kind: BorrowingShareState() for kind in BORROWER_KINDS})
|
||||
)
|
||||
priority_class_share: RateLimitDescriptor | None = None
|
||||
priority_class_share_holder: str | None = None
|
||||
dynamic_limiter_priority: str | None = None
|
||||
reservation_released: bool = False
|
||||
io_reservation_released: bool = False
|
||||
|
|
@ -2718,8 +2840,17 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
model: Final = self._rate_limited_model(requested_model)
|
||||
if model is None or user_api_key_dict.customer_hierarchy is None:
|
||||
return ()
|
||||
org_borrowing: Final = borrowing_customer_org_model_limits(user_api_key_dict, model)
|
||||
limits: Final = tuple(
|
||||
(scope, model.limit_in(scope.model_tpm_limit), model.limit_in(scope.model_rpm_limit))
|
||||
(
|
||||
scope,
|
||||
None
|
||||
if scope.key == "customer_org" and org_borrowing.tpm is not None
|
||||
else model.limit_in(scope.model_tpm_limit),
|
||||
None
|
||||
if scope.key == "customer_org" and org_borrowing.rpm is not None
|
||||
else model.limit_in(scope.model_rpm_limit),
|
||||
)
|
||||
for scope in user_api_key_dict.customer_hierarchy.scopes
|
||||
)
|
||||
return tuple(
|
||||
|
|
@ -2779,14 +2910,15 @@ class _PROXY_MaxParallelRequestsHandler_v3(CustomLogger):
|
|||
if model is None:
|
||||
return
|
||||
borrowing: Final = borrowing_team_model_limits(user_api_key_dict, model)
|
||||
key_borrowing: Final = borrowing_key_model_limits(user_api_key_dict, model)
|
||||
model_specific_tpm_limit: Final = (
|
||||
None
|
||||
if borrowing.tpm is not None
|
||||
if borrowing.tpm is not None or key_borrowing.tpm is not None
|
||||
else model.limit_in(get_key_model_tpm_limit(user_api_key_dict, model_name=model.group))
|
||||
)
|
||||
model_specific_rpm_limit: Final = (
|
||||
None
|
||||
if borrowing.rpm is not None
|
||||
if borrowing.rpm is not None or key_borrowing.rpm is not None
|
||||
else model.limit_in(get_key_model_rpm_limit(user_api_key_dict, model_name=model.group))
|
||||
)
|
||||
if model_specific_tpm_limit is None and model_specific_rpm_limit is None:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
"""Splits a priority class's share of a model among the teams active on it.
|
||||
"""Splits a priority class's share of a model among the callers active on it.
|
||||
|
||||
Enabled by ``priority_reservation_settings.priority_class_share_splits_equally_among_active_teams``.
|
||||
Enabled by ``priority_reservation_settings.priority_class_share_config``.
|
||||
"""
|
||||
|
||||
import time
|
||||
|
|
@ -40,6 +40,16 @@ def team_member(team_id: str) -> str:
|
|||
return f"team{MEMBER_SEPARATOR}{team_id}"
|
||||
|
||||
|
||||
def key_member(key_hash: str) -> str:
|
||||
"""The registry id and counter owner of a key that takes its own slice of its class, apart from every team id."""
|
||||
return f"key{MEMBER_SEPARATOR}{key_hash}"
|
||||
|
||||
|
||||
def customer_member(kind: str, entity_id: str) -> str:
|
||||
"""The registry id and counter owner of a customer organization or customer member taking its own slice."""
|
||||
return f"{kind}{MEMBER_SEPARATOR}{entity_id}"
|
||||
|
||||
|
||||
def is_pool_member(member: str) -> bool:
|
||||
return member.startswith(_POOL_PREFIX)
|
||||
|
||||
|
|
|
|||
|
|
@ -1595,7 +1595,11 @@ def _tenant_destination(
|
|||
name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None
|
||||
) -> "OtelDestination | None":
|
||||
from litellm.integrations.otel.model.baggage import parse_destination_metadata_keys
|
||||
from litellm.integrations.otel.model.config import parse_capture_message_content, parse_excluded_services
|
||||
from litellm.integrations.otel.model.config import (
|
||||
langfuse_destination_span_scope,
|
||||
parse_capture_message_content,
|
||||
parse_excluded_services,
|
||||
)
|
||||
from litellm.integrations.otel.presets.destinations import destination_for
|
||||
|
||||
callback_vars: Final = MappingProxyType(
|
||||
|
|
@ -1605,6 +1609,7 @@ def _tenant_destination(
|
|||
)
|
||||
)
|
||||
)
|
||||
otel_settings: Final = as_str_mapping((litellm.callback_settings or {}).get("otel")) or {}
|
||||
return destination_for(
|
||||
name,
|
||||
_tenant_otel_params(callback_vars),
|
||||
|
|
@ -1612,6 +1617,9 @@ def _tenant_destination(
|
|||
parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)),
|
||||
parse_excluded_services(callback_vars.get(EXCLUDED_SERVICES_VAR)),
|
||||
parse_destination_metadata_keys(callback_vars.get(BAGGAGE_METADATA_KEYS_VAR)),
|
||||
langfuse_destination_span_scope=langfuse_destination_span_scope(
|
||||
otel_settings.get("langfuse_destination_span_scope")
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -278,7 +278,12 @@ async def update_budget(
|
|||
return response
|
||||
|
||||
|
||||
@router.post("/budget/new", tags=["budget management"], description=new_budget.__doc__)
|
||||
@router.post(
|
||||
"/budget/new",
|
||||
tags=["budget management"],
|
||||
description=new_budget.__doc__,
|
||||
operation_id="new_budget_budget_new_post",
|
||||
)
|
||||
async def new_budget_endpoint(
|
||||
budget_obj: BudgetWriteRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
|
|
@ -286,7 +291,12 @@ async def new_budget_endpoint(
|
|||
return await new_budget(budget_obj, user_api_key_dict)
|
||||
|
||||
|
||||
@router.post("/budget/update", tags=["budget management"], description=update_budget.__doc__)
|
||||
@router.post(
|
||||
"/budget/update",
|
||||
tags=["budget management"],
|
||||
description=update_budget.__doc__,
|
||||
operation_id="update_budget_budget_update_post",
|
||||
)
|
||||
async def update_budget_endpoint(
|
||||
budget_obj: BudgetWriteRequest,
|
||||
user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
|
||||
|
|
|
|||
|
|
@ -103,7 +103,7 @@ from litellm.proxy.utils import handle_exception_on_proxy, hash_password, query_
|
|||
from litellm.redis.stores.spend_counters import (
|
||||
user_counter,
|
||||
)
|
||||
from litellm.repositories.chunked_in import update_many_in
|
||||
from litellm.repositories.chunked_in import find_many_in, update_many_in
|
||||
from litellm.repositories.organization_repository import OrganizationRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.repositories.table_repositories import (
|
||||
|
|
@ -159,6 +159,7 @@ class _AdminTeamAndKeyRows(TypedDict):
|
|||
_ADMIN_TEAM_AND_KEY_ROWS_ADAPTER: Final = TypeAdapter(
|
||||
list[_AdminTeamAndKeyRows], config=ConfigDict(hide_input_in_errors=True)
|
||||
)
|
||||
_TEAM_METADATA_ADAPTER: Final = TypeAdapter(dict[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
_ADMIN_KEY_ROW_ADAPTER: Final = TypeAdapter(
|
||||
dict[str, object], config=ConfigDict(strict=True, hide_input_in_errors=True)
|
||||
)
|
||||
|
|
@ -468,8 +469,12 @@ async def add_new_user_to_default_team(
|
|||
user_api_key_dict: UserAPIKeyAuth,
|
||||
teams: Sequence[str | NewUserRequestTeam],
|
||||
prisma_client: "PrismaClient",
|
||||
teams_are_defaults: bool = True,
|
||||
):
|
||||
memberships: Final = tuple(_default_team_membership(team) for team in teams)
|
||||
requested: Final = tuple(_default_team_membership(team) for team in teams)
|
||||
memberships: Final = (
|
||||
await _memberships_on_team_member_budget(requested, prisma_client) if teams_are_defaults else requested
|
||||
)
|
||||
for membership in memberships:
|
||||
with contextlib.suppress(Exception):
|
||||
await _add_user_to_team(
|
||||
|
|
@ -482,6 +487,25 @@ async def add_new_user_to_default_team(
|
|||
)
|
||||
|
||||
|
||||
async def _memberships_on_team_member_budget(
|
||||
memberships: tuple[NewUserRequestTeam, ...], prisma_client: "PrismaClient"
|
||||
) -> tuple[NewUserRequestTeam, ...]:
|
||||
"""A default team's max_budget_in_team is already the team's member budget, which also carries the reset."""
|
||||
capped_team_ids: Final = [m.team_id for m in memberships if m.max_budget_in_team is not None]
|
||||
if not capped_team_ids:
|
||||
return memberships
|
||||
team_rows: Final = await find_many_in(_team_table(prisma_client), "team_id", capped_team_ids)
|
||||
budgeted_team_ids: Final = frozenset(
|
||||
row.team_id
|
||||
for row in team_rows
|
||||
if isinstance(row.metadata, dict)
|
||||
and _TEAM_METADATA_ADAPTER.validate_python(row.metadata).get("team_member_budget_id") is not None
|
||||
)
|
||||
return tuple(
|
||||
m.model_copy(update={"max_budget_in_team": None}) if m.team_id in budgeted_team_ids else m for m in memberships
|
||||
)
|
||||
|
||||
|
||||
def _default_team_membership(team: str | NewUserRequestTeam) -> NewUserRequestTeam:
|
||||
if isinstance(team, str):
|
||||
return NewUserRequestTeam(team_id=team)
|
||||
|
|
@ -666,6 +690,7 @@ async def new_user(
|
|||
user_api_key_dict=user_api_key_dict,
|
||||
teams=teams,
|
||||
prisma_client=prisma_client,
|
||||
teams_are_defaults=data.teams is None,
|
||||
)
|
||||
|
||||
user_id: Final = cast(str | None, response.get("user_id", None))
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ from litellm.proxy.list_api.common import (
|
|||
)
|
||||
from litellm.proxy.management_endpoints.management_v1.common import MANAGEMENT_V1_PREFIX
|
||||
from litellm.proxy.management_helpers.customer_hierarchy import (
|
||||
BudgetSort,
|
||||
CustomerHierarchy,
|
||||
CustomerPortalLimits,
|
||||
ErrorCode,
|
||||
|
|
@ -54,13 +55,18 @@ _ERROR_DETAILS: Final[Mapping[ErrorCode, tuple[int, str, str]]] = MappingProxyTy
|
|||
{
|
||||
ErrorCode.CAPACITY_BUSY: (
|
||||
503,
|
||||
"Customer portal busy",
|
||||
"Customer portal capacity is being updated. Retry the request.",
|
||||
"Customer Management busy",
|
||||
"Customer Management capacity is being updated. Retry the request.",
|
||||
),
|
||||
ErrorCode.TRANSACTION_TIMEOUT: (
|
||||
504,
|
||||
"Customer Management operation timed out",
|
||||
"Customer Management operation timed out. The transaction was rolled back. Retry the request.",
|
||||
),
|
||||
ErrorCode.FREE_TIER_LIMIT: (
|
||||
403,
|
||||
"Free customer portal limit reached",
|
||||
"Free customer portal limit reached: up to 5 customer organizations, 5 teams per organization and "
|
||||
"Customer Management free tier limit reached",
|
||||
"Customer Management free tier limit reached: up to 5 customer organizations, 5 teams per organization and "
|
||||
"5 members per team without a LiteLLM Enterprise license. Upgrade to Enterprise to remove the limits: "
|
||||
"https://www.litellm.ai/enterprise",
|
||||
),
|
||||
|
|
@ -213,7 +219,8 @@ AuthCache: TypeAlias = Annotated[UserApiKeyCache, Depends(get_user_api_key_cache
|
|||
@router.get(
|
||||
"/customer_portal/limits",
|
||||
response_model=CustomerPortalLimits,
|
||||
description="Get customer portal free-tier limits and current capacity. Proxy admins only.",
|
||||
summary="Get Customer Management Limits",
|
||||
description="Get Customer Management free-tier limits and current capacity. Proxy admins only.",
|
||||
)
|
||||
@management_endpoint_wrapper
|
||||
async def get_customer_portal_limits(
|
||||
|
|
@ -407,9 +414,12 @@ async def list_customer_teams(
|
|||
page: Annotated[int, Query(ge=1)] = 1,
|
||||
page_size: Annotated[int, Query(ge=1)] = 50,
|
||||
parent_org_id: str | None = None,
|
||||
sort: Annotated[
|
||||
BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.")
|
||||
] = None,
|
||||
) -> ListResponse[Team]:
|
||||
size: Final = min(page_size, 100)
|
||||
result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size))
|
||||
result: Final = resolve(await hierarchy.list_teams(parent_org_id, page, size, sort))
|
||||
return list_response(result, request, page, size)
|
||||
|
||||
|
||||
|
|
@ -470,7 +480,9 @@ async def get_customer_member(
|
|||
@router.patch(
|
||||
"/customer_members/{customer_member_id}",
|
||||
response_model=CustomerMember,
|
||||
description="Update a customer member and its budgets. Proxy admins only.",
|
||||
description="Update a customer member and its budgets. Supply parent_team_ids to replace its teams within the "
|
||||
"same organization, with the first team as default. Identity, budget counters and historical usage are retained. "
|
||||
"Historical usage stays attributed to the original team. Proxy admins only.",
|
||||
)
|
||||
@management_endpoint_wrapper
|
||||
async def update_customer_member(
|
||||
|
|
@ -505,7 +517,10 @@ async def delete_customer_member(
|
|||
"/customer_members",
|
||||
response_model=ListResponse[CustomerMember],
|
||||
dependencies=(Depends(reject_duplicate_list_params),),
|
||||
description="List customer members with page-number pagination and resolved budgets. Proxy admins only.",
|
||||
description=(
|
||||
"List customer members with page-number pagination and resolved budgets. "
|
||||
"search filters by customer member ID or alias before paginating. Proxy admins only."
|
||||
),
|
||||
)
|
||||
@management_endpoint_wrapper
|
||||
async def list_customer_members(
|
||||
|
|
@ -515,9 +530,17 @@ async def list_customer_members(
|
|||
page_size: Annotated[int, Query(ge=1)] = 50,
|
||||
parent_team_id: str | None = None,
|
||||
parent_org_id: str | None = None,
|
||||
search: Annotated[str, Query(description="Case-insensitive substring of the customer member ID or alias")] = "",
|
||||
sort: Annotated[
|
||||
BudgetSort | None, Query(description="Sort by displayed budget limit, with uncapped rows last.")
|
||||
] = None,
|
||||
) -> ListResponse[CustomerMember]:
|
||||
size: Final = min(page_size, 100)
|
||||
result: Final = resolve(await hierarchy.list_members(parent_team_id, page, size, parent_org_id=parent_org_id))
|
||||
result: Final = resolve(
|
||||
await hierarchy.list_members(
|
||||
parent_team_id, page, size, parent_org_id=parent_org_id, search=search.strip(), sort=sort
|
||||
)
|
||||
)
|
||||
return list_response(result, request, page, size)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -108,7 +108,7 @@ PROXY_CONTROLLED_TEAM_METADATA_KEYS: Final[frozenset[str]] = frozenset(
|
|||
"phoenix_project_name_override",
|
||||
"phoenix_project_name",
|
||||
"opik",
|
||||
"team_model_limit_max_borrow_multiple",
|
||||
"model_limit_max_borrow_multiple",
|
||||
)
|
||||
)
|
||||
_NOT_COLUMNS: Final[frozenset[str]] = frozenset({"team_id", "metadata"})
|
||||
|
|
|
|||
|
|
@ -301,7 +301,10 @@ async def add_team_callbacks(
|
|||
- langfuse_secret: The secret for the Langfuse callback
|
||||
- langfuse_host: The host for the Langfuse callback
|
||||
- langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT)
|
||||
- langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans
|
||||
- langfuse_span_scope: For langfuse_otel, "full" sends the whole request trace,
|
||||
"llm_only" sends only model-call spans.
|
||||
Omitted follows LITELLM_OTEL_LANGFUSE_DESTINATION_SPAN_SCOPE (default "full"), also configurable as
|
||||
callback_settings.otel.langfuse_destination_span_scope.
|
||||
- capture_message_content: For OTel v2 callbacks, "no_content" or "span_only".
|
||||
An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting.
|
||||
"span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out
|
||||
|
|
|
|||
|
|
@ -238,7 +238,7 @@ from litellm.proxy.utils import (
|
|||
)
|
||||
from litellm.redis.stores.spend_counters import team_counter
|
||||
from litellm.repositories.budget_repository import BudgetRepository
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in
|
||||
from litellm.repositories.chunked_in import delete_many_in, find_many_in, update_many_in
|
||||
from litellm.repositories.daily_activity_buckets import api_key_from_bucket
|
||||
from litellm.repositories.organization_repository import OrganizationRepository
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
|
|
@ -919,6 +919,50 @@ class TeamMemberBudgetHandler:
|
|||
TeamMemberBudgetHandler.clean_team_member_fields(updated_kv)
|
||||
return updated_kv
|
||||
|
||||
@staticmethod
|
||||
async def member_budgets_on_team_reset_window(
|
||||
team_id: str, team_member_budget_id: str | None, prisma_client: PrismaClient
|
||||
) -> tuple[str | None, tuple[str, ...]]:
|
||||
"""The team default's reset window, and the budgets of members who take their cap from it and reset on a copy."""
|
||||
team_default: Final = (
|
||||
await _budget_db(prisma_client).find_unique(where={"budget_id": team_member_budget_id})
|
||||
if team_member_budget_id is not None
|
||||
else None
|
||||
)
|
||||
team_window: Final = team_default.budget_duration if team_default is not None else None
|
||||
memberships: Final = await _team_membership_db(prisma_client).find_many(
|
||||
where={"team_id": team_id, "budget_id": {"not": team_member_budget_id}},
|
||||
include={"litellm_budget_table": True},
|
||||
)
|
||||
return team_window, tuple(
|
||||
m.litellm_budget_table.budget_id
|
||||
for m in memberships
|
||||
if m.litellm_budget_table is not None
|
||||
and m.litellm_budget_table.max_budget is None
|
||||
and m.litellm_budget_table.budget_duration == team_window
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def move_members_to_team_reset_window(
|
||||
team_id: str, team_member_budget_id: str | None, budget_duration: str | None, prisma_client: PrismaClient
|
||||
) -> None:
|
||||
"""Runs before the team default is written, so a retry after a failure still finds the members left behind."""
|
||||
team_window, budget_ids = await TeamMemberBudgetHandler.member_budgets_on_team_reset_window(
|
||||
team_id=team_id, team_member_budget_id=team_member_budget_id, prisma_client=prisma_client
|
||||
)
|
||||
if team_window == budget_duration or not budget_ids:
|
||||
return
|
||||
_ = await update_many_in(
|
||||
_budget_db(prisma_client),
|
||||
"budget_id",
|
||||
budget_ids,
|
||||
data={
|
||||
"budget_duration": budget_duration,
|
||||
"budget_reset_at": get_budget_reset_time(budget_duration) if budget_duration else None,
|
||||
},
|
||||
atomicity="per_chunk_ok",
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
async def backfill_team_member_budget_entries(
|
||||
team_id: str,
|
||||
|
|
@ -2825,6 +2869,15 @@ async def update_team(
|
|||
},
|
||||
}
|
||||
|
||||
_current_member_budget_id: Final = (existing_team_row.metadata or {}).get("team_member_budget_id")
|
||||
if "team_member_budget_duration" in _team_member_fields_in_request:
|
||||
await TeamMemberBudgetHandler.move_members_to_team_reset_window(
|
||||
team_id=data.team_id,
|
||||
team_member_budget_id=_current_member_budget_id if isinstance(_current_member_budget_id, str) else None,
|
||||
budget_duration=data.team_member_budget_duration,
|
||||
prisma_client=prisma_client,
|
||||
)
|
||||
|
||||
if _team_member_fields_in_request and TeamMemberBudgetHandler.should_create_budget(
|
||||
team_member_budget=data.team_member_budget,
|
||||
team_member_rpm_limit=data.team_member_rpm_limit,
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from litellm.repositories.table_repositories import AuditLogRepository
|
|||
from litellm.types.utils import StandardAuditLogPayload
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.integrations.s3_v3 import S3Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger
|
||||
|
||||
_audit_log_callback_cache: Final[dict[str, "CustomLogger | S3Logger"]] = {}
|
||||
ALLOW_LITELLM_CHANGED_BY_HEADER_METADATA_KEY: Final = "allow_litellm_changed_by_header"
|
||||
|
|
@ -78,9 +78,9 @@ def _resolve_audit_log_callback(name: str) -> "CustomLogger | S3Logger | None":
|
|||
instance: CustomLogger | S3Logger | None
|
||||
audit_params: Final[Mapping[str, object] | None] = getattr(litellm, "s3_audit_callback_params", None)
|
||||
if name == "s3_v2" and audit_params is not None:
|
||||
from litellm.integrations import s3_v3
|
||||
from litellm.integrations import s3_v2
|
||||
|
||||
instance = s3_v3.S3Logger(s3_callback_params_override=audit_params)
|
||||
instance = s3_v2.S3Logger(s3_callback_params_override=audit_params)
|
||||
else:
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_init_custom_logger_compatible_class,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ from dataclasses import dataclass
|
|||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from itertools import chain
|
||||
from typing import TYPE_CHECKING, Final, Generic, Protocol, TypeVar
|
||||
from typing import TYPE_CHECKING, Final, Generic, Literal, Protocol, TypeAlias, TypeVar
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, model_validator
|
||||
|
|
@ -20,7 +20,7 @@ from litellm.proxy.db.exception_handler import PrismaDBExceptionHandler
|
|||
from litellm.proxy.list_api.common import escape_like
|
||||
from litellm.proxy.management_helpers.customer_budget_usage import customer_budget_usage
|
||||
from litellm.repositories.budget_repository import create_budget_row, update_budget_row
|
||||
from litellm.repositories.chunked_in import count_in, delete_many_in, find_many_in
|
||||
from litellm.repositories.chunked_in import IN_LIST_CHUNK_SIZE, count_in, delete_many_in, find_many_in
|
||||
from litellm.repositories.prisma_protocols import TableActions
|
||||
from litellm.types.proxy.management_endpoints.customer_hierarchy import (
|
||||
Budget,
|
||||
|
|
@ -29,6 +29,7 @@ from litellm.types.proxy.management_endpoints.customer_hierarchy import (
|
|||
BudgetTemplate,
|
||||
CustomerMember,
|
||||
MemberCreate,
|
||||
MemberFields,
|
||||
MemberPatch,
|
||||
Organization,
|
||||
OrganizationBudget,
|
||||
|
|
@ -45,9 +46,53 @@ if TYPE_CHECKING:
|
|||
from prisma.models import LiteLLM_BudgetWindowSpend
|
||||
|
||||
|
||||
BudgetSort: TypeAlias = Literal["budget_asc", "budget_desc"]
|
||||
|
||||
|
||||
def customer_budget_limit(budget: Budget | None) -> float | None:
|
||||
if budget is None:
|
||||
return None
|
||||
limits: Final = budget.budget_limits or ()
|
||||
monthly: Final = next(
|
||||
(
|
||||
limit.max_budget
|
||||
for limit in limits
|
||||
if limit.budget_duration.strip().lower() in ("1mo", "1month", "1months", "30d", "30days", "monthly")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if monthly is not None:
|
||||
return monthly
|
||||
if budget.max_budget is not None:
|
||||
return budget.max_budget
|
||||
return next(
|
||||
(
|
||||
limit.max_budget
|
||||
for limit in limits
|
||||
if limit.budget_duration.strip().lower() in ("12mo", "12month", "12months")
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def _budget_sort_key(budget: Budget | None, sort: BudgetSort) -> tuple[bool, float]:
|
||||
limit: Final = customer_budget_limit(budget)
|
||||
return limit is None, (limit or 0) * (-1 if sort == "budget_desc" else 1)
|
||||
|
||||
|
||||
class _BudgetSortRow(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
entity_id: str
|
||||
budget_id: str | None
|
||||
budget_override_id: str | None
|
||||
budget_override_expires_at: datetime | None
|
||||
|
||||
|
||||
class ErrorCode(str, Enum):
|
||||
FREE_TIER_LIMIT = "free_tier_limit"
|
||||
CAPACITY_BUSY = "capacity_busy"
|
||||
TRANSACTION_TIMEOUT = "transaction_timeout"
|
||||
NOT_FOUND = "not_found"
|
||||
PARENT_NOT_FOUND = "parent_not_found"
|
||||
ALREADY_EXISTS = "already_exists"
|
||||
|
|
@ -371,7 +416,7 @@ def team_response(row: PrismaRecord, now: datetime) -> Team:
|
|||
|
||||
|
||||
def override_expiry(
|
||||
row: Team | CustomerMember | None, payload: TeamPatch | MemberPatch, override_id: str | None
|
||||
row: Team | CustomerMember | None, payload: TeamPatch | MemberFields, override_id: str | None
|
||||
) -> datetime | None:
|
||||
if override_id is None:
|
||||
return None
|
||||
|
|
@ -486,6 +531,12 @@ _MEMBER_INCLUDE: Final = {"budget": True, "budget_override": True}
|
|||
_ID: Final = TypeAdapter(str)
|
||||
|
||||
|
||||
def _search_filter(search: str, fields: tuple[str, ...]) -> Mapping[str, object]:
|
||||
if not search:
|
||||
return {}
|
||||
return {"OR": tuple({field: {"contains": escape_like(search), "mode": "insensitive"}} for field in fields)}
|
||||
|
||||
|
||||
def _member_ids(rows: Sequence[PrismaRecord]) -> tuple[str, ...]:
|
||||
return tuple(_ID.validate_python(row.model_dump()["customer_member_id"]) for row in rows)
|
||||
|
||||
|
|
@ -995,9 +1046,20 @@ class CustomerHierarchyStore:
|
|||
return await self.update_budget(current_id, payload.budget_override, actor)
|
||||
|
||||
async def update_member(self, member_id: str, payload: MemberPatch, actor: str) -> CustomerMember | Failure:
|
||||
if payload.parent_team_ids is not None:
|
||||
await self.lock_customer_identities()
|
||||
member: Final = await self.get_member(member_id)
|
||||
if isinstance(member, Failure):
|
||||
return member
|
||||
if payload.parent_team_ids is not None:
|
||||
teams: Final = await find_many_in(self.teams, "customer_team_id", payload.parent_team_ids)
|
||||
if len(teams) != len(payload.parent_team_ids):
|
||||
return Failure(ErrorCode.PARENT_NOT_FOUND)
|
||||
if any(team.model_dump()["parent_org_id"] != member.parent_org_id for team in teams):
|
||||
return Failure(ErrorCode.CUSTOMER_ORG_MISMATCH)
|
||||
added: Final = tuple(team for team in payload.parent_team_ids if team not in member.parent_team_ids)
|
||||
if await self.member_capacity_reached(added):
|
||||
return Failure(ErrorCode.FREE_TIER_LIMIT)
|
||||
selected: Final = await self.member_template_patch(member, payload)
|
||||
if isinstance(selected, Failure):
|
||||
return selected
|
||||
|
|
@ -1012,6 +1074,7 @@ class CustomerHierarchyStore:
|
|||
where={"customer_member_id": member_id},
|
||||
data={
|
||||
**alias_patch(payload),
|
||||
**({"parent_team_ids": payload.parent_team_ids} if payload.parent_team_ids is not None else {}),
|
||||
**({"budget_id": selected} if "budget_id" in payload.model_fields_set else {}),
|
||||
**(
|
||||
{
|
||||
|
|
@ -1028,16 +1091,7 @@ class CustomerHierarchyStore:
|
|||
return await self.get_member(member_id)
|
||||
|
||||
async def list_organizations(self, page: int, page_size: int, search: str = "") -> Page[Organization]:
|
||||
where: Final = (
|
||||
{
|
||||
"OR": tuple(
|
||||
{field: {"contains": escape_like(search), "mode": "insensitive"}}
|
||||
for field in ("alias", "customer_org_id")
|
||||
)
|
||||
}
|
||||
if search
|
||||
else {}
|
||||
)
|
||||
where: Final = _search_filter(search, ("alias", "customer_org_id"))
|
||||
count: Final = await self.organizations.count(where=where)
|
||||
rows: Final = await self.organizations.find_many(
|
||||
where=where,
|
||||
|
|
@ -1048,9 +1102,49 @@ class CustomerHierarchyStore:
|
|||
)
|
||||
return Page(await self.organization_responses(rows), count)
|
||||
|
||||
async def list_teams(self, parent_org_id: str | None, page: int, page_size: int) -> Page[Team]:
|
||||
async def list_teams(
|
||||
self, parent_org_id: str | None, page: int, page_size: int, sort: BudgetSort | None = None
|
||||
) -> Page[Team]:
|
||||
where: Final = {"parent_org_id": parent_org_id} if parent_org_id is not None else {}
|
||||
count: Final = await self.teams.count(where=where)
|
||||
if sort is not None:
|
||||
ids: Final = await self._budget_sort_ids(
|
||||
await self.db.query_raw(
|
||||
"SELECT customer_team_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at "
|
||||
'FROM "LiteLLM_CustomerTeamTable" WHERE ($1::text IS NULL OR parent_org_id = $1) '
|
||||
"ORDER BY created_at ASC, customer_team_id ASC",
|
||||
parent_org_id,
|
||||
),
|
||||
sort,
|
||||
page,
|
||||
page_size,
|
||||
)
|
||||
chunks: Final = tuple(
|
||||
[
|
||||
await self.teams.find_many(
|
||||
# bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids.
|
||||
where={"customer_team_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}},
|
||||
include=_MEMBER_INCLUDE,
|
||||
)
|
||||
for start in range(0, len(ids), IN_LIST_CHUNK_SIZE)
|
||||
]
|
||||
)
|
||||
rows_by_id: Final = {
|
||||
_ID.validate_python(row.model_dump()["customer_team_id"]): row for row in chain.from_iterable(chunks)
|
||||
}
|
||||
return Page(
|
||||
await customer_budget_usage(
|
||||
tuple(
|
||||
team_response(rows_by_id[identifier], self.now)
|
||||
for identifier in ids
|
||||
if identifier in rows_by_id
|
||||
),
|
||||
self.window_spend,
|
||||
self.now,
|
||||
self.settings,
|
||||
),
|
||||
count,
|
||||
)
|
||||
rows: Final = await self.teams.find_many(
|
||||
where=where,
|
||||
skip=(page - 1) * page_size,
|
||||
|
|
@ -1060,19 +1154,72 @@ class CustomerHierarchyStore:
|
|||
)
|
||||
return Page(
|
||||
await customer_budget_usage(
|
||||
tuple(team_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings
|
||||
tuple(team_response(row, self.now) for row in rows),
|
||||
self.window_spend,
|
||||
self.now,
|
||||
self.settings,
|
||||
),
|
||||
count,
|
||||
)
|
||||
|
||||
async def list_members(
|
||||
self, parent_team_id: str | None, page: int, page_size: int, *, parent_org_id: str | None = None
|
||||
self,
|
||||
parent_team_id: str | None,
|
||||
page: int,
|
||||
page_size: int,
|
||||
*,
|
||||
parent_org_id: str | None = None,
|
||||
search: str = "",
|
||||
sort: BudgetSort | None = None,
|
||||
) -> Page[CustomerMember]:
|
||||
where: Final[Mapping[str, object]] = {
|
||||
**({"parent_team_ids": {"has": parent_team_id}} if parent_team_id is not None else {}),
|
||||
**({"parent_org_id": parent_org_id} if parent_org_id is not None else {}),
|
||||
**_search_filter(search, ("alias", "customer_member_id")),
|
||||
}
|
||||
count: Final = await self.members.count(where=where)
|
||||
if sort is not None:
|
||||
ids: Final = await self._budget_sort_ids(
|
||||
await self.db.query_raw(
|
||||
"SELECT customer_member_id AS entity_id, budget_id, budget_override_id, budget_override_expires_at "
|
||||
'FROM "LiteLLM_CustomerMemberTable" WHERE ($1::text IS NULL OR parent_org_id = $1) '
|
||||
"AND ($2::text IS NULL OR parent_team_ids @> ARRAY[$2]::text[]) "
|
||||
"AND ($3 = '' OR alias ILIKE $3 OR customer_member_id ILIKE $3) "
|
||||
"ORDER BY created_at ASC, customer_member_id ASC",
|
||||
parent_org_id,
|
||||
parent_team_id,
|
||||
f"%{escape_like(search)}%" if search else "",
|
||||
),
|
||||
sort,
|
||||
page,
|
||||
page_size,
|
||||
)
|
||||
chunks: Final = tuple(
|
||||
[
|
||||
await self.members.find_many(
|
||||
# bounded-ok: Each slice has at most IN_LIST_CHUNK_SIZE ids.
|
||||
where={"customer_member_id": {"in": list(ids[start : start + IN_LIST_CHUNK_SIZE])}},
|
||||
include=_MEMBER_INCLUDE,
|
||||
)
|
||||
for start in range(0, len(ids), IN_LIST_CHUNK_SIZE)
|
||||
]
|
||||
)
|
||||
rows_by_id: Final = {
|
||||
_ID.validate_python(row.model_dump()["customer_member_id"]): row for row in chain.from_iterable(chunks)
|
||||
}
|
||||
return Page(
|
||||
await customer_budget_usage(
|
||||
tuple(
|
||||
member_response(rows_by_id[identifier], self.now)
|
||||
for identifier in ids
|
||||
if identifier in rows_by_id
|
||||
),
|
||||
self.window_spend,
|
||||
self.now,
|
||||
self.settings,
|
||||
),
|
||||
count,
|
||||
)
|
||||
rows: Final = await self.members.find_many(
|
||||
where=where,
|
||||
skip=(page - 1) * page_size,
|
||||
|
|
@ -1082,11 +1229,36 @@ class CustomerHierarchyStore:
|
|||
)
|
||||
return Page(
|
||||
await customer_budget_usage(
|
||||
tuple(member_response(row, self.now) for row in rows), self.window_spend, self.now, self.settings
|
||||
tuple(member_response(row, self.now) for row in rows),
|
||||
self.window_spend,
|
||||
self.now,
|
||||
self.settings,
|
||||
),
|
||||
count,
|
||||
)
|
||||
|
||||
async def _budget_sort_ids(self, rows: object, sort: BudgetSort, page: int, page_size: int) -> tuple[str, ...]:
|
||||
candidates: Final = TypeAdapter(tuple[_BudgetSortRow, ...]).validate_python(rows)
|
||||
references: Final = chain.from_iterable((row.budget_id, row.budget_override_id) for row in candidates)
|
||||
budgets: Final = {
|
||||
_ID.validate_python(row.model_dump()["budget_id"]): row.model_dump()
|
||||
for row in await find_many_in(self.budgets, "budget_id", (ref for ref in references if ref is not None))
|
||||
}
|
||||
|
||||
def key(row: _BudgetSortRow) -> tuple[bool, float]:
|
||||
template: Final = related_budget(budgets.get(row.budget_id or ""), self.now)
|
||||
override: Final = related_budget(
|
||||
budgets.get(row.budget_override_id or ""),
|
||||
self.now,
|
||||
template.max_budget if template is not None else None,
|
||||
)
|
||||
return _budget_sort_key(
|
||||
effective_member_budget(template, override, row.budget_override_expires_at, self.now), sort
|
||||
)
|
||||
|
||||
ordered: Final = sorted(candidates, key=key)
|
||||
return tuple(row.entity_id for row in ordered[(page - 1) * page_size : page * page_size])
|
||||
|
||||
async def _delete_unused_budgets(self, budget_ids: Sequence[str]) -> None:
|
||||
where: Final[Mapping[str, object]] = {field: {"none": {}} for field in _BUDGET_REFERENCE_FIELDS}
|
||||
await delete_many_in(
|
||||
|
|
@ -1098,8 +1270,10 @@ class CustomerHierarchyStore:
|
|||
chunk_size=_DELETE_BATCH_SIZE,
|
||||
)
|
||||
|
||||
async def _delete_team_member_batch(self, team_id: str) -> bool:
|
||||
rows: Final = await self.members.find_many(where={"parent_team_ids": {"has": team_id}}, take=_DELETE_BATCH_SIZE)
|
||||
async def _delete_team_member_batch(self, team_ids: tuple[str, ...]) -> bool:
|
||||
rows: Final = await self.members.find_many(
|
||||
where={"parent_team_ids": {"has_some": list(team_ids)}}, take=_DELETE_BATCH_SIZE
|
||||
)
|
||||
members: Final = tuple(_MemberDeletionRow.model_validate(row) for row in rows)
|
||||
if not members:
|
||||
return False
|
||||
|
|
@ -1111,9 +1285,9 @@ class CustomerHierarchyStore:
|
|||
await self._delete_unused_budgets(budget_ids)
|
||||
return True
|
||||
|
||||
async def _delete_team_members(self, team_id: str) -> None:
|
||||
async def _delete_team_members(self, team_ids: tuple[str, ...]) -> None:
|
||||
while True:
|
||||
if not await self._delete_team_member_batch(team_id):
|
||||
if not await self._delete_team_member_batch(team_ids):
|
||||
return
|
||||
|
||||
async def _delete_organization_team_batch(self, organization_id: str) -> bool:
|
||||
|
|
@ -1121,9 +1295,8 @@ class CustomerHierarchyStore:
|
|||
teams: Final = tuple(_TeamDeletionRow.model_validate(row) for row in rows)
|
||||
if not teams:
|
||||
return False
|
||||
for team in teams:
|
||||
await self._delete_team_members(team.customer_team_id)
|
||||
team_ids: Final = tuple(team.customer_team_id for team in teams)
|
||||
await self._delete_team_members(team_ids)
|
||||
team_budget_ids: Final = chain.from_iterable((team.budget_id, team.budget_override_id) for team in teams)
|
||||
budget_ids: Final = tuple(budget_id for budget_id in team_budget_ids if budget_id is not None)
|
||||
await delete_many_in(self.teams, "customer_team_id", team_ids, atomicity="caller_transaction")
|
||||
|
|
@ -1154,11 +1327,12 @@ class CustomerHierarchyStore:
|
|||
return None
|
||||
|
||||
async def delete_team(self, team_id: str, *, cascade: bool = False) -> None | Failure:
|
||||
await self.lock_customer_identities()
|
||||
team: Final = await self.get_team(team_id)
|
||||
if isinstance(team, Failure):
|
||||
return team
|
||||
if cascade:
|
||||
await self._delete_team_members(team_id)
|
||||
await self._delete_team_members((team_id,))
|
||||
elif await self.members.count(where={"parent_team_ids": {"has": team_id}}):
|
||||
return Failure(ErrorCode.HAS_CHILDREN)
|
||||
await self.teams.delete(where={"customer_team_id": team_id})
|
||||
|
|
@ -1166,6 +1340,7 @@ class CustomerHierarchyStore:
|
|||
return None
|
||||
|
||||
async def delete_organization(self, organization_id: str, *, cascade: bool = False) -> None | Failure:
|
||||
await self.lock_customer_identities()
|
||||
organization: Final = await self.get_organization(organization_id)
|
||||
if isinstance(organization, Failure):
|
||||
return organization
|
||||
|
|
@ -1196,7 +1371,13 @@ class CustomerHierarchy:
|
|||
*,
|
||||
operation_type: TransactionOperation = TransactionOperation.READ_WRITE,
|
||||
) -> T | Failure:
|
||||
from prisma.errors import DataError, ForeignKeyViolationError, RecordNotFoundError, UniqueViolationError
|
||||
from prisma.errors import (
|
||||
DataError,
|
||||
ForeignKeyViolationError,
|
||||
RecordNotFoundError,
|
||||
TransactionExpiredError,
|
||||
UniqueViolationError,
|
||||
)
|
||||
|
||||
try:
|
||||
async with self.db.tx() as tx:
|
||||
|
|
@ -1205,6 +1386,8 @@ class CustomerHierarchy:
|
|||
return Failure(ErrorCode.ALREADY_EXISTS)
|
||||
except RecordNotFoundError:
|
||||
return Failure(ErrorCode.NOT_FOUND)
|
||||
except TransactionExpiredError:
|
||||
return Failure(ErrorCode.TRANSACTION_TIMEOUT)
|
||||
except ForeignKeyViolationError:
|
||||
return Failure(
|
||||
ErrorCode.HAS_CHILDREN if operation_type is TransactionOperation.DELETE else ErrorCode.PARENT_NOT_FOUND
|
||||
|
|
@ -1278,15 +1461,24 @@ class CustomerHierarchy:
|
|||
return await self.transaction(lambda store: store.list_organizations(page, page_size, search))
|
||||
|
||||
async def list_teams(
|
||||
self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50
|
||||
self, parent_org_id: str | None = None, page: int = 1, page_size: int = 50, sort: BudgetSort | None = None
|
||||
) -> Page[Team] | Failure:
|
||||
return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size))
|
||||
return await self.transaction(lambda store: store.list_teams(parent_org_id, page, page_size, sort))
|
||||
|
||||
async def list_members(
|
||||
self, parent_team_id: str | None = None, page: int = 1, page_size: int = 50, *, parent_org_id: str | None = None
|
||||
self,
|
||||
parent_team_id: str | None = None,
|
||||
page: int = 1,
|
||||
page_size: int = 50,
|
||||
*,
|
||||
parent_org_id: str | None = None,
|
||||
search: str = "",
|
||||
sort: BudgetSort | None = None,
|
||||
) -> Page[CustomerMember] | Failure:
|
||||
return await self.transaction(
|
||||
lambda store: store.list_members(parent_team_id, page, page_size, parent_org_id=parent_org_id)
|
||||
lambda store: store.list_members(
|
||||
parent_team_id, page, page_size, parent_org_id=parent_org_id, search=search, sort=sort
|
||||
)
|
||||
)
|
||||
|
||||
async def add_template(self, organization_id: str, payload: BudgetTemplate, actor: str) -> Organization | Failure:
|
||||
|
|
|
|||
|
|
@ -460,6 +460,8 @@ async def _resolve_member_budget_id(
|
|||
changes reach them; ``/team/member_update`` clones that row on first write.
|
||||
A lone ``budget_duration`` clones the default with the reset window
|
||||
overridden, or creates a window-only budget when there is no team default.
|
||||
``allowed_models`` with no cap creates a budget that takes the default's reset
|
||||
window and leaves the cap to the default.
|
||||
With nothing set the member gets no budget, though ``add_new_member`` still writes its membership row.
|
||||
"""
|
||||
has_explicit_limit: Final = max_budget_in_team is not None or allowed_models is not None
|
||||
|
|
@ -493,14 +495,34 @@ async def _resolve_member_budget_id(
|
|||
budget_data["max_budget"] = max_budget_in_team
|
||||
if allowed_models is not None:
|
||||
budget_data["allowed_models"] = allowed_models
|
||||
if budget_duration is not None:
|
||||
budget_data["budget_duration"] = budget_duration
|
||||
budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=budget_duration)
|
||||
member_budget_duration: Final = budget_duration or (
|
||||
await _team_default_budget_duration(prisma_client, default_team_budget_id, tx)
|
||||
if max_budget_in_team is None and default_team_budget_id is not None
|
||||
else None
|
||||
)
|
||||
if member_budget_duration is not None:
|
||||
budget_data["budget_duration"] = member_budget_duration
|
||||
budget_data["budget_reset_at"] = get_budget_reset_time(budget_duration=member_budget_duration)
|
||||
budget_table: Final[_PrismaBudgetTable] = _budget_table(prisma_client, tx)
|
||||
response: Final = await budget_table.create(data=budget_data)
|
||||
return response.budget_id
|
||||
|
||||
|
||||
async def _team_default_budget_duration(
|
||||
prisma_client: PrismaClient, default_team_budget_id: str, tx: MemberWriteTx | None
|
||||
) -> str | None:
|
||||
"""The reset period a member with models of their own and no cap of their own must carry.
|
||||
|
||||
Their cap is read from the team default at auth time, but spend is only reset for a
|
||||
member whose own budget row has a reset date.
|
||||
"""
|
||||
default_budget: Final = await _budget_table(prisma_client, tx).find_unique(
|
||||
where={"budget_id": default_team_budget_id}
|
||||
)
|
||||
duration: Final = default_budget.model_dump().get("budget_duration") if default_budget is not None else None
|
||||
return duration if isinstance(duration, str) and duration else None
|
||||
|
||||
|
||||
async def _append_team_id_if_absent(
|
||||
prisma_client: PrismaClient, user_id: str, team_id: str, tx: MemberWriteTx | None = None
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -598,6 +598,13 @@ def _fal_target(endpoint: str) -> httpx.URL:
|
|||
)
|
||||
|
||||
|
||||
def _fal_upstream_endpoint(target: httpx.URL) -> str:
|
||||
base_url: Final = httpx.URL(fal_ai_queue_base())
|
||||
base_path: Final = base_url.copy_with(path=base_url.path).raw_path.partition(b"?")[0].decode("ascii").strip("/")
|
||||
upstream_path: Final = target.raw_path.partition(b"?")[0].decode("ascii").lstrip("/")
|
||||
return upstream_path.removeprefix(f"{base_path}/") if base_path else upstream_path
|
||||
|
||||
|
||||
@router.api_route(
|
||||
"/fal_ai/{endpoint:path}",
|
||||
methods=["GET", "POST", "PUT", "DELETE", "PATCH"],
|
||||
|
|
@ -619,10 +626,14 @@ async def fal_ai_proxy_route(
|
|||
status_code=401,
|
||||
detail="FAL_AI_API_KEY is not set and no fal_ai pass-through deployment credentials are configured",
|
||||
)
|
||||
if "/requests/" not in endpoint and fal_ai_passthrough_cost(endpoint, await read_request_body(request)) is None:
|
||||
upstream_endpoint: Final = _fal_upstream_endpoint(updated_url)
|
||||
if (
|
||||
"/requests/" not in upstream_endpoint
|
||||
and fal_ai_passthrough_cost(upstream_endpoint, await read_request_body(request)) is None
|
||||
):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"fal_ai/{endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai",
|
||||
detail=f"fal_ai/{upstream_endpoint} has no pricing entry for this request; only priced Fal requests can be submitted through /fal_ai",
|
||||
)
|
||||
endpoint_func: Final = create_pass_through_route(
|
||||
endpoint=endpoint,
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ def _build_db_connection_url_params(
|
|||
socket_timeout: float | None = None,
|
||||
disable_prepared_statements: bool = False,
|
||||
extra_params: dict | None = None,
|
||||
) -> dict:
|
||||
) -> dict[str, str | int | float]:
|
||||
"""Build the Prisma DATABASE_URL query params controlling connection pool behavior.
|
||||
|
||||
`connect_timeout` / `socket_timeout` map to the Prisma URL params of the same
|
||||
|
|
@ -91,7 +91,7 @@ def _build_db_connection_url_params(
|
|||
`extra_params` is an untyped passthrough — keys it provides win over the
|
||||
named arguments above, so it can be used to override any default we set here.
|
||||
"""
|
||||
params: Final[dict] = {
|
||||
params: Final[dict[str, str | int | float]] = {
|
||||
"connection_limit": connection_limit,
|
||||
}
|
||||
if pool_timeout is not None:
|
||||
|
|
@ -1637,11 +1637,13 @@ def run_server(
|
|||
db_connection_timeout = LiteLLMDatabaseConnectionPool.database_connection_pool_timeout.value
|
||||
|
||||
if os.getenv("DATABASE_URL", None) is not None or os.getenv("DIRECT_URL", None) is not None:
|
||||
from litellm.proxy.db.connection_budget import ConnectionBudget
|
||||
from litellm.proxy.db.db_url_settings import (
|
||||
DISABLE_PREPARED_STATEMENTS_ENV_VAR,
|
||||
add_missing_query_params,
|
||||
idle_lifetime_params,
|
||||
postgres_connection_budget_message,
|
||||
prisma_default_params,
|
||||
reader_shareable_params,
|
||||
translate_libpq_ssl_params,
|
||||
unsupported_db_scheme,
|
||||
|
|
@ -1676,7 +1678,13 @@ def run_server(
|
|||
disable_prepared_statements=disable_prepared_statements,
|
||||
extra_params=db_extra_connection_params,
|
||||
)
|
||||
writer_budget: Final = ConnectionBudget.of(connection_url_params["connection_limit"])
|
||||
rust_writer_connections: Final = 0 if writer_budget is None else writer_budget.rust
|
||||
prisma_url_params: Final = (
|
||||
connection_url_params if writer_budget is None else writer_budget.prisma_params(connection_url_params)
|
||||
)
|
||||
lifetime_params: Final = idle_lifetime_params(general_settings.get("database_max_idle_connection_lifetime"))
|
||||
prisma_defaults: Final = prisma_default_params(lifetime_params, rust_writer_connections)
|
||||
if os.getenv("DATABASE_URL", None) is not None:
|
||||
database_url = get_secret("DATABASE_URL", default_value=None)
|
||||
resolved_url: Final[str | None] = str(database_url) if database_url else None
|
||||
|
|
@ -1693,10 +1701,10 @@ def run_server(
|
|||
)
|
||||
modified_url = append_query_params(
|
||||
writer_url,
|
||||
connection_url_params,
|
||||
dict(prisma_url_params),
|
||||
)
|
||||
os.environ["DATABASE_URL"] = translate_libpq_ssl_params(
|
||||
add_missing_query_params(modified_url, lifetime_params)
|
||||
add_missing_query_params(modified_url, prisma_defaults)
|
||||
)
|
||||
if os.getenv("DIRECT_URL", None) is not None:
|
||||
database_url = os.getenv("DIRECT_URL")
|
||||
|
|
@ -1710,6 +1718,14 @@ def run_server(
|
|||
# on the writer. Anything pinned on the replica URL wins, unlike the
|
||||
# writer where the config is applied on top.
|
||||
read_replica_url: Final[str | None] = os.getenv("DATABASE_URL_READ_REPLICA")
|
||||
reader_budget: Final = (
|
||||
ConnectionBudget.of(
|
||||
_url_query_value(read_replica_url, "connection_limit") or connection_url_params["connection_limit"]
|
||||
)
|
||||
if read_replica_url
|
||||
else None
|
||||
)
|
||||
reader_prisma_limit: Final = None if reader_budget is None else reader_budget.prisma_limit
|
||||
if read_replica_url:
|
||||
reader_options: Final[str] = _pg_options_with_timeouts(
|
||||
_url_query_value(read_replica_url, "options"),
|
||||
|
|
@ -1717,27 +1733,34 @@ def run_server(
|
|||
db_lock_timeout,
|
||||
db_idle_in_transaction_timeout,
|
||||
)
|
||||
reader_url: Final = (
|
||||
_with_query_value(read_replica_url, "options", reader_options)
|
||||
if reader_options
|
||||
else read_replica_url
|
||||
)
|
||||
os.environ["DATABASE_URL_READ_REPLICA"] = translate_libpq_ssl_params(
|
||||
add_missing_query_params(
|
||||
add_missing_query_params(
|
||||
_with_query_value(read_replica_url, "options", reader_options)
|
||||
if reader_options
|
||||
else read_replica_url,
|
||||
reader_shareable_params(connection_url_params),
|
||||
reader_url
|
||||
if reader_prisma_limit is None
|
||||
else _with_query_value(reader_url, "connection_limit", str(reader_prisma_limit)),
|
||||
reader_shareable_params(prisma_url_params),
|
||||
),
|
||||
lifetime_params,
|
||||
prisma_defaults,
|
||||
)
|
||||
)
|
||||
print_startup_message(
|
||||
postgres_connection_budget_message(
|
||||
writer_limit=_url_query_value(os.getenv("DATABASE_URL"), "connection_limit")
|
||||
or f"{db_connection_pool_limit}",
|
||||
rust_writer_limit=rust_writer_connections,
|
||||
reader_limit=(
|
||||
_url_query_value(os.getenv("DATABASE_URL_READ_REPLICA"), "connection_limit")
|
||||
or f"{db_connection_pool_limit}"
|
||||
)
|
||||
if read_replica_url
|
||||
else None,
|
||||
rust_reader_limit=0 if reader_budget is None else reader_budget.rust,
|
||||
num_workers=f"{1 if run_hypercorn else num_workers}",
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -213,7 +213,7 @@ from litellm.router_strategy.auto_router.auto_router_tuning_baseline import (
|
|||
snapshot_tuning_baselines,
|
||||
tuning_limit_violation,
|
||||
)
|
||||
from litellm.rust_bridge.catalog import Target
|
||||
from litellm.rust_bridge.catalog import NATIVE_MESSAGES_TARGETS, Target
|
||||
from litellm.types.utils import (
|
||||
PRICING_OVERRIDES_KEY,
|
||||
CallTypes,
|
||||
|
|
@ -681,6 +681,7 @@ from litellm.proxy.config_resolvers.settings_rules import (
|
|||
from litellm.proxy.config_resolvers.sso import SSO_FIELD_ENV_VARS
|
||||
from litellm.proxy.container_endpoints.endpoints import router as container_router
|
||||
from litellm.proxy.credential_endpoints.endpoints import router as credential_router
|
||||
from litellm.proxy.db.connection_budget import RUST_RESERVED_CONNECTIONS
|
||||
from litellm.proxy.db.create_views import SupportsRawQueries
|
||||
from litellm.proxy.db.db_transaction_queue.pod_lock_manager import PodLockManager
|
||||
from litellm.proxy.db.db_transaction_queue.spend_log_cleanup import (
|
||||
|
|
@ -1620,7 +1621,12 @@ async def _connect_to_count_stored_values(resources: AsyncExitStack) -> Supports
|
|||
async def _own_database(resources: AsyncExitStack, client: PrismaClient | None) -> database_runtime.Database | None:
|
||||
if client is None:
|
||||
return None
|
||||
database: Final = await database_runtime.Database.open(client)
|
||||
database: Final = await database_runtime.Database.open(
|
||||
client,
|
||||
os.environ["DATABASE_URL"],
|
||||
os.environ.get("DATABASE_URL_READ_REPLICA") or None,
|
||||
RUST_RESERVED_CONNECTIONS,
|
||||
)
|
||||
resources.push_async_callback(database.aclose)
|
||||
return database
|
||||
|
||||
|
|
@ -7446,7 +7452,7 @@ class ProxyConfig:
|
|||
if key == "request_timeout":
|
||||
litellm.request_timeout_explicitly_set = True
|
||||
if key in {"s3_audit_callback_params", "s3_callback_params"}:
|
||||
from litellm.integrations.s3_v3 import S3Logger as S3V2Logger
|
||||
from litellm.integrations.s3_v2 import S3Logger as S3V2Logger
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
_in_memory_loggers,
|
||||
)
|
||||
|
|
@ -16124,7 +16130,6 @@ def _count_context_management(
|
|||
|
||||
|
||||
_GATEWAY_EDIT_ROUTES: Final = frozenset({Target.CHAT_COMPLETIONS, Target.RESPONSES})
|
||||
_MESSAGES_ROUTES: Final = frozenset({Target.MESSAGES, Target.MESSAGES_PASSTHROUGH})
|
||||
|
||||
|
||||
def _messages_route(deployment: _TokenCountDeployment, request_model: str) -> Target:
|
||||
|
|
@ -16663,7 +16668,7 @@ async def _count_request_tokens(
|
|||
|
||||
local_view: Final = (
|
||||
None
|
||||
if route not in _MESSAGES_ROUTES or context_edits is None or request_messages is None
|
||||
if route not in NATIVE_MESSAGES_TARGETS or context_edits is None or request_messages is None
|
||||
else await _bridged_request_view(
|
||||
request_messages,
|
||||
system,
|
||||
|
|
|
|||
|
|
@ -85,6 +85,8 @@ model LiteLLM_CustomerTeamTable {
|
|||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String
|
||||
@@index([parent_org_id])
|
||||
@@index([budget_id])
|
||||
@@index([budget_override_id])
|
||||
@@index([spend])
|
||||
}
|
||||
|
||||
|
|
@ -107,6 +109,9 @@ model LiteLLM_CustomerMemberTable {
|
|||
updated_at DateTime @default(now()) @updatedAt
|
||||
updated_by String
|
||||
@@index([parent_org_id])
|
||||
@@index([parent_team_ids], type: Gin)
|
||||
@@index([budget_id])
|
||||
@@index([budget_override_id])
|
||||
@@index([spend])
|
||||
}
|
||||
|
||||
|
|
@ -2308,7 +2313,7 @@ enum AccountingDeliveryState {
|
|||
}
|
||||
|
||||
model LiteLLM_SpendLogDelivery {
|
||||
delivery_id String @id @default(dbgenerated("(gen_random_uuid())::text"))
|
||||
delivery_id String @id @default(uuid())
|
||||
request_id String
|
||||
state AccountingDeliveryState @default(pending)
|
||||
accounting_payload Json
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from litellm.litellm_core_utils.duration_parser import (
|
|||
duration_in_seconds,
|
||||
get_annual_budget_window_start,
|
||||
is_annual_budget_duration,
|
||||
normalize_budget_window_start,
|
||||
)
|
||||
from litellm.litellm_core_utils.duration_parser import (
|
||||
get_budget_window_start as calendar_budget_window_start,
|
||||
|
|
@ -30,7 +31,6 @@ from litellm.litellm_core_utils.duration_parser import (
|
|||
from litellm.litellm_core_utils.llm_cost_calc.tiered_pricing import select_tier_for_input, tier_rate
|
||||
from litellm.models.budget import BudgetLimit
|
||||
from litellm.proxy._types import (
|
||||
Litellm_EntityType,
|
||||
LiteLLM_TeamMembership,
|
||||
LiteLLM_TeamTable,
|
||||
LiteLLM_UserTable,
|
||||
|
|
@ -60,6 +60,11 @@ from litellm.proxy.spend_tracking.input_tokens import (
|
|||
models_counted_for,
|
||||
openai_chat_pdf_file_tokens,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.native_budgets import (
|
||||
new_reservation_id,
|
||||
reserve_native,
|
||||
settle_native_reservation,
|
||||
)
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.proxy.utils import PrismaClient, ProxyLogging
|
||||
from litellm.redis.stores.spend_counters import (
|
||||
|
|
@ -145,21 +150,6 @@ def _customer_window_counter(
|
|||
)
|
||||
|
||||
|
||||
_COUNTER_ENTITY_TYPES: Final[Mapping[str, str]] = {
|
||||
"Key": Litellm_EntityType.KEY.value,
|
||||
"Team": Litellm_EntityType.TEAM.value,
|
||||
"TeamMember": Litellm_EntityType.TEAM_MEMBER.value,
|
||||
"User": Litellm_EntityType.USER.value,
|
||||
"EndUser": Litellm_EntityType.END_USER.value,
|
||||
"Tag": Litellm_EntityType.TAG.value,
|
||||
"Model access group": Litellm_EntityType.MODEL_ACCESS_GROUP.value,
|
||||
"Organization": Litellm_EntityType.ORGANIZATION.value,
|
||||
"Project": Litellm_EntityType.PROJECT.value,
|
||||
"customer_org": Litellm_EntityType.CUSTOMER_ORG.value,
|
||||
"customer_team": Litellm_EntityType.CUSTOMER_TEAM.value,
|
||||
"customer_member": Litellm_EntityType.CUSTOMER_MEMBER.value,
|
||||
}
|
||||
|
||||
_CACHED_VALUE: Final = TypeAdapter(object)
|
||||
_WINDOW_OBJECT: Final = TypeAdapter(Mapping[str, object], config=ConfigDict(hide_input_in_errors=True))
|
||||
|
||||
|
|
@ -203,7 +193,11 @@ def get_reserved_counter_keys(budget_reservation: dict | None) -> set:
|
|||
_lease_renewals: Final[set[asyncio.Task[None]]] = set() # mutable-ok: asyncio only weak-refs pending tasks
|
||||
|
||||
|
||||
def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], counter_keys: frozenset[str]) -> None:
|
||||
def start_reservation_lease_renewal(
|
||||
budget_reservation: Mapping[str, object],
|
||||
counter_keys: frozenset[str],
|
||||
request_task: asyncio.Task[object] | None = None,
|
||||
) -> None:
|
||||
"""A reservation lives inside spend counter keys that expire on their Redis TTL. Renew the TTL
|
||||
while the request is in flight so a request longer than the TTL does not drop its
|
||||
reservation and admit concurrent requests against the DB floor on any worker."""
|
||||
|
|
@ -217,7 +211,7 @@ def _start_reservation_lease_renewal(budget_reservation: Mapping[str, object], c
|
|||
budget_reservation=budget_reservation,
|
||||
counter_keys=counter_keys,
|
||||
interval=redis_ttl / 2,
|
||||
request_task=asyncio.current_task(),
|
||||
request_task=request_task or asyncio.current_task(),
|
||||
)
|
||||
)
|
||||
_lease_renewals.add(task)
|
||||
|
|
@ -243,80 +237,6 @@ async def _renew_reservation_lease(
|
|||
await refresh_spend_counter_ttl(counter_key=counter_key)
|
||||
|
||||
|
||||
def _key_reservation_should_release_for_throttle(counter_key: str, valid_token: UserAPIKeyAuth | None) -> bool:
|
||||
"""
|
||||
Whether an over-budget key's own ``max_budget`` reservation should be
|
||||
released rather than blocked, because the key opted into throttling: the
|
||||
rate limiter slows it instead. Only the key's own ``max_budget`` counter is
|
||||
exempt; team/user/window counters still enforce normally, and under-budget
|
||||
requests never reach this branch so their concurrent-overspend protection is
|
||||
untouched.
|
||||
"""
|
||||
if valid_token is None:
|
||||
return False
|
||||
return counter_key == key_counter(str(valid_token.token)) and should_throttle_budget_exceeded(valid_token)
|
||||
|
||||
|
||||
async def _apply_over_budget_reservation_policy(
|
||||
counter: _BudgetCounter,
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
entry: dict[str, float | str],
|
||||
applied_entries: list[dict[str, float | str]],
|
||||
reservation_cost: float,
|
||||
current_spend: float,
|
||||
fail_closed_budget_enforcement: bool = False,
|
||||
) -> float:
|
||||
"""
|
||||
Decide what to do when a counter is over budget, and return the reservation
|
||||
cost to carry into the next counter. Three outcomes: an over-budget key that
|
||||
opted into throttling releases its own reservation (the rate limiter slows
|
||||
it) and keeps the cost; a partially-remaining budget resizes the reservation
|
||||
down to what is left, unless strict enforcement is on, because the known
|
||||
estimate already does not fit; anything else hard-blocks by raising.
|
||||
"""
|
||||
if _key_reservation_should_release_for_throttle(counter.counter_key, valid_token):
|
||||
await _release_applied_entries_best_effort(entries=[entry], default_reserved_cost=reservation_cost)
|
||||
applied_entries.remove(entry)
|
||||
return reservation_cost
|
||||
|
||||
remaining_before_reservation: Final = counter.max_budget - (current_spend - reservation_cost)
|
||||
if remaining_before_reservation <= 1e-12:
|
||||
_raise_counter_budget_exceeded(counter=counter, current_cost=current_spend)
|
||||
if fail_closed_budget_enforcement and current_spend - counter.max_budget > 1e-12:
|
||||
_raise_counter_budget_exceeded(
|
||||
counter=counter,
|
||||
current_cost=current_spend - reservation_cost,
|
||||
estimated_cost=reservation_cost,
|
||||
)
|
||||
await _resize_applied_reservation(
|
||||
entries=applied_entries,
|
||||
current_reserved_cost=reservation_cost,
|
||||
new_reserved_cost=remaining_before_reservation,
|
||||
)
|
||||
return remaining_before_reservation
|
||||
|
||||
|
||||
def _raise_counter_budget_exceeded(
|
||||
counter: _BudgetCounter,
|
||||
current_cost: float,
|
||||
estimated_cost: float | None = None,
|
||||
) -> NoReturn:
|
||||
estimate_detail: Final = "" if estimated_cost is None else f"Estimated request cost: {estimated_cost}, "
|
||||
raise litellm.BudgetExceededError(
|
||||
current_cost=current_cost,
|
||||
max_budget=counter.max_budget,
|
||||
message=(
|
||||
"Budget has been exceeded! "
|
||||
f"{counter.entity_type}={counter.entity_id} "
|
||||
f"Current cost: {current_cost}, "
|
||||
f"{estimate_detail}"
|
||||
f"Max budget: {counter.max_budget}"
|
||||
),
|
||||
entity_type=_COUNTER_ENTITY_TYPES.get(counter.entity_type),
|
||||
entity_id=counter.spend_log_entity_id or counter.entity_id,
|
||||
)
|
||||
|
||||
|
||||
async def reserve_budget_for_request(
|
||||
request_body: dict,
|
||||
route: str,
|
||||
|
|
@ -365,7 +285,7 @@ async def reserve_budget_for_request(
|
|||
path_params=path_params,
|
||||
)
|
||||
|
||||
reservation_cost = estimate_request_max_cost(
|
||||
reservation_cost: Final = estimate_request_max_cost(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
|
|
@ -377,51 +297,64 @@ async def reserve_budget_for_request(
|
|||
# In that case we fall back to read-time enforcement only.
|
||||
if reservation_cost is None or reservation_cost <= 0:
|
||||
return None
|
||||
|
||||
applied_entries: Final[list[dict[str, float | str]]] = []
|
||||
try:
|
||||
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
|
||||
reservable: Final = await _initialize_reservation_counters(
|
||||
counters=counters,
|
||||
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
|
||||
)
|
||||
reservation_cost = await _reserve_reservable_counters(
|
||||
reservable=reservable,
|
||||
valid_token=valid_token,
|
||||
applied_entries=applied_entries,
|
||||
reservation_cost=reservation_cost,
|
||||
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
|
||||
)
|
||||
except Exception:
|
||||
await _release_applied_entries_best_effort(
|
||||
entries=applied_entries,
|
||||
default_reserved_cost=reservation_cost,
|
||||
)
|
||||
raise
|
||||
|
||||
if not applied_entries:
|
||||
return None
|
||||
|
||||
input_cost: Final = estimate_request_input_cost(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
return await _native_budget_plan(
|
||||
counters=counters,
|
||||
valid_token=valid_token,
|
||||
reservation_cost=reservation_cost,
|
||||
input_token_counts=input_token_counts,
|
||||
path_params=path_params,
|
||||
input_cost=estimate_request_input_cost(
|
||||
request_body=request_body,
|
||||
route=route,
|
||||
llm_router=llm_router,
|
||||
input_token_counts=input_token_counts,
|
||||
path_params=path_params,
|
||||
),
|
||||
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
|
||||
)
|
||||
budget_reservation: Final = {
|
||||
"reserved_cost": reservation_cost,
|
||||
"entries": applied_entries,
|
||||
"finalized": False,
|
||||
"callback_bound": False,
|
||||
"input_cost": min(float(input_cost or 0.0), reservation_cost),
|
||||
"input_tokens": max(input_token_counts.values(), default=None),
|
||||
}
|
||||
_start_reservation_lease_renewal(
|
||||
budget_reservation=budget_reservation,
|
||||
counter_keys=frozenset(get_reserved_counter_keys(budget_reservation=budget_reservation)),
|
||||
|
||||
|
||||
async def _native_budget_plan(
|
||||
counters: Sequence[_BudgetCounter],
|
||||
valid_token: UserAPIKeyAuth,
|
||||
reservation_cost: float,
|
||||
input_token_counts: Mapping[str, int],
|
||||
input_cost: float | None,
|
||||
fail_closed_budget_enforcement: bool,
|
||||
) -> dict[str, object] | None: # mutable-ok: settlement stamps finalized on the reservation it returns
|
||||
with _counters_batch_scope(frozenset(counter.counter_key for counter in counters)):
|
||||
reservable: Final = await _initialize_reservation_counters(
|
||||
counters=counters,
|
||||
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
|
||||
)
|
||||
if not reservable:
|
||||
return None
|
||||
input_tokens: Final = max(input_token_counts.values(), default=None)
|
||||
return await reserve_native(
|
||||
{
|
||||
"counters": [
|
||||
{
|
||||
"counter_key": counter.counter_key,
|
||||
"max_budget": counter.max_budget,
|
||||
"fallback_spend": counter.fallback_spend,
|
||||
"entity_type": counter.entity_type,
|
||||
"entity_id": counter.entity_id,
|
||||
"spend_log_entity_id": counter.spend_log_entity_id,
|
||||
}
|
||||
for counter in reservable
|
||||
],
|
||||
"reservation_cost": float(reservation_cost),
|
||||
"input_cost": float(input_cost or 0.0),
|
||||
"input_tokens": input_tokens,
|
||||
"throttle_counter": key_counter(str(valid_token.token))
|
||||
if should_throttle_budget_exceeded(valid_token)
|
||||
else None,
|
||||
"fail_closed": fail_closed_budget_enforcement,
|
||||
"status_code": int(litellm.budget_exceeded_status_code),
|
||||
"reservation_id": new_reservation_id(),
|
||||
},
|
||||
valid_token,
|
||||
{counter.counter_key: counter.reset_at.isoformat() for counter in reservable if counter.reset_at is not None},
|
||||
)
|
||||
return budget_reservation
|
||||
|
||||
|
||||
async def reconcile_budget_reservation(
|
||||
|
|
@ -430,19 +363,11 @@ async def reconcile_budget_reservation(
|
|||
finalize: bool = True,
|
||||
apply_consistent: bool = True,
|
||||
) -> tuple[PendingSpendIncrement, ...]:
|
||||
"""Settle every reserved counter on ``actual_cost``. With ``apply_consistent`` False the adjustments for
|
||||
counters that still hold the reservation are returned instead of written, so the caller can pipeline them with
|
||||
its own increments and then call ``stamp_budget_reservation_actual_cost``."""
|
||||
if not budget_reservation or budget_reservation.get("finalized") is True:
|
||||
return ()
|
||||
|
||||
reserved_cost: Final = float(budget_reservation.get("reserved_cost") or 0.0)
|
||||
actual: Final = float(actual_cost or 0.0)
|
||||
pending: Final = await _set_reserved_entries_actual_cost(
|
||||
entries=budget_reservation.get("entries") or [],
|
||||
actual_cost=actual,
|
||||
default_reserved_cost=reserved_cost,
|
||||
apply_consistent=apply_consistent,
|
||||
_ = float(budget_reservation.get("reserved_cost") or 0.0)
|
||||
pending: Final = await settle_native_reservation(
|
||||
budget_reservation, float(actual_cost or 0.0), apply_consistent=apply_consistent
|
||||
)
|
||||
if finalize:
|
||||
budget_reservation["finalized"] = True
|
||||
|
|
@ -1088,121 +1013,6 @@ async def _initialize_reservation_counter(counter: _BudgetCounter) -> None:
|
|||
raise _CounterReservationUnavailable
|
||||
|
||||
|
||||
async def _reserve_reservable_counters(
|
||||
reservable: Sequence[_BudgetCounter],
|
||||
valid_token: UserAPIKeyAuth | None,
|
||||
applied_entries: list[dict[str, float | str]],
|
||||
reservation_cost: float,
|
||||
fail_closed_budget_enforcement: bool,
|
||||
) -> float:
|
||||
"""Charge the counters group by group (see ``_reservation_groups``), settling the over-budget policy on each
|
||||
group before the next is charged, and hand back the reservation cost the policy left standing."""
|
||||
current_spend_by_counter_key: Final = {
|
||||
counter.counter_key: await _get_current_counter_value(counter=counter) for counter in reservable
|
||||
}
|
||||
for group in _reservation_groups(
|
||||
counters=reservable,
|
||||
current_spend_by_counter_key=current_spend_by_counter_key,
|
||||
reservation_cost=reservation_cost,
|
||||
):
|
||||
charged_cost = reservation_cost
|
||||
entries = tuple(_counter_to_reservation_entry(counter=counter, reserved_cost=charged_cost) for counter in group)
|
||||
applied_entries.extend(entries)
|
||||
reserved_values = await _reserve_counters(counters=group, entries=entries, reservation_cost=charged_cost)
|
||||
if reserved_values is None:
|
||||
for entry in entries:
|
||||
applied_entries.remove(entry)
|
||||
if fail_closed_budget_enforcement:
|
||||
_raise_reservation_unavailable(counter_key=group[0].counter_key)
|
||||
continue
|
||||
for counter, entry, reserved_value in zip(group, entries, reserved_values):
|
||||
if entry not in applied_entries:
|
||||
continue
|
||||
if reserved_value is not None:
|
||||
current_spend = reserved_value - (charged_cost - reservation_cost)
|
||||
else:
|
||||
current_spend = current_spend_by_counter_key[counter.counter_key] + reservation_cost
|
||||
if current_spend > counter.max_budget:
|
||||
reservation_cost = await _apply_over_budget_reservation_policy(
|
||||
counter=counter,
|
||||
valid_token=valid_token,
|
||||
entry=entry,
|
||||
applied_entries=applied_entries,
|
||||
reservation_cost=reservation_cost,
|
||||
current_spend=current_spend,
|
||||
fail_closed_budget_enforcement=fail_closed_budget_enforcement,
|
||||
)
|
||||
return reservation_cost
|
||||
|
||||
|
||||
def _reservation_groups(
|
||||
counters: Sequence[_BudgetCounter],
|
||||
current_spend_by_counter_key: Mapping[str, float],
|
||||
reservation_cost: float,
|
||||
) -> tuple[tuple[_BudgetCounter, ...], ...]:
|
||||
"""Every counter the batch read says still has room for the estimate is charged in one pipeline. As soon as one
|
||||
does not, the counters are charged one at a time so the over-budget policy settles each before the next is
|
||||
touched, and a rejection charges nothing after it."""
|
||||
if not counters:
|
||||
return ()
|
||||
if all(
|
||||
current_spend_by_counter_key[counter.counter_key] + reservation_cost <= counter.max_budget
|
||||
for counter in counters
|
||||
):
|
||||
return (tuple(counters),)
|
||||
return tuple((counter,) for counter in counters)
|
||||
|
||||
|
||||
async def _reserve_counters(
|
||||
counters: Sequence[_BudgetCounter],
|
||||
entries: Sequence[dict[str, float | str]],
|
||||
reservation_cost: float,
|
||||
) -> tuple[float | None, ...] | None:
|
||||
"""One INCRBYFLOAT pipeline reserves every counter. When it fails each counter is dropped, and one that cannot
|
||||
be dropped is released instead in case its increment landed, so nothing is left to release by the caller."""
|
||||
from litellm.proxy.proxy_server import invalidate_spend_counter, run_spend_counter_pipeline
|
||||
|
||||
if not counters:
|
||||
return ()
|
||||
try:
|
||||
reserved: Final = await run_spend_counter_pipeline(
|
||||
pending=tuple(
|
||||
PendingSpendIncrement(counter_key=counter.counter_key, increment=reservation_cost)
|
||||
for counter in counters
|
||||
)
|
||||
)
|
||||
except Exception:
|
||||
verbose_proxy_logger.warning(
|
||||
"Skipping budget reservation for %s because spend counter reservation failed",
|
||||
tuple(counter.counter_key for counter in counters),
|
||||
exc_info=True,
|
||||
)
|
||||
for counter, entry in zip(counters, entries):
|
||||
try:
|
||||
await invalidate_spend_counter(counter_key=counter.counter_key)
|
||||
except Exception:
|
||||
verbose_proxy_logger.warning(
|
||||
"Failed to invalidate spend counter after budget reservation failure for %s",
|
||||
counter.counter_key,
|
||||
exc_info=True,
|
||||
)
|
||||
await _release_applied_entries_best_effort(
|
||||
entries=[entry],
|
||||
default_reserved_cost=reservation_cost,
|
||||
)
|
||||
return None
|
||||
return tuple(reserved) + (None,) * (len(counters) - len(reserved))
|
||||
|
||||
|
||||
async def _get_current_counter_value(counter: _BudgetCounter) -> float:
|
||||
from litellm.proxy.proxy_server import get_current_spend
|
||||
|
||||
return await get_current_spend(
|
||||
counter_key=counter.counter_key,
|
||||
fallback_spend=counter.fallback_spend,
|
||||
)
|
||||
|
||||
|
||||
def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_scope:
|
||||
"""Each counter is read once, then written, so one MGET up front serves every read in the loop."""
|
||||
from litellm.proxy.proxy_server import spend_counter_store
|
||||
|
|
@ -1210,176 +1020,17 @@ def _counters_batch_scope(counter_keys: frozenset[str]) -> spend_counter_batch_s
|
|||
return spend_counter_batch_scope(spend_counter_store, counter_keys=counter_keys)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _EntryAdjustment:
|
||||
entry: dict[str, float | str]
|
||||
counter_key: str
|
||||
target_adjustment: float
|
||||
adjustment: float
|
||||
actual_cost: float
|
||||
|
||||
|
||||
def _entry_settled_cost(entry: Mapping[str, object], actual_cost: float, settlement_time: datetime | None) -> float:
|
||||
reset_at: Final = _coerce_datetime(entry.get("window_reset_at"))
|
||||
return 0.0 if settlement_time is not None and reset_at is not None and reset_at <= settlement_time else actual_cost
|
||||
|
||||
|
||||
def _entry_adjustment(
|
||||
entry: dict[str, float | str],
|
||||
actual_cost: float,
|
||||
default_reserved_cost: float,
|
||||
settlement_time: datetime | None = None,
|
||||
) -> _EntryAdjustment | None:
|
||||
counter_key: Final = entry.get("counter_key")
|
||||
if counter_key is None:
|
||||
return None
|
||||
settled_cost: Final = _entry_settled_cost(entry, actual_cost, settlement_time)
|
||||
target_adjustment: Final = settled_cost - _get_entry_reserved_cost(
|
||||
entry=entry, default_reserved_cost=default_reserved_cost
|
||||
)
|
||||
adjustment: Final = target_adjustment - float(entry.get("applied_adjustment") or 0.0)
|
||||
if adjustment == 0:
|
||||
return None
|
||||
return _EntryAdjustment(
|
||||
entry=entry,
|
||||
counter_key=str(counter_key),
|
||||
target_adjustment=target_adjustment,
|
||||
adjustment=adjustment,
|
||||
actual_cost=settled_cost,
|
||||
)
|
||||
|
||||
|
||||
async def _set_reserved_entries_actual_cost(
|
||||
entries: list[dict],
|
||||
actual_cost: float,
|
||||
default_reserved_cost: float,
|
||||
reseed_on_inconsistent: bool = True,
|
||||
apply_consistent: bool = True,
|
||||
) -> tuple[PendingSpendIncrement, ...]:
|
||||
"""Every reserved counter is read from one MGET and the consistent adjustments go out in one pipeline, or are
|
||||
returned unwritten when ``apply_consistent`` is False. A counter that was flushed or reseeded since reservation
|
||||
is settled on its own after the pipeline."""
|
||||
from litellm.proxy.proxy_server import increment_spend_counters_pipeline
|
||||
|
||||
settlement_time: Final = datetime.now(timezone.utc) if reseed_on_inconsistent else None
|
||||
with _counters_batch_scope(frozenset(str(entry["counter_key"]) for entry in entries if "counter_key" in entry)):
|
||||
adjustments: Final = tuple(
|
||||
adjustment
|
||||
for entry in entries
|
||||
if (adjustment := _entry_adjustment(entry, actual_cost, default_reserved_cost, settlement_time)) is not None
|
||||
)
|
||||
consistent: Final = tuple(
|
||||
await asyncio.gather(
|
||||
*(
|
||||
_counter_can_apply_adjustment(counter_key=item.counter_key, adjustment=item.adjustment)
|
||||
for item in adjustments
|
||||
)
|
||||
)
|
||||
)
|
||||
inconsistent: Final = tuple(item for item, ok in zip(adjustments, consistent) if not ok)
|
||||
if inconsistent and not reseed_on_inconsistent:
|
||||
# Pre-call admission resize: the in-flight reservation cost is not yet
|
||||
# persisted, so the DB floor would discard it. Keep the original
|
||||
# fail-closed behavior (raise -> reserve_budget_for_request releases and
|
||||
# denies) rather than admitting against an inconsistent counter.
|
||||
raise RuntimeError(
|
||||
f"Cannot resize budget reservation against inconsistent counter {inconsistent[0].counter_key}"
|
||||
)
|
||||
applicable: Final = tuple(item for item, ok in zip(adjustments, consistent) if ok)
|
||||
applicable_pending: Final = tuple(
|
||||
PendingSpendIncrement(counter_key=item.counter_key, increment=item.adjustment) for item in applicable
|
||||
)
|
||||
if apply_consistent:
|
||||
await increment_spend_counters_pipeline(pending=applicable_pending)
|
||||
for item in inconsistent:
|
||||
await _reseed_reserved_entry(item=item, actual_cost=item.actual_cost)
|
||||
for item in adjustments if apply_consistent else inconsistent:
|
||||
item.entry["applied_adjustment"] = item.target_adjustment
|
||||
return () if apply_consistent else applicable_pending
|
||||
|
||||
|
||||
async def _reseed_reserved_entry(item: _EntryAdjustment, actual_cost: float) -> None:
|
||||
"""Post-call reconcile / release of a counter that was flushed, expired or reseeded between reservation and
|
||||
reconcile: the optimistic delta no longer applies, so reseed from the DB floor and add the settled cost, since
|
||||
increment_spend_counters skips reserved keys. The reconcile runs before this request's spend is enqueued to the
|
||||
DB, so the reseeded floor excludes it."""
|
||||
async def reseed_reserved_counter(counter_key: str, actual_cost: float) -> None:
|
||||
from litellm.proxy.proxy_server import increment_spend_counter_cache, reseed_spend_counter_from_db
|
||||
|
||||
reseeded: Final = await reseed_spend_counter_from_db(counter_key=item.counter_key)
|
||||
reseeded: Final = await reseed_spend_counter_from_db(counter_key=counter_key)
|
||||
if reseeded and actual_cost > 0:
|
||||
await increment_spend_counter_cache(counter_key=item.counter_key, increment=actual_cost)
|
||||
|
||||
|
||||
async def _counter_can_apply_adjustment(
|
||||
counter_key: str,
|
||||
adjustment: float,
|
||||
) -> bool:
|
||||
from litellm.proxy.proxy_server import read_spend_counter_cache_value
|
||||
|
||||
try:
|
||||
current_value, _ = await read_spend_counter_cache_value(counter_key=counter_key)
|
||||
except (TypeError, ValueError):
|
||||
return False
|
||||
if current_value is None:
|
||||
return False
|
||||
|
||||
return not (adjustment < 0 and current_value + adjustment < -1e-12)
|
||||
|
||||
|
||||
async def _release_applied_entries_best_effort(
|
||||
entries: list[dict],
|
||||
default_reserved_cost: float,
|
||||
) -> None:
|
||||
for entry in entries:
|
||||
try:
|
||||
await _set_reserved_entries_actual_cost(
|
||||
entries=[entry],
|
||||
actual_cost=0.0,
|
||||
default_reserved_cost=default_reserved_cost,
|
||||
)
|
||||
except Exception:
|
||||
counter_key = entry.get("counter_key")
|
||||
verbose_proxy_logger.exception("Failed to release partial budget reservation during exception cleanup")
|
||||
if counter_key is None:
|
||||
continue
|
||||
try:
|
||||
from litellm.proxy.proxy_server import invalidate_spend_counter
|
||||
|
||||
await invalidate_spend_counter(counter_key=counter_key)
|
||||
except Exception:
|
||||
verbose_proxy_logger.exception(
|
||||
"Failed to invalidate partial budget reservation counter during exception cleanup"
|
||||
)
|
||||
|
||||
|
||||
async def _resize_applied_reservation(
|
||||
entries: list[dict],
|
||||
current_reserved_cost: float,
|
||||
new_reserved_cost: float,
|
||||
) -> None:
|
||||
await _set_reserved_entries_actual_cost(
|
||||
entries=entries,
|
||||
actual_cost=new_reserved_cost,
|
||||
default_reserved_cost=current_reserved_cost,
|
||||
reseed_on_inconsistent=False,
|
||||
)
|
||||
for entry in entries:
|
||||
entry["reserved_cost"] = new_reserved_cost
|
||||
entry["applied_adjustment"] = 0.0
|
||||
|
||||
|
||||
def _counter_to_reservation_entry(
|
||||
counter: _BudgetCounter,
|
||||
reserved_cost: float,
|
||||
) -> dict[str, float | str]:
|
||||
return {
|
||||
"counter_key": counter.counter_key,
|
||||
"entity_type": counter.entity_type,
|
||||
"entity_id": counter.entity_id,
|
||||
"reserved_cost": reserved_cost,
|
||||
"applied_adjustment": 0.0,
|
||||
**({"window_reset_at": counter.reset_at.isoformat()} if counter.reset_at is not None else {}),
|
||||
}
|
||||
await increment_spend_counter_cache(counter_key=counter_key, increment=actual_cost)
|
||||
|
||||
|
||||
def _get_entry_reserved_cost(entry: dict, default_reserved_cost: float) -> float:
|
||||
|
|
@ -1416,10 +1067,10 @@ def get_budget_window_start(window: object, reset_settings: BudgetResetSettings
|
|||
|
||||
reset_at = _coerce_datetime(window_dict.get("reset_at"))
|
||||
if reset_at is None:
|
||||
return datetime.now(timezone.utc) - timedelta(seconds=duration_seconds)
|
||||
return normalize_budget_window_start(datetime.now(timezone.utc) - timedelta(seconds=duration_seconds))
|
||||
if reset_at.tzinfo is None:
|
||||
reset_at = reset_at.replace(tzinfo=timezone.utc)
|
||||
return reset_at - timedelta(seconds=duration_seconds)
|
||||
return normalize_budget_window_start(reset_at - timedelta(seconds=duration_seconds))
|
||||
|
||||
|
||||
def _coerce_datetime(value: object) -> datetime | None:
|
||||
|
|
|
|||
320
litellm/proxy/spend_tracking/native_budgets.py
Normal file
320
litellm/proxy/spend_tracking/native_budgets.py
Normal file
|
|
@ -0,0 +1,320 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import uuid
|
||||
from collections.abc import Iterable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime
|
||||
from typing import TYPE_CHECKING, Annotated, Final, Literal
|
||||
|
||||
from fastapi import HTTPException, status
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError
|
||||
from pydantic import Field as PydanticField
|
||||
|
||||
import litellm
|
||||
from litellm.integrations.custom_logger_v2.derivations import (
|
||||
budget_reservation,
|
||||
mapping,
|
||||
write_budget_reservation,
|
||||
)
|
||||
from litellm.integrations.custom_logger_v2.field_sources import HookArgs
|
||||
from litellm.proxy._types import ProxyErrorTypes, ProxyException, UserAPIKeyAuth
|
||||
from litellm.proxy.spend_tracking.spend_counter_batch import PendingSpendIncrement, spend_counter_batch_scope
|
||||
from litellm.redis.stores.spend_counters import SpendCounterStore
|
||||
from litellm.types.integrations.custom_logger_v2 import BudgetPlan, BudgetReservation, coerce_record
|
||||
from litellm.types.proxy.native_budgets import BudgetCheck, NativeSettlement
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.rust_bridge._native import NativeBudgets
|
||||
|
||||
|
||||
def new_reservation_id() -> str:
|
||||
return uuid.uuid4().hex
|
||||
|
||||
|
||||
class _BudgetExceededFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
current_cost: float
|
||||
max_budget: float
|
||||
message: str
|
||||
entity_type: str | None
|
||||
entity_id: str | None
|
||||
|
||||
|
||||
class _DetailFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
detail: str
|
||||
|
||||
|
||||
class _MessageFields(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
message: str
|
||||
|
||||
|
||||
class _Held(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
admission: Literal["held"]
|
||||
reservation: Mapping[str, object]
|
||||
|
||||
|
||||
class _Skipped(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
admission: Literal["skipped"]
|
||||
|
||||
|
||||
class _Refused(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
admission: Literal["refused"]
|
||||
kind: Literal["budget_exceeded", "budget_unavailable", "budget_reservation_failed"]
|
||||
fields: Mapping[str, object]
|
||||
|
||||
|
||||
_ADMISSION: Final[TypeAdapter[_Held | _Skipped | _Refused]] = TypeAdapter(
|
||||
Annotated[_Held | _Skipped | _Refused, PydanticField(discriminator="admission")]
|
||||
)
|
||||
_SETTLEMENT: Final = TypeAdapter(NativeSettlement)
|
||||
_CHECK: Final[TypeAdapter[BudgetCheck]] = TypeAdapter(BudgetCheck)
|
||||
_HELD_ENTRIES: Final = TypeAdapter(Iterable[object])
|
||||
|
||||
|
||||
def _refusal(refused: _Refused) -> Exception:
|
||||
match refused.kind:
|
||||
case "budget_exceeded":
|
||||
exceeded: Final = _BudgetExceededFields.model_validate(refused.fields)
|
||||
return litellm.BudgetExceededError(
|
||||
current_cost=exceeded.current_cost,
|
||||
max_budget=exceeded.max_budget,
|
||||
message=exceeded.message,
|
||||
entity_type=exceeded.entity_type,
|
||||
entity_id=exceeded.entity_id,
|
||||
)
|
||||
case "budget_unavailable":
|
||||
return HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail=_DetailFields.model_validate(refused.fields).detail,
|
||||
)
|
||||
case "budget_reservation_failed":
|
||||
return ProxyException(
|
||||
message="Authentication Error, " + _MessageFields.model_validate(refused.fields).message,
|
||||
type=ProxyErrorTypes.auth_error,
|
||||
param=None,
|
||||
code=status.HTTP_401_UNAUTHORIZED,
|
||||
)
|
||||
|
||||
|
||||
def _with_window_reset(entry: object, window_resets: Mapping[str, str]) -> object:
|
||||
held: Final = mapping(entry)
|
||||
if held is None:
|
||||
return entry
|
||||
reset: Final = window_resets.get(str(held.get("counter_key")))
|
||||
return entry if reset is None else {**held, "window_reset_at": reset}
|
||||
|
||||
|
||||
def _with_window_resets(value: Mapping[str, object], window_resets: Mapping[str, str]) -> Mapping[str, object]:
|
||||
entries: Final = value.get("entries")
|
||||
if not window_resets or not isinstance(entries, list | tuple):
|
||||
return value
|
||||
return {
|
||||
**value,
|
||||
"entries": [_with_window_reset(entry, window_resets) for entry in _HELD_ENTRIES.validate_python(entries)],
|
||||
}
|
||||
|
||||
|
||||
def _planned_counters(plan: BudgetPlan) -> tuple[tuple[str, float], ...]:
|
||||
return tuple(
|
||||
(counter.counter_key, float(counter.fallback_spend or 0.0))
|
||||
for counter in plan.counters or ()
|
||||
if counter.counter_key is not None
|
||||
)
|
||||
|
||||
|
||||
async def observed_counter_spend(plan: BudgetPlan) -> tuple[Mapping[str, object], ...]:
|
||||
from litellm.proxy.proxy_server import get_current_spend, spend_counter_store
|
||||
|
||||
counters: Final = _planned_counters(plan)
|
||||
with spend_counter_batch_scope(spend_counter_store, counter_keys=frozenset(key for key, _ in counters)):
|
||||
return tuple(
|
||||
[
|
||||
{
|
||||
"counter_key": counter_key,
|
||||
"spend": await get_current_spend(counter_key=counter_key, fallback_spend=fallback_spend),
|
||||
}
|
||||
for counter_key, fallback_spend in counters
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
_engine: Final[list[tuple[SpendCounterStore, NativeBudgets]]] = [] # mutable-ok: the engine for the proxy's store
|
||||
|
||||
|
||||
def native_budgets() -> NativeBudgets | None:
|
||||
from litellm.proxy.proxy_server import spend_counter_store
|
||||
|
||||
if _engine and _engine[0][0] is spend_counter_store:
|
||||
return _engine[0][1]
|
||||
from litellm.rust_bridge._native import NativeBudgets, NativeSpendCounters
|
||||
|
||||
handle: Final = spend_counter_store.handle
|
||||
if not isinstance(handle, NativeSpendCounters):
|
||||
return None
|
||||
budgets: Final = NativeBudgets(handle)
|
||||
_engine[:] = [(spend_counter_store, budgets)]
|
||||
return budgets
|
||||
|
||||
|
||||
async def reserve_native(
|
||||
plan: Mapping[str, object],
|
||||
identity: UserAPIKeyAuth,
|
||||
window_resets: Mapping[str, str],
|
||||
) -> dict[str, object] | None: # mutable-ok: settlement stamps the reservation it returns
|
||||
budgets: Final = native_budgets()
|
||||
if budgets is None:
|
||||
return None
|
||||
record: Final = coerce_record(BudgetPlan, plan)
|
||||
admission: Final = _ADMISSION.validate_python(await budgets.reserve(record, await observed_counter_spend(record)))
|
||||
if isinstance(admission, _Refused):
|
||||
raise _refusal(admission)
|
||||
if isinstance(admission, _Skipped):
|
||||
return None
|
||||
held: Final = _with_window_resets(admission.reservation, window_resets)
|
||||
reservation: Final[dict[str, object]] = { # mutable-ok: settlement stamps the reservation it returns
|
||||
"reserved_cost": 0.0,
|
||||
"entries": [],
|
||||
"finalized": False,
|
||||
"callback_bound": False,
|
||||
"input_cost": 0.0,
|
||||
"input_tokens": None,
|
||||
}
|
||||
identity.budget_reservation = reservation
|
||||
write_budget_reservation(HookArgs(auth=identity), held)
|
||||
from litellm.proxy.spend_tracking.budget_reservation import start_reservation_lease_renewal
|
||||
|
||||
start_reservation_lease_renewal(
|
||||
budget_reservation=reservation,
|
||||
counter_keys=frozenset(
|
||||
key for key in map(_held_counter_key, _held_entries(reservation.get("entries") or ())) if key is not None
|
||||
),
|
||||
request_task=asyncio.current_task(),
|
||||
)
|
||||
return reservation
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Reserved:
|
||||
budget_reservation: object
|
||||
|
||||
|
||||
def _reservation_record(reservation: object) -> BudgetReservation | None:
|
||||
return budget_reservation(HookArgs(auth=_Reserved(reservation)))
|
||||
|
||||
|
||||
def _window_reset(entry: object) -> str | None:
|
||||
held: Final = mapping(entry)
|
||||
reset: Final = None if held is None else held.get("window_reset_at")
|
||||
if isinstance(reset, datetime):
|
||||
return reset.isoformat()
|
||||
return reset if isinstance(reset, str) else None
|
||||
|
||||
|
||||
def _held_entries(raw: object) -> tuple[object, ...]:
|
||||
try:
|
||||
return tuple(_HELD_ENTRIES.validate_python(raw))
|
||||
except ValidationError:
|
||||
raise TypeError(f"'{type(raw).__name__}' object is not iterable") from None
|
||||
|
||||
|
||||
def _held_counter_key(entry: object) -> str | None:
|
||||
held: Final = mapping(entry)
|
||||
counter_key: Final = None if held is None else held.get("counter_key")
|
||||
return counter_key if isinstance(counter_key, str) else None
|
||||
|
||||
|
||||
async def _observed_spend(counter_key: str | None) -> float | None:
|
||||
if counter_key is None:
|
||||
return None
|
||||
from litellm.proxy.proxy_server import read_spend_counter_cache_value
|
||||
|
||||
try:
|
||||
value, _ = await read_spend_counter_cache_value(counter_key=counter_key)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
async def settle_native_reservation(
|
||||
reservation: Mapping[str, object],
|
||||
actual_cost: float,
|
||||
apply_consistent: bool = True,
|
||||
) -> tuple[PendingSpendIncrement, ...]:
|
||||
budgets: Final = native_budgets()
|
||||
if budgets is None:
|
||||
return ()
|
||||
entries: Final = _held_entries(reservation.get("entries") or ())
|
||||
record: Final = _reservation_record(reservation)
|
||||
if record is None:
|
||||
return ()
|
||||
from litellm.proxy.proxy_server import spend_counter_store
|
||||
from litellm.proxy.spend_tracking.budget_reservation import reseed_reserved_counter
|
||||
|
||||
counter_keys: Final = tuple(map(_held_counter_key, entries))
|
||||
with spend_counter_batch_scope(
|
||||
spend_counter_store, counter_keys=frozenset(key for key in counter_keys if key is not None)
|
||||
):
|
||||
observed: Final = tuple([await _observed_spend(counter_key) for counter_key in counter_keys])
|
||||
settled: Final = _SETTLEMENT.validate_python(
|
||||
await budgets.settle(record, tuple(map(_window_reset, entries)), observed, actual_cost, apply_consistent)
|
||||
)
|
||||
for reseed in settled.reseed:
|
||||
await reseed_reserved_counter(reseed.counter_key, reseed.actual_cost)
|
||||
for entry, target in zip(entries, settled.adjustments):
|
||||
if target is not None and isinstance(entry, dict):
|
||||
entry["applied_adjustment"] = target
|
||||
return tuple(
|
||||
PendingSpendIncrement(counter_key=pending.counter_key, increment=pending.increment)
|
||||
for pending in settled.pending
|
||||
)
|
||||
|
||||
|
||||
class BudgetDecision(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
decision: Literal["skip", "pass", "alert", "throttle", "refuse"]
|
||||
pct: int | float | None = None
|
||||
current_cost: int | float | None = None
|
||||
max_budget: int | float | None = None
|
||||
message: str | None = None
|
||||
entity_type: str | None = None
|
||||
entity_id: str | None = None
|
||||
|
||||
@property
|
||||
def skipped(self) -> bool:
|
||||
return self.decision == "skip"
|
||||
|
||||
@property
|
||||
def alerts(self) -> bool:
|
||||
return self.decision == "alert"
|
||||
|
||||
@property
|
||||
def refused(self) -> bool:
|
||||
return self.decision in ("refuse", "throttle")
|
||||
|
||||
def error(self) -> litellm.BudgetExceededError:
|
||||
return litellm.BudgetExceededError(
|
||||
current_cost=self.current_cost if self.current_cost is not None else 0.0,
|
||||
max_budget=self.max_budget if self.max_budget is not None else 0.0,
|
||||
message=self.message,
|
||||
entity_type=self.entity_type,
|
||||
entity_id=self.entity_id,
|
||||
)
|
||||
|
||||
|
||||
def native_budget_decision(check: Mapping[str, object]) -> BudgetDecision:
|
||||
from litellm.rust_bridge._native import budget_check_decision
|
||||
|
||||
return BudgetDecision.model_validate(budget_check_decision(_CHECK.validate_python(check)))
|
||||
|
|
@ -78,6 +78,10 @@ class SpendCounterStore:
|
|||
if redis.client is not None:
|
||||
self._handle.attach(redis.client)
|
||||
|
||||
@property
|
||||
def handle(self) -> SpendCountersHandle:
|
||||
return self._handle
|
||||
|
||||
@property
|
||||
def redis_ttl(self) -> float | None:
|
||||
"""The TTL every Redis write gives a counter, None when there is no Redis."""
|
||||
|
|
|
|||
|
|
@ -1,34 +1,110 @@
|
|||
import asyncio
|
||||
from collections.abc import AsyncGenerator, Coroutine
|
||||
import urllib.parse
|
||||
from collections.abc import AsyncGenerator, Callable, Coroutine
|
||||
from contextlib import asynccontextmanager
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Final
|
||||
|
||||
from litellm.rust_bridge._native import NativeDatabase
|
||||
from litellm._logging import verbose_proxy_logger
|
||||
from litellm.proxy.db.db_span import proxy_service_logging
|
||||
from litellm.proxy.db.native_database import (
|
||||
NativeEndpointSettings,
|
||||
ServiceLoggingEvents,
|
||||
TokenRotation,
|
||||
keep_token_fresh,
|
||||
reader_token_rotation,
|
||||
resolve_native_endpoint,
|
||||
writer_token_rotation,
|
||||
)
|
||||
from litellm.rust_bridge._native import NativeDatabase, NativeDatabaseFailure
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from litellm.proxy.utils import PrismaClient
|
||||
|
||||
|
||||
def _token_refresh(
|
||||
install: Callable[[str], None], rotation: TokenRotation | None, url: str
|
||||
) -> "tuple[asyncio.Task[None], ...]":
|
||||
if rotation is None:
|
||||
return ()
|
||||
current: Final = urllib.parse.unquote(urllib.parse.urlsplit(url).password or "")
|
||||
return (asyncio.create_task(keep_token_fresh(install, rotation, current)),)
|
||||
|
||||
|
||||
def _log_ignored_options(env_var: str, settings: NativeEndpointSettings) -> None:
|
||||
if settings.ignored_options:
|
||||
verbose_proxy_logger.error(
|
||||
"Native database pool cannot send the %s options %s; only -c name=value settings carry over",
|
||||
env_var,
|
||||
settings.ignored_options,
|
||||
)
|
||||
|
||||
|
||||
async def _warn_if_unreachable(native: NativeDatabase) -> None:
|
||||
try:
|
||||
await native.health_check()
|
||||
except NativeDatabaseFailure as error:
|
||||
verbose_proxy_logger.warning(
|
||||
"Native database pool cannot reach the database, so operations ported to it will fail until it"
|
||||
" connects. Prisma is checked separately. Cause: %s",
|
||||
error,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Database:
|
||||
prisma: "PrismaClient"
|
||||
native: NativeDatabase
|
||||
native: NativeDatabase | None
|
||||
tasks: "tuple[asyncio.Task[None], ...]" = ()
|
||||
boot_probe: "asyncio.Task[None] | None" = None
|
||||
|
||||
@classmethod
|
||||
async def open(cls, prisma: "PrismaClient") -> "Database":
|
||||
async def open(
|
||||
cls, prisma: "PrismaClient", writer_url: str, reader_url: str | None, native_connections: int
|
||||
) -> "Database":
|
||||
if native_connections == 0:
|
||||
return cls(prisma=prisma, native=None)
|
||||
try:
|
||||
native: Final = await NativeDatabase.connect()
|
||||
writer: Final = await resolve_native_endpoint(writer_url, native_connections)
|
||||
reader: Final = (
|
||||
None if reader_url is None else await resolve_native_endpoint(reader_url, native_connections)
|
||||
)
|
||||
native: Final = await NativeDatabase.connect(
|
||||
ServiceLoggingEvents(loop=asyncio.get_running_loop(), service_logging=proxy_service_logging),
|
||||
writer.endpoint,
|
||||
None if reader is None else reader.endpoint,
|
||||
)
|
||||
except BaseException:
|
||||
await prisma.aclose()
|
||||
raise
|
||||
return cls(prisma=prisma, native=native)
|
||||
_log_ignored_options("DATABASE_URL", writer)
|
||||
if reader is not None:
|
||||
_log_ignored_options("DATABASE_URL_READ_REPLICA", reader)
|
||||
writer_refresh: Final = _token_refresh(
|
||||
native.set_writer_password, writer_token_rotation(writer_url), writer_url
|
||||
)
|
||||
reader_refresh: Final = (
|
||||
()
|
||||
if reader_url is None
|
||||
else _token_refresh(native.set_reader_password, reader_token_rotation(reader_url), reader_url)
|
||||
)
|
||||
return cls(
|
||||
prisma=prisma,
|
||||
native=native,
|
||||
tasks=(*writer_refresh, *reader_refresh),
|
||||
boot_probe=asyncio.create_task(_warn_if_unreachable(native)),
|
||||
)
|
||||
|
||||
async def aclose(self) -> None:
|
||||
background: Final = (*self.tasks, *(() if self.boot_probe is None else (self.boot_probe,)))
|
||||
for task in background:
|
||||
task.cancel()
|
||||
await asyncio.gather(*background, return_exceptions=True)
|
||||
try:
|
||||
await self.prisma.aclose()
|
||||
finally:
|
||||
await self.native.close()
|
||||
if self.native is not None:
|
||||
await self.native.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
|
|
|
|||
|
|
@ -4434,6 +4434,13 @@ def _log_attempt(plan: Plan, model_name: object, error: BaseException | None = N
|
|||
verbose_router_logger.info("%s(model=%s)\x1b[32m 200 OK\x1b[0m", served_name, model_name)
|
||||
|
||||
|
||||
def _params_base_model(core: RustCore, deployment_id: str) -> str | None:
|
||||
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
|
||||
record: Final = as_str_mapping(core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
|
||||
class _Driver:
|
||||
"""Answers the attempt and fallback steps of one call, and shapes its result.
|
||||
|
||||
|
|
@ -4496,7 +4503,7 @@ class _Driver:
|
|||
self._route_phase: contextlib.ExitStack | None = None
|
||||
self._route_model: str = ""
|
||||
self._walk: Final = WalkMetadata(
|
||||
request, plan.metadata_field, self._params_base_model, live=fresh and not plan.single_attempt
|
||||
request, plan.metadata_field, partial(_params_base_model, core), live=fresh and not plan.single_attempt
|
||||
)
|
||||
self.outcomes: Final = FallbackOutcomes(request)
|
||||
# The fallback level the walk is at, from its fallback events.
|
||||
|
|
@ -4548,12 +4555,6 @@ class _Driver:
|
|||
endpoints.counters.retract_success(self._plan, call)
|
||||
endpoints.attempt_failed(self._plan, error, deployment, call, call.group)
|
||||
|
||||
def _params_base_model(self, deployment_id: str) -> str | None:
|
||||
"""``_deployment_params_base_model``: the ``base_model`` under the deployment's ``litellm_params``."""
|
||||
record: Final = as_str_mapping(self._core._handle.deployment(deployment_id)) or {} # pyright: ignore[reportPrivateUsage] # the driver serves its own core
|
||||
base_model: Final = (as_str_mapping(record.get("litellm_params")) or {}).get("base_model")
|
||||
return base_model if isinstance(base_model, str) else None
|
||||
|
||||
def _logged_error(self, event: Mapping[str, object]) -> BaseException | None:
|
||||
"""The error a ``retry_logged`` event records."""
|
||||
return self._cause_error(event)
|
||||
|
|
@ -5544,6 +5545,7 @@ class _Driver:
|
|||
def finish(self, step: Mapping[str, object]) -> object:
|
||||
"""The response with the router's headers, in the order ``make_call`` and the wrappers above it
|
||||
stamp them: a usage read that raises leaves the counts off."""
|
||||
self._clear_errors()
|
||||
if not self._plan.sets_headers:
|
||||
self.log_outcomes_sync()
|
||||
return self._last_response
|
||||
|
|
@ -5564,6 +5566,7 @@ class _Driver:
|
|||
|
||||
async def finish_async(self, step: Mapping[str, object]) -> object:
|
||||
"""``finish``, with the usage read on the call's own loop."""
|
||||
self._clear_errors()
|
||||
if not self._plan.sets_headers:
|
||||
await await_all(self.outcomes.take_pending())
|
||||
return self._last_response
|
||||
|
|
@ -5585,6 +5588,18 @@ class _Driver:
|
|||
return self._request.get("messages") is not None or self._request.get("input") is not None
|
||||
|
||||
def failed(self, step: Mapping[str, object]) -> BaseException:
|
||||
try:
|
||||
return self._failure(step)
|
||||
finally:
|
||||
self._clear_errors()
|
||||
|
||||
def _clear_errors(self) -> None:
|
||||
self._last_error = None
|
||||
self._level_errors.clear()
|
||||
self._hop_first.clear()
|
||||
self._level_picks.clear()
|
||||
|
||||
def _failure(self, step: Mapping[str, object]) -> BaseException:
|
||||
model: Final = str(self._request.get("model"))
|
||||
primary: Final = self._level_errors.get(self._base_level)
|
||||
if primary is None:
|
||||
|
|
|
|||
|
|
@ -1575,6 +1575,7 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
|
|||
|
||||
def __init__(self, async_generator: AsyncGenerator[object, None]) -> None: # pyright: ignore[reportMissingSuperCall] # no httpx response of its own
|
||||
self._async_generator: Final = async_generator
|
||||
self.attempt: Final = attempt
|
||||
self._billing_source: object = source_iterator
|
||||
self._last_sequence_number = -1
|
||||
self.response = getattr(source_iterator, "response", None) # pyright: ignore[reportAttributeAccessIssue] # copied from the source
|
||||
|
|
@ -1699,10 +1700,12 @@ def wrap_responses_stream( # noqa: C901 # one function per Python wrapper it p
|
|||
if response_id_exposed:
|
||||
_raise_released_stream_error(attempt, e)
|
||||
async with contextlib.aclosing(
|
||||
_responses_fallback_attempt(attempt, e, source_iterator, wrapper.adopt_fallback_headers, held)
|
||||
) as fallback_stream:
|
||||
async for fallback_item in fallback_stream:
|
||||
yield fallback_item
|
||||
_responses_recover_stream_error(
|
||||
attempt, e, source_iterator, wrapper, wrapper.adopt_fallback_headers, held
|
||||
)
|
||||
) as recovered_stream:
|
||||
async for recovered_item in recovered_stream:
|
||||
yield recovered_item
|
||||
except Exception:
|
||||
for held_event in held:
|
||||
yield held_event
|
||||
|
|
@ -1791,6 +1794,99 @@ def _responses_continuation_request(
|
|||
}
|
||||
|
||||
|
||||
def _responses_stream_retries(
|
||||
attempt: StreamAttempt, e: MidStreamFallbackError, source_iterator: "BaseResponsesAPIStreamingIterator"
|
||||
) -> bool:
|
||||
"""Whether a broken Responses stream retries in its own group first: it broke before any output and
|
||||
before a tool call, with an error the retry rule takes (408, 409, 429, 5xx, or no status), as a
|
||||
``/v1/messages`` stream does. After output the continuation fallback takes over instead."""
|
||||
if not _mid_stream_error_before_output(e) or source_iterator.started_tool_call:
|
||||
return False
|
||||
status_code: Final = _stream_raised_error_status(_mid_stream_retry_trigger(e))
|
||||
return status_code is None or attempt.should_retry_status(status_code)
|
||||
|
||||
|
||||
async def _responses_recover_stream_error(
|
||||
attempt: StreamAttempt,
|
||||
e: MidStreamFallbackError,
|
||||
source_iterator: "BaseResponsesAPIStreamingIterator",
|
||||
wrapper: object,
|
||||
adopt_headers: Callable[[object], _Prepared],
|
||||
held: tuple[object, ...],
|
||||
) -> AsyncGenerator[object, None]:
|
||||
"""A Responses stream that broke before any output is retried in its own group, then falls back
|
||||
with the newest failure once the retries are spent. The primary's held lifecycle events are dropped
|
||||
when a retry or a fallback answers."""
|
||||
last_error: MidStreamFallbackError = e # rebind-ok: the newest failure once a retry also failed
|
||||
if _responses_stream_retries(attempt, e, source_iterator):
|
||||
if hasattr(source_iterator, "aclose"):
|
||||
with anyio.CancelScope(shield=True), contextlib.suppress(Exception):
|
||||
await source_iterator.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
|
||||
await attempt.failure_recorded()
|
||||
trigger: Final = _mid_stream_retry_trigger(e)
|
||||
retried: Final = await attempt.retry(trigger)
|
||||
if not isinstance(retried, BaseException):
|
||||
async for retried_item in _responses_yield_retried(retried, wrapper, adopt_headers):
|
||||
yield retried_item
|
||||
return
|
||||
if retried is not trigger:
|
||||
last_error = _stream_pre_content_error( # rebind-ok: the retry failed again, so its failure is the newest
|
||||
retried if isinstance(retried, Exception) else e, e.model or "", e.llm_provider
|
||||
)
|
||||
async with contextlib.aclosing(
|
||||
_responses_fallback_attempt(attempt, last_error, source_iterator, adopt_headers, held)
|
||||
) as fallback_stream:
|
||||
async for fallback_item in fallback_stream:
|
||||
yield fallback_item
|
||||
|
||||
|
||||
async def _responses_yield_retried(
|
||||
retried: tuple[object, int, int], wrapper: object, adopt_headers: Callable[[object], _Prepared]
|
||||
) -> AsyncGenerator[object, None]:
|
||||
"""Hands a same-group retry's stream to the client through the wrapper, with the retry's headers, and
|
||||
closes it after. A retried stream that breaks again counts on from the retries already made, and when
|
||||
it recovers onto another source (its own fallback) the wrapper and the events follow that source."""
|
||||
response, attempted, budget = retried
|
||||
inner_attempt: Final = getattr(response, "attempt", None)
|
||||
if isinstance(inner_attempt, StreamAttempt):
|
||||
inner_attempt.attempted_retries = attempted
|
||||
inner_attempt.committed_budget = budget
|
||||
|
||||
served_as: Final = _prepare_fallback_hidden_params(wrapper)[1].get("x-litellm-model-group")
|
||||
|
||||
def follow_source() -> _Prepared:
|
||||
prepared: Final = adopt_headers(response)
|
||||
if served_as is not None:
|
||||
_ = prepared[1].setdefault("x-litellm-model-group", served_as)
|
||||
newest: Final = prepared[1].get("x-litellm-attempted-retries")
|
||||
_ = add_retry_headers_to_response(
|
||||
response=wrapper,
|
||||
attempted_retries=max(attempted, newest) if isinstance(newest, int) else attempted,
|
||||
max_retries=budget,
|
||||
)
|
||||
return prepared
|
||||
|
||||
followed: _Prepared = follow_source() # rebind-ok: refreshed when the retried stream changes its source
|
||||
source: object = getattr(response, "_billing_source", None) # rebind-ok: the retried stream's current source
|
||||
try:
|
||||
if hasattr(response, "__aiter__"):
|
||||
async for retried_item in cast("AsyncIterator[object]", response): # cast-ok: __aiter__ checked above
|
||||
if (current_source := getattr(response, "_billing_source", None)) is not source:
|
||||
source = current_source
|
||||
followed = follow_source()
|
||||
_apply_fallback_hidden_params_to_item(retried_item, followed)
|
||||
yield retried_item
|
||||
else:
|
||||
yield response
|
||||
finally:
|
||||
if hasattr(response, "aclose"):
|
||||
with anyio.CancelScope(shield=True):
|
||||
try:
|
||||
await response.aclose() # pyright: ignore[reportAttributeAccessIssue, reportUnknownMemberType] # checked above
|
||||
except Exception as exc: # noqa: BLE001 # closing is best-effort
|
||||
verbose_router_logger.debug("stream_with_fallbacks(aresponses): error closing retry: %s", exc)
|
||||
|
||||
|
||||
async def _responses_fallback_attempt(
|
||||
attempt: StreamAttempt,
|
||||
e: MidStreamFallbackError,
|
||||
|
|
|
|||
|
|
@ -133,6 +133,7 @@ from litellm.types.utils import (
|
|||
)
|
||||
from litellm.utils import (
|
||||
ModelInfoLookupMiss,
|
||||
base_fields_replaced_by,
|
||||
catalog_with_runtime_registrations,
|
||||
get_potential_model_names,
|
||||
get_secret,
|
||||
|
|
@ -340,10 +341,11 @@ def _inherited_off_peak_base_rates(
|
|||
backend_entry: Final = _backend_cost_entry(host, backend_model, provider)
|
||||
if backend_entry is None:
|
||||
return {}
|
||||
superseded: Final = base_fields_replaced_by(model_info)
|
||||
return {
|
||||
field: copy.deepcopy(value)
|
||||
for field, value in backend_entry.items()
|
||||
if model_info.get(field) is None and value is not None
|
||||
if model_info.get(field) is None and value is not None and field not in superseded
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
87
litellm/rust/Cargo.lock
generated
87
litellm/rust/Cargo.lock
generated
|
|
@ -258,6 +258,8 @@ checksum = "47712fde1909402600ccfbb26e47d482d2e58bb9e9e603d9f17e67cc435a6319"
|
|||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-sdk-sso",
|
||||
"aws-sdk-ssooidc",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
|
|
@ -269,11 +271,14 @@ dependencies = [
|
|||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"hex",
|
||||
"http 1.4.2",
|
||||
"sha1 0.10.7",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"url",
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -428,6 +433,58 @@ dependencies = [
|
|||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sso"
|
||||
version = "1.111.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "65700d75f6da78c89a61a49f1539979400a4609d3329376b1dc2133d7a1b2b90"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-ssooidc"
|
||||
version = "1.113.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eb1147c254af7b6cee3ace90ae7da6f21366ce9a9b899035fa8a0fdf586eb75a"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
"aws-smithy-async",
|
||||
"aws-smithy-http",
|
||||
"aws-smithy-json",
|
||||
"aws-smithy-observability",
|
||||
"aws-smithy-runtime",
|
||||
"aws-smithy-runtime-api",
|
||||
"aws-smithy-schema",
|
||||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"fastrand",
|
||||
"http 0.2.12",
|
||||
"http 1.4.2",
|
||||
"regex-lite",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-sts"
|
||||
version = "1.108.0"
|
||||
|
|
@ -3455,6 +3512,7 @@ dependencies = [
|
|||
"aws-smithy-types",
|
||||
"aws-types",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"futures-util",
|
||||
"litellm-auth-types",
|
||||
"litellm-error",
|
||||
|
|
@ -3467,6 +3525,7 @@ dependencies = [
|
|||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"tempfile",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"url",
|
||||
|
|
@ -3705,6 +3764,22 @@ dependencies = [
|
|||
"thiserror 2.0.19",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-budgets"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"litellm-callbacks",
|
||||
"litellm-coroutine",
|
||||
"litellm-python-compat",
|
||||
"litellm-redis-store",
|
||||
"litellm-tracing",
|
||||
"rstest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "litellm-callbacks-legacy-python"
|
||||
version = "0.1.0"
|
||||
|
|
@ -3836,6 +3911,10 @@ name = "litellm-db"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"futures-util",
|
||||
"litellm-circuit-breaker",
|
||||
"litellm-error",
|
||||
"litellm-secret-value",
|
||||
"macro_rules_attribute",
|
||||
"rstest",
|
||||
"serde",
|
||||
|
|
@ -3844,6 +3923,7 @@ dependencies = [
|
|||
"strum",
|
||||
"thiserror 2.0.19",
|
||||
"tokio",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3983,6 +4063,7 @@ dependencies = [
|
|||
"litellm-auth",
|
||||
"litellm-cache-memory",
|
||||
"litellm-cache-response",
|
||||
"litellm-core-utils",
|
||||
"litellm-error",
|
||||
"litellm-gateway-auth",
|
||||
"litellm-host",
|
||||
|
|
@ -4248,6 +4329,7 @@ dependencies = [
|
|||
"litellm-inference-testing",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-python-compat",
|
||||
"litellm-router-types",
|
||||
"litellm-secret-value",
|
||||
"litellm-secrets",
|
||||
|
|
@ -4370,6 +4452,7 @@ dependencies = [
|
|||
"aws-smithy-types",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
"data-url",
|
||||
"fancy-regex 0.19.2",
|
||||
"futures-util",
|
||||
|
|
@ -4399,6 +4482,7 @@ dependencies = [
|
|||
"thiserror 2.0.19",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"url",
|
||||
]
|
||||
|
||||
|
|
@ -4467,6 +4551,7 @@ name = "litellm-python-bridge"
|
|||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-credential-types",
|
||||
"aws-sdk-secretsmanager",
|
||||
"azure_core",
|
||||
"bytes",
|
||||
|
|
@ -4480,11 +4565,13 @@ dependencies = [
|
|||
"litellm-cache",
|
||||
"litellm-cache-response",
|
||||
"litellm-callbacks",
|
||||
"litellm-callbacks-budgets",
|
||||
"litellm-callbacks-legacy-python",
|
||||
"litellm-callbacks-python",
|
||||
"litellm-callbacks-s3-log",
|
||||
"litellm-circuit-breaker",
|
||||
"litellm-core-utils",
|
||||
"litellm-db",
|
||||
"litellm-error",
|
||||
"litellm-host",
|
||||
"litellm-host-python",
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ litellm-host-native = { path = "crates/host-native" }
|
|||
litellm-callbacks = { path = "crates/callbacks" }
|
||||
litellm-callbacks-legacy-python = { path = "crates/callbacks-legacy-python" }
|
||||
litellm-callbacks-python = { path = "crates/callbacks-python" }
|
||||
litellm-callbacks-budgets = { path = "crates/callbacks-budgets" }
|
||||
litellm-callbacks-s3-log = { path = "crates/callbacks-s3-log" }
|
||||
litellm-framer = { path = "crates/framer" }
|
||||
litellm-auth = { path = "crates/auth" }
|
||||
|
|
|
|||
|
|
@ -16,12 +16,13 @@ litellm-secret-value.workspace = true
|
|||
serde.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
sha2.workspace = true
|
||||
chrono.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio.workspace = true
|
||||
url.workspace = true
|
||||
veil.workspace = true
|
||||
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process"] }
|
||||
aws-config = { version = "1.9.0", default-features = false, features = ["rustls", "rt-tokio", "credentials-process", "sso"] }
|
||||
aws-credential-types = { version = "1.3.0", features = ["hardcoded-credentials"] }
|
||||
aws-sdk-sts = { version = "1.108.0", default-features = false, features = ["rustls", "rt-tokio"] }
|
||||
aws-sigv4 = "1.5.1"
|
||||
|
|
@ -33,6 +34,7 @@ litellm-secrets.workspace = true
|
|||
aws-smithy-types = "1.6.4"
|
||||
aws-runtime = { version = "1.9.4", default-features = false }
|
||||
rstest.workspace = true
|
||||
tempfile = "3"
|
||||
litellm-http = { workspace = true, features = ["test-support"] }
|
||||
bytes.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -0,0 +1 @@
|
|||
{"Version":"2012-10-17","Statement":[{"Sid":"BedrockLiteLLM","Effect":"Allow","Action":["bedrock:InvokeModel","bedrock:InvokeModelWithResponseStream","bedrock:CountTokens","bedrock:Rerank","bedrock:Retrieve","bedrock:ListKnowledgeBases","bedrock:InvokeAgent","bedrock:ApplyGuardrail","bedrock:GetGuardrail","bedrock:ListGuardrails"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"BedrockAgentCoreLiteLLM","Effect":"Allow","Action":["bedrock-agentcore:InvokeAgentRuntime","bedrock-agentcore:InvokeAgentRuntimeForUser","bedrock-agentcore:InvokeGateway"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformLiteLLM","Effect":"Allow","Action":["aws-external-anthropic:CreateInference","aws-external-anthropic:CreateBatchInference","aws-external-anthropic:CancelBatchInference","aws-external-anthropic:DeleteBatchInference","aws-external-anthropic:CountTokens","aws-external-anthropic:Get*","aws-external-anthropic:List*"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}},{"Sid":"ClaudePlatformOutboundIdentityLiteLLM","Effect":"Allow","Action":["sts:GetWebIdentityToken","sts:TagGetWebIdentityToken"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"},"StringEquals":{"aws:CalledViaLast":"aws-external-anthropic.amazonaws.com"}}},{"Sid":"BedrockMantleLiteLLM","Effect":"Allow","Action":["bedrock-mantle:CreateInference","bedrock-mantle:CountTokens"],"Resource":"*","Condition":{"Bool":{"aws:SecureTransport":"true"}}}]}
|
||||
|
|
@ -42,5 +42,6 @@ pub const SIGV4_COMPUTED_HEADER_NAMES: &[&str] = &[
|
|||
pub const BEDROCK_SERVICE: &str = "bedrock";
|
||||
pub const DEFAULT_SESSION_NAME_PREFIX: &str = "litellm-session";
|
||||
pub const DEFAULT_BEDROCK_REGION: &str = "us-west-2";
|
||||
pub const BEDROCK_RUNTIME_ENDPOINT_TEMPLATE: &str =
|
||||
"https://bedrock-runtime.{region}.amazonaws.com";
|
||||
pub const BEDROCK_SESSION_POLICY: &str = include_str!("bedrock_session_policy.json");
|
||||
pub const WEB_IDENTITY_SESSION_SECONDS: i32 = 3600;
|
||||
pub const AWS4_AUTHORIZATION_PREFIX: &str = "AWS4-HMAC-SHA256";
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
use aws_credential_types::provider::error::CredentialsError;
|
||||
use litellm_error::Source;
|
||||
|
||||
#[derive(Clone, Debug, thiserror::Error, PartialEq, Eq)]
|
||||
|
|
@ -10,6 +11,14 @@ pub enum Error {
|
|||
#[source]
|
||||
source: Source,
|
||||
},
|
||||
#[error(
|
||||
"the AWS web identity token must name a secret or an oidc/ provider, not an environment variable"
|
||||
)]
|
||||
WebIdentityTokenReference,
|
||||
#[error("the AWS web identity token reference resolved to no token")]
|
||||
MissingWebIdentityToken,
|
||||
#[error("could not read the AWS web identity token")]
|
||||
WebIdentityToken(#[source] Source),
|
||||
#[error("AWS web identity response had no credentials")]
|
||||
MissingWebIdentityCredentials,
|
||||
#[error("AWS web identity response had an invalid expiration")]
|
||||
|
|
@ -49,6 +58,20 @@ pub enum SigningStage {
|
|||
}
|
||||
|
||||
impl Error {
|
||||
pub fn is_empty_default_chain(&self) -> bool {
|
||||
let Self::Credentials {
|
||||
flow: CredentialFlow::DefaultChain,
|
||||
source,
|
||||
} = self
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
matches!(
|
||||
source.downcast_ref::<CredentialsError>(),
|
||||
Some(CredentialsError::CredentialsNotLoaded(_))
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn credentials<E: std::error::Error + Send + Sync + 'static>(
|
||||
flow: CredentialFlow,
|
||||
) -> impl FnOnce(E) -> Self {
|
||||
|
|
@ -73,3 +96,40 @@ impl From<Error> for litellm_auth_types::Error {
|
|||
Self::provider(error)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::empty_default_chain(
|
||||
CredentialFlow::DefaultChain,
|
||||
CredentialsError::not_loaded("no providers in chain provided credentials"),
|
||||
true
|
||||
)]
|
||||
#[case::failed_default_chain(
|
||||
CredentialFlow::DefaultChain,
|
||||
CredentialsError::provider_error("IMDS answered 500"),
|
||||
false
|
||||
)]
|
||||
#[case::empty_profile(
|
||||
CredentialFlow::Profile,
|
||||
CredentialsError::not_loaded("the profile has no credentials"),
|
||||
false
|
||||
)]
|
||||
fn only_a_default_chain_that_loaded_nothing_is_empty(
|
||||
#[case] flow: CredentialFlow,
|
||||
#[case] failure: CredentialsError,
|
||||
#[case] empty: bool,
|
||||
) {
|
||||
assert_eq!(
|
||||
Error::credentials(flow)(failure).is_empty_default_chain(),
|
||||
empty
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_web_identity_failure_is_not_an_empty_default_chain() {
|
||||
assert!(!Error::MissingWebIdentityToken.is_empty_default_chain());
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,10 +2,12 @@ mod aws;
|
|||
pub mod constants;
|
||||
mod error;
|
||||
mod provider;
|
||||
mod region;
|
||||
mod signer;
|
||||
|
||||
pub use aws::*;
|
||||
pub use aws_credential_types::Credentials;
|
||||
pub use error::{CredentialFlow, Error, SigningStage};
|
||||
pub use provider::AwsCredentialsProvider;
|
||||
pub use signer::SigV4Signer;
|
||||
pub use region::{AwsPartition, aws_partition, bedrock_runtime_endpoint};
|
||||
pub use signer::{CallerSigningHeaders, SigV4Signer};
|
||||
|
|
|
|||
239
litellm/rust/crates/auth-aws/src/region.rs
Normal file
239
litellm/rust/crates/auth-aws/src/region.rs
Normal file
|
|
@ -0,0 +1,239 @@
|
|||
use std::path::PathBuf;
|
||||
|
||||
use crate::constants::AWS_DEFAULT_REGION;
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub struct AwsPartition {
|
||||
pub partition: &'static str,
|
||||
pub dns_suffix: &'static str,
|
||||
}
|
||||
|
||||
const COMMERCIAL_PARTITION: AwsPartition = AwsPartition {
|
||||
partition: "aws",
|
||||
dns_suffix: "amazonaws.com",
|
||||
};
|
||||
|
||||
const PARTITIONS_BY_REGION_PREFIX: [(&str, AwsPartition); 6] = [
|
||||
(
|
||||
"cn-",
|
||||
AwsPartition {
|
||||
partition: "aws-cn",
|
||||
dns_suffix: "amazonaws.com.cn",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-gov-",
|
||||
AwsPartition {
|
||||
partition: "aws-us-gov",
|
||||
dns_suffix: "amazonaws.com",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-isob-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-b",
|
||||
dns_suffix: "sc2s.sgov.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-isof-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-f",
|
||||
dns_suffix: "csp.hci.ic.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"us-iso-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso",
|
||||
dns_suffix: "c2s.ic.gov",
|
||||
},
|
||||
),
|
||||
(
|
||||
"eu-isoe-",
|
||||
AwsPartition {
|
||||
partition: "aws-iso-e",
|
||||
dns_suffix: "cloud.adc-e.uk",
|
||||
},
|
||||
),
|
||||
];
|
||||
|
||||
pub fn aws_partition(region: &str) -> AwsPartition {
|
||||
PARTITIONS_BY_REGION_PREFIX
|
||||
.iter()
|
||||
.find(|(prefix, _)| region.starts_with(prefix))
|
||||
.map_or(COMMERCIAL_PARTITION, |(_, partition)| *partition)
|
||||
}
|
||||
|
||||
pub fn bedrock_runtime_endpoint(region: &str) -> String {
|
||||
format!(
|
||||
"https://bedrock-runtime.{region}.{}",
|
||||
aws_partition(region).dns_suffix
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn session_region(
|
||||
env_lookup: &dyn Fn(&str) -> Option<String>,
|
||||
process_env: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
env_lookup(AWS_DEFAULT_REGION)
|
||||
.filter(|region| !region.is_empty())
|
||||
.or_else(|| profile_region(process_env))
|
||||
}
|
||||
|
||||
fn profile_region(process_env: &dyn Fn(&str) -> Option<String>) -> Option<String> {
|
||||
let profile = process_env("AWS_PROFILE")
|
||||
.or_else(|| process_env("AWS_DEFAULT_PROFILE"))
|
||||
.filter(|profile| !profile.is_empty())
|
||||
.unwrap_or_else(|| "default".to_string());
|
||||
let home = process_env("HOME");
|
||||
let path = match process_env("AWS_CONFIG_FILE").filter(|path| !path.is_empty()) {
|
||||
Some(path) => match (path.strip_prefix("~/"), &home) {
|
||||
(Some(rest), Some(home)) => PathBuf::from(home).join(rest),
|
||||
_ => PathBuf::from(path),
|
||||
},
|
||||
None => PathBuf::from(home?).join(".aws").join("config"),
|
||||
};
|
||||
config_file_region(&std::fs::read_to_string(path).ok()?, &profile)
|
||||
}
|
||||
|
||||
fn config_file_region(contents: &str, profile: &str) -> Option<String> {
|
||||
let named = format!("profile {profile}");
|
||||
let selects = |section: &str| section == named || (profile == "default" && section == profile);
|
||||
let mut selected = false;
|
||||
for line in contents.lines().map(str::trim) {
|
||||
if line.is_empty() || line.starts_with('#') || line.starts_with(';') {
|
||||
continue;
|
||||
}
|
||||
if let Some(section) = line
|
||||
.strip_prefix('[')
|
||||
.and_then(|line| line.strip_suffix(']'))
|
||||
{
|
||||
selected = selects(§ion.split_whitespace().collect::<Vec<_>>().join(" "));
|
||||
continue;
|
||||
}
|
||||
if selected
|
||||
&& let Some((key, value)) = line.split_once('=')
|
||||
&& key.trim() == "region"
|
||||
{
|
||||
return Some(value.trim().to_string()).filter(|region| !region.is_empty());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub(crate) fn sts_endpoint_region(endpoint: &str) -> Option<String> {
|
||||
let url = url::Url::parse(endpoint).ok()?;
|
||||
let labels: Vec<&str> = url.host_str()?.split('.').collect();
|
||||
(0..labels.len()).find_map(|index| {
|
||||
let ["sts" | "sts-fips", region, rest @ ..] = &labels[index..] else {
|
||||
return None;
|
||||
};
|
||||
let rest = rest.join(".");
|
||||
let valid_region = !region.is_empty()
|
||||
&& region
|
||||
.chars()
|
||||
.all(|char| char.is_ascii_lowercase() || char.is_ascii_digit() || char == '-');
|
||||
(valid_region
|
||||
&& (rest.starts_with("amazonaws.com") || rest.starts_with("vpce.amazonaws.com")))
|
||||
.then(|| region.to_string())
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::commercial("us-east-1", "https://bedrock-runtime.us-east-1.amazonaws.com")]
|
||||
#[case::china("cn-north-1", "https://bedrock-runtime.cn-north-1.amazonaws.com.cn")]
|
||||
#[case::gov_cloud("us-gov-west-1", "https://bedrock-runtime.us-gov-west-1.amazonaws.com")]
|
||||
#[case::iso("us-iso-east-1", "https://bedrock-runtime.us-iso-east-1.c2s.ic.gov")]
|
||||
#[case::iso_b(
|
||||
"us-isob-east-1",
|
||||
"https://bedrock-runtime.us-isob-east-1.sc2s.sgov.gov"
|
||||
)]
|
||||
#[case::iso_f(
|
||||
"us-isof-south-1",
|
||||
"https://bedrock-runtime.us-isof-south-1.csp.hci.ic.gov"
|
||||
)]
|
||||
#[case::iso_e(
|
||||
"eu-isoe-west-1",
|
||||
"https://bedrock-runtime.eu-isoe-west-1.cloud.adc-e.uk"
|
||||
)]
|
||||
fn the_runtime_endpoint_uses_the_partition_dns_suffix(
|
||||
#[case] region: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(bedrock_runtime_endpoint(region), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::china("cn-northwest-1", "aws-cn")]
|
||||
#[case::gov_cloud("us-gov-east-1", "aws-us-gov")]
|
||||
#[case::commercial("eu-central-1", "aws")]
|
||||
fn a_region_names_its_partition(#[case] region: &str, #[case] expected: &str) {
|
||||
assert_eq!(aws_partition(region).partition, expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::regional("https://sts.eu-west-1.amazonaws.com", Some("eu-west-1"))]
|
||||
#[case::fips("https://sts-fips.us-east-2.amazonaws.com/", Some("us-east-2"))]
|
||||
#[case::china("https://sts.cn-north-1.amazonaws.com.cn", Some("cn-north-1"))]
|
||||
#[case::vpc_endpoint(
|
||||
"https://vpce-0abc-1234.sts.ap-south-1.vpce.amazonaws.com",
|
||||
Some("ap-south-1")
|
||||
)]
|
||||
#[case::global("https://sts.amazonaws.com", None)]
|
||||
#[case::custom_host("https://sts.internal.example.com", None)]
|
||||
#[case::no_scheme("sts.eu-west-1.amazonaws.com", None)]
|
||||
fn the_sts_region_is_read_from_a_regional_endpoint(
|
||||
#[case] endpoint: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(sts_endpoint_region(endpoint).as_deref(), expected);
|
||||
}
|
||||
|
||||
const CONFIG: &str = "[default]\nregion = eu-west-3\n\n[profile work]\noutput = json\nregion=ap-northeast-1\n\n[profile empty]\nregion =\n";
|
||||
|
||||
#[rstest]
|
||||
#[case::default_profile("default", Some("eu-west-3"))]
|
||||
#[case::named_profile("work", Some("ap-northeast-1"))]
|
||||
#[case::blank_region("empty", None)]
|
||||
#[case::unknown_profile("missing", None)]
|
||||
fn the_profile_region_comes_from_its_config_section(
|
||||
#[case] profile: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
assert_eq!(config_file_region(CONFIG, profile).as_deref(), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_region_wins(Some("sa-east-1"), None, Some("sa-east-1"))]
|
||||
#[case::blank_default_region_reads_the_profile(Some(""), Some("work"), Some("ap-northeast-1"))]
|
||||
#[case::profile_from_the_environment(None, Some("work"), Some("ap-northeast-1"))]
|
||||
#[case::default_profile(None, None, Some("eu-west-3"))]
|
||||
fn the_session_region_reads_the_default_region_then_the_profile(
|
||||
#[case] default_region: Option<&str>,
|
||||
#[case] profile: Option<&str>,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let file = tempfile::NamedTempFile::new().unwrap();
|
||||
std::fs::write(file.path(), CONFIG).unwrap();
|
||||
let config = file.path().to_string_lossy().into_owned();
|
||||
let env = |name: &str| {
|
||||
(name == AWS_DEFAULT_REGION)
|
||||
.then(|| default_region.map(str::to_string))
|
||||
.flatten()
|
||||
};
|
||||
let process_env = |name: &str| match name {
|
||||
"AWS_CONFIG_FILE" => Some(config.clone()),
|
||||
"AWS_PROFILE" => profile.map(str::to_string),
|
||||
_ => None,
|
||||
};
|
||||
|
||||
assert_eq!(session_region(&env, &process_env).as_deref(), expected);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,11 +1,18 @@
|
|||
use std::collections::BTreeMap;
|
||||
use std::time::SystemTime;
|
||||
|
||||
use aws_credential_types::Credentials;
|
||||
use reqwest::{Request, header::HeaderMap};
|
||||
use aws_sigv4::sign::v4;
|
||||
use chrono::{DateTime, Utc};
|
||||
use reqwest::{
|
||||
Request,
|
||||
header::{HeaderMap, HeaderName, HeaderValue},
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::{
|
||||
AwsAuthService, AwsCredentialSource, Error, SigningScope, aws_signature_headers,
|
||||
is_sigv4_computed_header, sign_headers,
|
||||
AwsAuthService, AwsCredentialSource, Error, SigningScope, SigningStage, aws_signature_headers,
|
||||
constants::AWS4_AUTHORIZATION_PREFIX, is_sigv4_computed_header, sign_headers,
|
||||
};
|
||||
use litellm_http::outbound::RequestSigner;
|
||||
|
||||
|
|
@ -15,6 +22,14 @@ pub struct SigV4Signer {
|
|||
service: &'static str,
|
||||
credentials: Credentials,
|
||||
clock: fn() -> SystemTime,
|
||||
caller_headers: CallerSigningHeaders,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
pub enum CallerSigningHeaders {
|
||||
#[default]
|
||||
Dropped,
|
||||
Botocore,
|
||||
}
|
||||
|
||||
impl SigV4Signer {
|
||||
|
|
@ -24,6 +39,7 @@ impl SigV4Signer {
|
|||
service,
|
||||
credentials,
|
||||
clock: SystemTime::now,
|
||||
caller_headers: CallerSigningHeaders::Dropped,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -31,6 +47,18 @@ impl SigV4Signer {
|
|||
Self { clock, ..self }
|
||||
}
|
||||
|
||||
pub fn with_caller_headers(self, caller_headers: CallerSigningHeaders) -> Self {
|
||||
Self {
|
||||
caller_headers,
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
fn keeps_caller_security_token(&self) -> bool {
|
||||
self.caller_headers == CallerSigningHeaders::Botocore
|
||||
&& self.credentials.session_token().is_none()
|
||||
}
|
||||
|
||||
pub async fn resolve(
|
||||
auth: &AwsAuthService,
|
||||
region: String,
|
||||
|
|
@ -38,48 +66,246 @@ impl SigV4Signer {
|
|||
credentials: AwsCredentialSource,
|
||||
env_lookup: &(dyn Fn(&str) -> Option<String> + Sync),
|
||||
) -> Result<Self, Error> {
|
||||
Ok(Self::new(
|
||||
region,
|
||||
service,
|
||||
credentials.resolve(auth, env_lookup).await?,
|
||||
))
|
||||
let credentials = credentials
|
||||
.resolve(auth, ®ion, service, env_lookup)
|
||||
.await?;
|
||||
Ok(Self::new(region, service, credentials))
|
||||
}
|
||||
}
|
||||
|
||||
impl RequestSigner for SigV4Signer {
|
||||
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
|
||||
if let Some(name) = request
|
||||
.headers()
|
||||
.keys()
|
||||
.find(|name| is_sigv4_computed_header(name.as_str()))
|
||||
{
|
||||
return Err(litellm_http::Error::computed_header(name.as_str()));
|
||||
const CALLER_HEADERS_THE_SIGNER_REPLACES: &[&str] = &["x-amz-date", "x-amz-security-token", "date"];
|
||||
|
||||
const CALLER_DATE_HEADERS: &[&str] = &["x-amz-date", "date"];
|
||||
|
||||
const SECURITY_TOKEN: &str = "x-amz-security-token";
|
||||
|
||||
const BOTOCORE_UNSIGNED_HEADERS: &[&str] = &["expect", "user-agent", "x-amzn-trace-id"];
|
||||
|
||||
fn signable(headers: &HeaderMap, keep_security_token: bool) -> HeaderMap {
|
||||
aws_signature_headers(headers)
|
||||
.iter()
|
||||
.filter(|(name, _)| {
|
||||
!is_sigv4_computed_header(name.as_str())
|
||||
|| (keep_security_token && name.as_str() == SECURITY_TOKEN)
|
||||
})
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn hex_sha256(bytes: &[u8]) -> String {
|
||||
Sha256::digest(bytes)
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn quote_path(path: &str) -> String {
|
||||
path.bytes()
|
||||
.map(|byte| match byte {
|
||||
b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'_' | b'.' | b'-' | b'~' | b'/' => {
|
||||
char::from(byte).to_string()
|
||||
}
|
||||
other => format!("%{other:02X}"),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn canonical_uri(path: &str) -> String {
|
||||
if path.is_empty() {
|
||||
return "/".to_string();
|
||||
}
|
||||
let segments = path.split('/').fold(Vec::new(), |mut kept, segment| {
|
||||
match segment {
|
||||
"" | "." => {}
|
||||
".." => {
|
||||
kept.pop();
|
||||
}
|
||||
other => kept.push(other),
|
||||
}
|
||||
kept
|
||||
});
|
||||
let leading = if path.starts_with('/') { "/" } else { "" };
|
||||
let trailing = if path.ends_with('/') && !segments.is_empty() {
|
||||
"/"
|
||||
} else {
|
||||
""
|
||||
};
|
||||
quote_path(&format!("{leading}{}{trailing}", segments.join("/")))
|
||||
}
|
||||
|
||||
fn canonical_query(query: Option<&str>) -> String {
|
||||
let mut pairs: Vec<(&str, &str)> = query
|
||||
.filter(|query| !query.is_empty())
|
||||
.map(|query| {
|
||||
query
|
||||
.split('&')
|
||||
.map(|pair| pair.split_once('=').unwrap_or((pair, "")))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
pairs.sort_unstable();
|
||||
pairs
|
||||
.iter()
|
||||
.map(|(key, value)| format!("{key}={value}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("&")
|
||||
}
|
||||
|
||||
fn canonical_host(url: &reqwest::Url) -> String {
|
||||
let host = url.host_str().unwrap_or_default();
|
||||
match url.port() {
|
||||
Some(port) => format!("{host}:{port}"),
|
||||
None => host.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn header_pair(name: &str, value: &str) -> Result<(HeaderName, HeaderValue), Error> {
|
||||
Ok((
|
||||
HeaderName::from_bytes(name.as_bytes()).map_err(Error::signing(SigningStage::Output))?,
|
||||
HeaderValue::from_str(value).map_err(Error::signing(SigningStage::Output))?,
|
||||
))
|
||||
}
|
||||
|
||||
fn sign_with_date_header(
|
||||
request: &Request,
|
||||
body: &[u8],
|
||||
headers: &HeaderMap,
|
||||
scope: SigningScope<'_>,
|
||||
) -> Result<HeaderMap, Error> {
|
||||
let time: DateTime<Utc> = scope.time.into();
|
||||
let amz_date = time.format("%Y%m%dT%H%M%SZ").to_string();
|
||||
let http_date = time.format("%a, %d %b %Y %H:%M:%S -0000").to_string();
|
||||
let session_token = scope.credentials.session_token();
|
||||
let mut canonical: BTreeMap<String, Vec<String>> = BTreeMap::new();
|
||||
for (name, value) in headers {
|
||||
if BOTOCORE_UNSIGNED_HEADERS.contains(&name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
let value = value.to_str().map_err(|_| Error::NonAsciiHeader {
|
||||
name: name.to_string(),
|
||||
})?;
|
||||
canonical
|
||||
.entry(name.as_str().to_string())
|
||||
.or_default()
|
||||
.push(value.split_whitespace().collect::<Vec<_>>().join(" "));
|
||||
}
|
||||
canonical.insert("date".into(), vec![http_date.clone()]);
|
||||
if let Some(token) = session_token {
|
||||
canonical.insert(SECURITY_TOKEN.into(), vec![token.to_string()]);
|
||||
}
|
||||
canonical
|
||||
.entry("host".into())
|
||||
.or_insert_with(|| vec![canonical_host(request.url())]);
|
||||
let payload = headers
|
||||
.get("x-amz-content-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map_or_else(|| hex_sha256(body), str::to_string);
|
||||
let canonical_headers: String = canonical
|
||||
.iter()
|
||||
.map(|(name, values)| format!("{name}:{}\n", values.join(",")))
|
||||
.collect();
|
||||
let signed_headers = canonical.keys().cloned().collect::<Vec<_>>().join(";");
|
||||
let canonical_request = format!(
|
||||
"{}\n{}\n{}\n{canonical_headers}\n{signed_headers}\n{payload}",
|
||||
request.method().as_str(),
|
||||
canonical_uri(request.url().path()),
|
||||
canonical_query(request.url().query()),
|
||||
);
|
||||
let credential_scope = format!(
|
||||
"{}/{}/{}/aws4_request",
|
||||
&amz_date[..8],
|
||||
scope.region,
|
||||
scope.service
|
||||
);
|
||||
let string_to_sign = format!(
|
||||
"{AWS4_AUTHORIZATION_PREFIX}\n{amz_date}\n{credential_scope}\n{}",
|
||||
hex_sha256(canonical_request.as_bytes())
|
||||
);
|
||||
let signing_key = v4::generate_signing_key(
|
||||
scope.credentials.secret_access_key(),
|
||||
scope.time,
|
||||
scope.region,
|
||||
scope.service,
|
||||
);
|
||||
let signature = v4::calculate_signature(signing_key, string_to_sign.as_bytes());
|
||||
let authorization = format!(
|
||||
"{AWS4_AUTHORIZATION_PREFIX} Credential={}/{credential_scope}, SignedHeaders={signed_headers}, Signature={signature}",
|
||||
scope.credentials.access_key_id()
|
||||
);
|
||||
[
|
||||
Some(("date", http_date.as_str())),
|
||||
session_token.map(|token| (SECURITY_TOKEN, token)),
|
||||
Some(("authorization", authorization.as_str())),
|
||||
]
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.map(|(name, value)| header_pair(name, value))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn overrides_the_signature(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(reqwest::header::AUTHORIZATION)
|
||||
.is_some_and(|value| {
|
||||
!value
|
||||
.as_bytes()
|
||||
.starts_with(AWS4_AUTHORIZATION_PREFIX.as_bytes())
|
||||
})
|
||||
}
|
||||
|
||||
impl RequestSigner for SigV4Signer {
|
||||
fn owned_headers(&self) -> &'static [&'static str] {
|
||||
if self.keeps_caller_security_token() {
|
||||
CALLER_DATE_HEADERS
|
||||
} else {
|
||||
CALLER_HEADERS_THE_SIGNER_REPLACES
|
||||
}
|
||||
}
|
||||
|
||||
fn sign(&self, request: &Request) -> Result<HeaderMap, litellm_http::Error> {
|
||||
let body = match request.body() {
|
||||
None => &[],
|
||||
Some(body) => body
|
||||
.as_bytes()
|
||||
.ok_or_else(litellm_http::Error::unsignable_body)?,
|
||||
};
|
||||
sign_headers(
|
||||
request.method().as_str(),
|
||||
request.url().as_str(),
|
||||
body,
|
||||
&aws_signature_headers(request.headers()),
|
||||
SigningScope {
|
||||
region: &self.region,
|
||||
service: self.service,
|
||||
credentials: &self.credentials,
|
||||
time: (self.clock)(),
|
||||
},
|
||||
)
|
||||
let headers = signable(request.headers(), self.keeps_caller_security_token());
|
||||
let scope = SigningScope {
|
||||
region: &self.region,
|
||||
service: self.service,
|
||||
credentials: &self.credentials,
|
||||
time: (self.clock)(),
|
||||
};
|
||||
let signed = match self.caller_headers == CallerSigningHeaders::Botocore
|
||||
&& request.headers().contains_key(reqwest::header::DATE)
|
||||
{
|
||||
true => sign_with_date_header(request, body, &headers, scope),
|
||||
false => sign_headers(
|
||||
request.method().as_str(),
|
||||
request.url().as_str(),
|
||||
body,
|
||||
&headers,
|
||||
scope,
|
||||
),
|
||||
}
|
||||
.map_err(|error| match error {
|
||||
Error::NonAsciiHeader { .. } => litellm_http::Error::unsignable_header(error),
|
||||
Error::ExpiredCredentials
|
||||
| Error::Credentials { .. }
|
||||
| Error::WebIdentityTokenReference
|
||||
| Error::MissingWebIdentityToken
|
||||
| Error::WebIdentityToken(_)
|
||||
| Error::MissingWebIdentityCredentials
|
||||
| Error::WebIdentityExpiration(_)
|
||||
| Error::Signing { .. } => litellm_http::Error::signature(error),
|
||||
})?;
|
||||
Ok(match overrides_the_signature(request.headers()) {
|
||||
true => signed
|
||||
.iter()
|
||||
.filter(|(name, _)| *name != reqwest::header::AUTHORIZATION)
|
||||
.map(|(name, value)| (name.clone(), value.clone()))
|
||||
.collect(),
|
||||
false => signed,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -176,22 +402,166 @@ mod tests {
|
|||
assert!(original.contains("/us-east-1/textract/aws4_request"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_forwarded_computed_header_is_refused_instead_of_sent_twice() {
|
||||
let error = signed(
|
||||
fn sent(headers: &[(&str, &str)], credentials: Credentials) -> Request {
|
||||
signed(
|
||||
json(
|
||||
URL,
|
||||
&[("authorization".into(), "Bearer caller".into())],
|
||||
&headers
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect::<Vec<_>>(),
|
||||
&json_value!({}),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
&signer("textract"),
|
||||
&SigV4Signer::new("us-east-1".into(), "bedrock", credentials).with_clock(fixed_clock),
|
||||
)
|
||||
.unwrap_err();
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
assert!(error.is_computed_header());
|
||||
assert!(error.to_string().contains("authorization"), "{error}");
|
||||
fn header<'a>(request: &'a Request, name: &str) -> Vec<&'a str> {
|
||||
request
|
||||
.headers()
|
||||
.get_all(name)
|
||||
.iter()
|
||||
.map(|value| value.to_str().unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::caller_date("date", None)]
|
||||
#[case::caller_amz_date("x-amz-date", Some("20231114T221320Z"))]
|
||||
#[case::caller_security_token_without_a_session("x-amz-security-token", None)]
|
||||
fn a_header_the_signer_computes_is_dropped_from_the_callers_copy(
|
||||
#[case] name: &str,
|
||||
#[case] expected: Option<&str>,
|
||||
) {
|
||||
let request = sent(
|
||||
&[(name, "caller-value")],
|
||||
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
header(&request, name),
|
||||
expected.into_iter().collect::<Vec<_>>()
|
||||
);
|
||||
assert!(header(&request, "authorization")[0].starts_with("AWS4-HMAC-SHA256 "));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_session_token_replaces_the_callers_security_token() {
|
||||
let request = sent(
|
||||
&[("X-Amz-Security-Token", "caller-value")],
|
||||
Credentials::new(
|
||||
"AKIDEXAMPLE",
|
||||
"secret",
|
||||
Some("session".into()),
|
||||
None,
|
||||
"test",
|
||||
),
|
||||
);
|
||||
|
||||
assert_eq!(header(&request, "x-amz-security-token"), ["session"]);
|
||||
}
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::bearer_overrides_the_signature("Bearer gateway", "Bearer gateway")]
|
||||
#[case::a_stale_signature_is_replaced(
|
||||
"AWS4-HMAC-SHA256 Credential=stale",
|
||||
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders=content-type;host;x-amz-date"
|
||||
)]
|
||||
fn a_callers_authorization_overrides_the_signature_unless_it_is_sigv4(
|
||||
#[case] caller: &str,
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let request = sent(
|
||||
&[("Authorization", caller)],
|
||||
Credentials::new("AKIDEXAMPLE", "secret", None, None, "test"),
|
||||
);
|
||||
let authorization = header(&request, "authorization");
|
||||
|
||||
assert_eq!(authorization.len(), 1);
|
||||
assert_eq!(
|
||||
authorization[0].split(", Signature=").next(),
|
||||
Some(expected)
|
||||
);
|
||||
assert_eq!(header(&request, "x-amz-date"), ["20231114T221320Z"]);
|
||||
}
|
||||
|
||||
const BEDROCK_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/us-west-2/anthropic.claude-3-5-sonnet-20240620-v1:0/invoke";
|
||||
const ARN_URL: &str = "https://bedrock-runtime.us-east-1.amazonaws.com/model/arn%3Aaws%3Abedrock%3Aus-east-1%3A1%3Ainference-profile%2Fx/invoke?b=2&a=1";
|
||||
const CALLER_SIGNING_HEADERS: &[(&str, &str)] = &[
|
||||
("date", "Wed, 01 Jan 2020 00:00:00 GMT"),
|
||||
("x-amz-date", "20200101T000000Z"),
|
||||
("x-amz-security-token", "caller-token"),
|
||||
];
|
||||
const CALLER_TOKEN: &[(&str, &str)] = &[("x-amz-security-token", "caller-token")];
|
||||
const CALLER_DATE: &[(&str, &str)] = &[("Date", "Wed, 01 Jan 2020 00:00:00 GMT")];
|
||||
const HTTP_DATE: &str = "Tue, 14 Nov 2023 22:13:20 -0000";
|
||||
const AMZ_DATE: &str = "20231114T221320Z";
|
||||
|
||||
#[rstest::rstest]
|
||||
#[case::date_and_caller_token(URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "1b3652d523d5296b3904a80981e936a201167da9cb7bd200c2da8171dadb2c08")]
|
||||
#[case::caller_token_only(URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "0fbd1a8e98efbfbcdb6fa9d665fe89ffffcdd8a7656865d63def21a5344d02dd")]
|
||||
#[case::date_only(URL, CALLER_DATE, None, (Some(HTTP_DATE), None, None), "content-type;date;host", "cccdac7b04f20ebfe37a156c4e806a14e6c0a4b0aad11ca67e25dca337237085")]
|
||||
#[case::session_replaces_the_caller_token_under_a_date(URL, CALLER_SIGNING_HEADERS, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "3406b8200cae95c8b66037dc5a9a838e1fe4da94b57b89fada3c05cde8a2c15c")]
|
||||
#[case::session_replaces_the_caller_token(URL, CALLER_TOKEN, Some("session"), (None, Some(AMZ_DATE), Some("session")), "content-type;host;x-amz-date;x-amz-security-token", "d75425952d9f5ef7465a6ef3acabf15908c0edfd8ce2529cc3f0f8d2e9232e76")]
|
||||
#[case::colon_in_the_model_path(BEDROCK_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "47d75a0dcb00499a025fc0f9f010f6dce89c7661eb54cb97c3a293e9c585b153")]
|
||||
#[case::encoded_arn_and_query(ARN_URL, CALLER_SIGNING_HEADERS, None, (Some(HTTP_DATE), None, Some("caller-token")), "content-type;date;host;x-amz-security-token", "8fb7d97a0ad5df9d1c3bbfc55897266059a5a2f35a1aaed9687f370047e78363")]
|
||||
#[case::encoded_arn_and_query_without_a_date(ARN_URL, CALLER_TOKEN, None, (None, Some(AMZ_DATE), Some("caller-token")), "content-type;host;x-amz-date;x-amz-security-token", "f5eb993af90c7896203bf6b3b85c0087087f712a0634e44ac0a04c9d87d6fe12")]
|
||||
#[case::encoded_arn_and_query_with_a_session(ARN_URL, CALLER_DATE, Some("session"), (Some(HTTP_DATE), None, Some("session")), "content-type;date;host;x-amz-security-token", "15e8d5a9f64eb64366363f951db64de2a747959573a8cda9b5faa0fcbabc713c")]
|
||||
fn botocore_caller_headers_sign_like_botocore(
|
||||
#[case] url: &str,
|
||||
#[case] caller: &[(&str, &str)],
|
||||
#[case] session: Option<&str>,
|
||||
#[case] sent: (Option<&str>, Option<&str>, Option<&str>),
|
||||
#[case] signed_headers: &str,
|
||||
#[case] signature: &str,
|
||||
) {
|
||||
let request = signed(
|
||||
json(
|
||||
url,
|
||||
&caller
|
||||
.iter()
|
||||
.map(|(name, value)| (name.to_string(), value.to_string()))
|
||||
.collect::<Vec<_>>(),
|
||||
&json_value!({}),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
&SigV4Signer::new(
|
||||
"us-east-1".into(),
|
||||
"bedrock",
|
||||
Credentials::new(
|
||||
"AKIDEXAMPLE",
|
||||
"secret",
|
||||
session.map(str::to_string),
|
||||
None,
|
||||
"test",
|
||||
),
|
||||
)
|
||||
.with_clock(fixed_clock)
|
||||
.with_caller_headers(CallerSigningHeaders::Botocore),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
(
|
||||
header(&request, "date"),
|
||||
header(&request, "x-amz-date"),
|
||||
header(&request, "x-amz-security-token"),
|
||||
header(&request, "authorization"),
|
||||
),
|
||||
(
|
||||
sent.0.into_iter().collect::<Vec<_>>(),
|
||||
sent.1.into_iter().collect::<Vec<_>>(),
|
||||
sent.2.into_iter().collect::<Vec<_>>(),
|
||||
vec![format!(
|
||||
"AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/20231114/us-east-1/bedrock/aws4_request, SignedHeaders={signed_headers}, Signature={signature}"
|
||||
)
|
||||
.as_str()],
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -52,6 +52,6 @@ pub use credential::{
|
|||
};
|
||||
pub use error::{Error, ErrorDetail};
|
||||
pub use http::CredentialPlacement;
|
||||
pub use params::{AwsParams, ParamKind, ParamSpec, VertexParams};
|
||||
pub use params::{AwsParams, AwsSessionTag, ParamKind, ParamSpec, VertexParams};
|
||||
pub use policy::{CredentialPlanKind, CredentialRule, ExistingHeaderBehavior, ProviderAuthPolicy};
|
||||
pub use token::{ResolvedCredential, TokenFuture, TokenProvider, TokenProviderHandle};
|
||||
|
|
|
|||
|
|
@ -21,6 +21,14 @@ const fn spec(
|
|||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub struct AwsSessionTag {
|
||||
#[serde(rename = "Key")]
|
||||
pub key: String,
|
||||
#[serde(rename = "Value")]
|
||||
pub value: String,
|
||||
}
|
||||
|
||||
/// The `aws_*` fields of Python's `GenericLiteLLMParams`.
|
||||
#[derive(Redact, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct AwsParams {
|
||||
|
|
@ -50,11 +58,15 @@ pub struct AwsParams {
|
|||
pub aws_external_id: Option<String>,
|
||||
#[serde(default)]
|
||||
pub aws_bedrock_runtime_endpoint: Option<String>,
|
||||
#[serde(default)]
|
||||
pub aws_session_tags: Option<Vec<AwsSessionTag>>,
|
||||
}
|
||||
|
||||
impl AwsParams {
|
||||
pub const ROLE_ARN_ENV: &str = "AWS_ROLE_ARN";
|
||||
pub const WEB_IDENTITY_TOKEN_FILE_ENV: &str = "AWS_WEB_IDENTITY_TOKEN_FILE";
|
||||
pub const DEFAULT_REGION_ENV: &str = "AWS_DEFAULT_REGION";
|
||||
pub const SESSION_TAGS: &str = "aws_session_tags";
|
||||
|
||||
pub const ACCESS_KEY_ID: ParamSpec = spec(
|
||||
"access key id",
|
||||
|
|
@ -144,6 +156,7 @@ impl AwsParams {
|
|||
Self::SPECS
|
||||
.iter()
|
||||
.flat_map(|spec| spec.wire.iter().copied())
|
||||
.chain([Self::SESSION_TAGS])
|
||||
}
|
||||
|
||||
/// The environment names the specs read, for hosts that resolve secrets up front.
|
||||
|
|
@ -153,6 +166,7 @@ impl AwsParams {
|
|||
.iter()
|
||||
.flat_map(|spec| spec.env.iter().copied())
|
||||
.chain([
|
||||
AwsParams::DEFAULT_REGION_ENV,
|
||||
AwsParams::ROLE_ARN_ENV,
|
||||
AwsParams::WEB_IDENTITY_TOKEN_FILE_ENV,
|
||||
])
|
||||
|
|
@ -205,6 +219,9 @@ impl AwsParams {
|
|||
aws_sts_endpoint: value("aws_sts_endpoint"),
|
||||
aws_external_id: value("aws_external_id"),
|
||||
aws_bedrock_runtime_endpoint: value("aws_bedrock_runtime_endpoint"),
|
||||
aws_session_tags: optional_params
|
||||
.get(Self::SESSION_TAGS)
|
||||
.and_then(|tags| serde_json::from_value(tags.clone()).ok()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
mod aws;
|
||||
mod vertex;
|
||||
|
||||
pub use aws::AwsParams;
|
||||
pub use aws::{AwsParams, AwsSessionTag};
|
||||
pub use vertex::VertexParams;
|
||||
|
||||
/// One connection param of Python's `CredentialLiteLLMParams` and every place Python reads
|
||||
|
|
|
|||
|
|
@ -1,11 +1,16 @@
|
|||
use litellm_auth_types::AwsParams;
|
||||
use rstest::rstest;
|
||||
use serde_json::{Map, Value};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
#[rstest]
|
||||
fn every_named_param_roundtrips_without_loss() {
|
||||
let filled: Map<String, Value> = AwsParams::fields()
|
||||
.map(|name| (name.to_string(), Value::from(format!("value-of-{name}"))))
|
||||
.map(|name| match name {
|
||||
AwsParams::SESSION_TAGS => {
|
||||
(name.to_string(), json!([{"Key": "team", "Value": "genai"}]))
|
||||
}
|
||||
_ => (name.to_string(), Value::from(format!("value-of-{name}"))),
|
||||
})
|
||||
.collect();
|
||||
let typed = AwsParams::from_optional_params(&filled);
|
||||
let serialized = serde_json::to_value(&typed).unwrap();
|
||||
|
|
|
|||
20
litellm/rust/crates/callbacks-budgets/Cargo.toml
Normal file
20
litellm/rust/crates/callbacks-budgets/Cargo.toml
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
[package]
|
||||
name = "litellm-callbacks-budgets"
|
||||
version = "0.1.0"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[dependencies]
|
||||
litellm-callbacks.workspace = true
|
||||
litellm-python-compat.workspace = true
|
||||
litellm-redis-store.workspace = true
|
||||
litellm-tracing.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
litellm-coroutine.workspace = true
|
||||
rstest.workspace = true
|
||||
tokio = { workspace = true, features = ["macros", "rt", "time", "test-util"] }
|
||||
128
litellm/rust/crates/callbacks-budgets/src/admission.rs
Normal file
128
litellm/rust/crates/callbacks-budgets/src/admission.rs
Normal file
|
|
@ -0,0 +1,128 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_callbacks::{BudgetPlan, BudgetReservation, ReservationEntry};
|
||||
use litellm_redis_store::spend_counters::SpendCounters;
|
||||
use litellm_tracing::Logger;
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{
|
||||
engine::{self, BudgetCounterSpend, EngineError, Reservation, Settle, Settlement, UNAVAILABLE},
|
||||
plan::Plan,
|
||||
window::Moment,
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
#[serde(tag = "admission", rename_all = "snake_case")]
|
||||
pub enum Admission {
|
||||
Skipped,
|
||||
Held {
|
||||
reservation: BudgetReservation,
|
||||
},
|
||||
Refused {
|
||||
kind: &'static str,
|
||||
fields: Map<String, Value>,
|
||||
},
|
||||
}
|
||||
|
||||
pub struct Budgets {
|
||||
store: Arc<SpendCounters>,
|
||||
logger: Option<Logger>,
|
||||
}
|
||||
|
||||
fn refusal(error: &EngineError) -> Admission {
|
||||
let (kind, fields) = match error {
|
||||
EngineError::Exceeded(exceeded) => (
|
||||
"budget_exceeded",
|
||||
Map::from_iter([
|
||||
("current_cost".to_owned(), json!(exceeded.current_cost)),
|
||||
("max_budget".to_owned(), json!(exceeded.max_budget)),
|
||||
("message".to_owned(), json!(exceeded.message)),
|
||||
("entity_type".to_owned(), json!(exceeded.entity_type)),
|
||||
("entity_id".to_owned(), json!(exceeded.entity_id)),
|
||||
]),
|
||||
),
|
||||
EngineError::Unavailable => (
|
||||
"budget_unavailable",
|
||||
Map::from_iter([("detail".to_owned(), json!(UNAVAILABLE))]),
|
||||
),
|
||||
other => (
|
||||
"budget_reservation_failed",
|
||||
Map::from_iter([("message".to_owned(), json!(other.to_string()))]),
|
||||
),
|
||||
};
|
||||
Admission::Refused { kind, fields }
|
||||
}
|
||||
|
||||
fn reservation_record(reservation: &Reservation) -> BudgetReservation {
|
||||
BudgetReservation {
|
||||
reserved_cost: Some(reservation.reserved_cost),
|
||||
entries: Some(
|
||||
reservation
|
||||
.entries
|
||||
.iter()
|
||||
.map(|entry| {
|
||||
Some(ReservationEntry {
|
||||
counter_key: Some(entry.counter_key.clone()),
|
||||
entity_type: Some(entry.entity_type.clone()),
|
||||
entity_id: Some(entry.entity_id.clone()),
|
||||
reserved_cost: Some(entry.reserved_cost),
|
||||
applied_adjustment: Some(entry.applied_adjustment),
|
||||
..ReservationEntry::default()
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
),
|
||||
input_cost: Some(reservation.input_cost),
|
||||
input_tokens: reservation.input_tokens,
|
||||
..BudgetReservation::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Budgets {
|
||||
pub fn new(store: Arc<SpendCounters>) -> Self {
|
||||
Self {
|
||||
store,
|
||||
logger: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_logger(self, logger: Logger) -> Self {
|
||||
Self {
|
||||
logger: Some(logger),
|
||||
..self
|
||||
}
|
||||
}
|
||||
|
||||
async fn logged<T>(&self, future: impl Future<Output = T>) -> T {
|
||||
match &self.logger {
|
||||
Some(logger) => logger.instrument(future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn settle(&self, request: &Settle<'_>) -> Result<Settlement, EngineError> {
|
||||
self.logged(engine::settle(&self.store, request, &Moment::now()))
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn admit(
|
||||
&self,
|
||||
plan: BudgetPlan,
|
||||
observed: &[BudgetCounterSpend],
|
||||
) -> Result<Admission, String> {
|
||||
let plan = Plan::try_from(plan).map_err(|error| error.to_string())?;
|
||||
Ok(
|
||||
match self
|
||||
.logged(engine::reserve(&self.store, &plan, observed))
|
||||
.await
|
||||
{
|
||||
Ok(None) => Admission::Skipped,
|
||||
Ok(Some(reservation)) => Admission::Held {
|
||||
reservation: reservation_record(&reservation),
|
||||
},
|
||||
Err(error) => refusal(&error),
|
||||
},
|
||||
)
|
||||
}
|
||||
}
|
||||
496
litellm/rust/crates/callbacks-budgets/src/checks.rs
Normal file
496
litellm/rust/crates/callbacks-budgets/src/checks.rs
Normal file
|
|
@ -0,0 +1,496 @@
|
|||
use litellm_python_compat::{number::PyNumber, unicode::isspace};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
fn is_masked_key_name(name: &str) -> bool {
|
||||
let Some(rest) = name.strip_prefix("sk-...") else {
|
||||
return false;
|
||||
};
|
||||
rest.is_empty()
|
||||
|| (rest.chars().count() == 4
|
||||
&& rest
|
||||
.chars()
|
||||
.all(|c| !isspace(c) && !matches!(c, '\u{0}'..='\u{1f}' | '\u{7f}'..='\u{9f}')))
|
||||
}
|
||||
|
||||
fn py_str(value: Option<&str>) -> &str {
|
||||
value.unwrap_or("None")
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Deserialize)]
|
||||
#[serde(tag = "check", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum Check {
|
||||
KeyMax {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
token: Option<String>,
|
||||
key_alias: Option<String>,
|
||||
key_name: Option<String>,
|
||||
throttle_pct: Option<PyNumber>,
|
||||
},
|
||||
KeyWindow {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
budget_duration: String,
|
||||
token: Option<String>,
|
||||
},
|
||||
KeySoft {
|
||||
spend: PyNumber,
|
||||
soft_budget: Option<PyNumber>,
|
||||
},
|
||||
TeamMax {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
team_id: Option<String>,
|
||||
},
|
||||
TeamWindow {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
budget_duration: String,
|
||||
team_id: Option<String>,
|
||||
},
|
||||
UserWindow {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
budget_duration: String,
|
||||
user_id: String,
|
||||
},
|
||||
TeamSoft {
|
||||
spend: Option<PyNumber>,
|
||||
soft_budget: Option<PyNumber>,
|
||||
},
|
||||
ProjectMax {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
project_id: Option<String>,
|
||||
},
|
||||
ProjectSoft {
|
||||
spend: Option<PyNumber>,
|
||||
soft_budget: Option<PyNumber>,
|
||||
},
|
||||
Organization {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
org_id: String,
|
||||
},
|
||||
Tag {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
tag: String,
|
||||
},
|
||||
ModelAccessGroup {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
group: String,
|
||||
},
|
||||
User {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
user_id: Option<String>,
|
||||
},
|
||||
TeamMember {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
user_id: Option<String>,
|
||||
team_id: Option<String>,
|
||||
},
|
||||
TeamMemberPositive {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
user_id: Option<String>,
|
||||
team_id: Option<String>,
|
||||
},
|
||||
EndUser {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
end_user_id: Option<String>,
|
||||
},
|
||||
Proxy {
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Refusal {
|
||||
pub current_cost: PyNumber,
|
||||
pub max_budget: PyNumber,
|
||||
pub message: String,
|
||||
pub entity_type: &'static str,
|
||||
pub entity_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
#[serde(tag = "decision", rename_all = "snake_case")]
|
||||
pub enum Decision {
|
||||
Skip,
|
||||
Pass,
|
||||
Alert,
|
||||
Throttle { pct: PyNumber },
|
||||
Refuse(Refusal),
|
||||
}
|
||||
|
||||
fn refuse(
|
||||
spend: PyNumber,
|
||||
max_budget: PyNumber,
|
||||
message: String,
|
||||
entity_type: &'static str,
|
||||
entity_id: Option<String>,
|
||||
) -> Decision {
|
||||
Decision::Refuse(Refusal {
|
||||
current_cost: spend,
|
||||
max_budget,
|
||||
message,
|
||||
entity_type,
|
||||
entity_id,
|
||||
})
|
||||
}
|
||||
|
||||
fn current(spend: PyNumber, max_budget: PyNumber, label: &str) -> String {
|
||||
format!(
|
||||
"Budget has been exceeded! {label} Current cost: {}, Max budget: {}",
|
||||
spend.text(),
|
||||
max_budget.text()
|
||||
)
|
||||
}
|
||||
|
||||
fn window(spend: PyNumber, max_budget: PyNumber) -> String {
|
||||
format!("Spend=${}, Limit=${}", spend.fixed(4), max_budget.fixed(2))
|
||||
}
|
||||
|
||||
fn at_or_over(spend: PyNumber, max_budget: PyNumber) -> bool {
|
||||
max_budget.value().is_finite() && spend.value() >= max_budget.value()
|
||||
}
|
||||
|
||||
fn over(spend: PyNumber, max_budget: PyNumber) -> bool {
|
||||
max_budget.value().is_finite() && spend.value() > max_budget.value()
|
||||
}
|
||||
|
||||
fn crossed(spend: Option<PyNumber>, soft_budget: Option<PyNumber>) -> Decision {
|
||||
match (spend, soft_budget) {
|
||||
(Some(spend), Some(soft)) if spend.value() >= soft.value() => Decision::Alert,
|
||||
_ => Decision::Pass,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decide(check: &Check) -> Decision {
|
||||
match check.clone() {
|
||||
Check::KeyMax {
|
||||
spend,
|
||||
max_budget,
|
||||
token,
|
||||
key_alias,
|
||||
key_name,
|
||||
throttle_pct,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
if let Some(pct) = throttle_pct {
|
||||
return Decision::Throttle { pct };
|
||||
}
|
||||
let label = key_alias.filter(|alias| !alias.is_empty());
|
||||
let label = label.as_deref().unwrap_or("key");
|
||||
let descriptor = match key_name.as_deref() {
|
||||
Some(name) if !name.is_empty() && is_masked_key_name(name) => {
|
||||
format!("{label} ({name})")
|
||||
}
|
||||
_ => label.to_owned(),
|
||||
};
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("Key={descriptor}")),
|
||||
"key",
|
||||
token,
|
||||
)
|
||||
}
|
||||
Check::KeyWindow {
|
||||
spend,
|
||||
max_budget,
|
||||
budget_duration,
|
||||
token,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"ExceededBudget: Key over {budget_duration} budget. {}",
|
||||
window(spend, max_budget)
|
||||
),
|
||||
"key",
|
||||
token,
|
||||
)
|
||||
}
|
||||
Check::KeySoft { spend, soft_budget } => match soft_budget {
|
||||
Some(soft) if soft.value() != 0.0 && spend.value() >= soft.value() => Decision::Alert,
|
||||
_ => Decision::Pass,
|
||||
},
|
||||
Check::TeamMax {
|
||||
spend,
|
||||
max_budget,
|
||||
team_id,
|
||||
} => {
|
||||
if !over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
let label = format!("Team={}", py_str(team_id.as_deref()));
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &label),
|
||||
"team",
|
||||
team_id,
|
||||
)
|
||||
}
|
||||
Check::TeamWindow {
|
||||
spend,
|
||||
max_budget,
|
||||
budget_duration,
|
||||
team_id,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"ExceededBudget: Team={} over {budget_duration} budget. {}",
|
||||
py_str(team_id.as_deref()),
|
||||
window(spend, max_budget)
|
||||
),
|
||||
"team",
|
||||
team_id,
|
||||
)
|
||||
}
|
||||
Check::UserWindow {
|
||||
spend,
|
||||
max_budget,
|
||||
budget_duration,
|
||||
user_id,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"ExceededBudget: User={user_id} over {budget_duration} budget. {}",
|
||||
window(spend, max_budget)
|
||||
),
|
||||
"user",
|
||||
Some(user_id),
|
||||
)
|
||||
}
|
||||
Check::TeamSoft { spend, soft_budget } | Check::ProjectSoft { spend, soft_budget } => {
|
||||
crossed(spend, soft_budget)
|
||||
}
|
||||
Check::ProjectMax {
|
||||
spend,
|
||||
max_budget,
|
||||
project_id,
|
||||
} => {
|
||||
if !max_budget.value().is_finite() {
|
||||
return Decision::Skip;
|
||||
}
|
||||
if spend.value() < max_budget.value() || spend.value().is_nan() {
|
||||
return Decision::Pass;
|
||||
}
|
||||
let label = format!("Project={}", py_str(project_id.as_deref()));
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &label),
|
||||
"project",
|
||||
project_id,
|
||||
)
|
||||
}
|
||||
Check::Organization {
|
||||
spend,
|
||||
max_budget,
|
||||
org_id,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("Organization={org_id}")),
|
||||
"organization",
|
||||
Some(org_id),
|
||||
)
|
||||
}
|
||||
Check::Tag {
|
||||
spend,
|
||||
max_budget,
|
||||
tag,
|
||||
} => {
|
||||
if spend.value() <= max_budget.value() {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("Tag={tag}")),
|
||||
"tag",
|
||||
Some(tag),
|
||||
)
|
||||
}
|
||||
Check::ModelAccessGroup {
|
||||
spend,
|
||||
max_budget,
|
||||
group,
|
||||
} => {
|
||||
if max_budget.value() <= 0.0 {
|
||||
return Decision::Skip;
|
||||
}
|
||||
if spend.value() < max_budget.value() {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("Model access group={group}")),
|
||||
"model_access_group",
|
||||
Some(group),
|
||||
)
|
||||
}
|
||||
Check::User {
|
||||
spend,
|
||||
max_budget,
|
||||
user_id,
|
||||
} => {
|
||||
if !at_or_over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"ExceededBudget: User={} over budget. Spend={}, Budget={}",
|
||||
py_str(user_id.as_deref()),
|
||||
spend.text(),
|
||||
max_budget.text()
|
||||
),
|
||||
"user",
|
||||
user_id,
|
||||
)
|
||||
}
|
||||
Check::TeamMember {
|
||||
spend,
|
||||
max_budget,
|
||||
user_id,
|
||||
team_id,
|
||||
} => {
|
||||
if !max_budget.value().is_finite() {
|
||||
return Decision::Skip;
|
||||
}
|
||||
if spend.value() < max_budget.value() || spend.value().is_nan() {
|
||||
return Decision::Pass;
|
||||
}
|
||||
let (user, team) = (py_str(user_id.as_deref()), py_str(team_id.as_deref()));
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("User={user} in Team={team}")),
|
||||
"team_member",
|
||||
Some(format!("{user}:{team}")),
|
||||
)
|
||||
}
|
||||
Check::TeamMemberPositive {
|
||||
spend,
|
||||
max_budget,
|
||||
user_id,
|
||||
team_id,
|
||||
} => {
|
||||
if spend.value() < max_budget.value()
|
||||
|| spend.value().is_nan()
|
||||
|| max_budget.value().is_nan()
|
||||
{
|
||||
return Decision::Pass;
|
||||
}
|
||||
let entity = format!(
|
||||
"{}:{}",
|
||||
py_str(user_id.as_deref()),
|
||||
py_str(team_id.as_deref())
|
||||
);
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
current(spend, max_budget, &format!("TeamMember={entity}")),
|
||||
"team_member",
|
||||
Some(entity),
|
||||
)
|
||||
}
|
||||
Check::EndUser {
|
||||
spend,
|
||||
max_budget,
|
||||
end_user_id,
|
||||
} => {
|
||||
if spend.value() <= max_budget.value()
|
||||
|| spend.value().is_nan()
|
||||
|| max_budget.value().is_nan()
|
||||
{
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"ExceededBudget: End User={} over budget. Spend={}, Budget={}",
|
||||
py_str(end_user_id.as_deref()),
|
||||
spend.text(),
|
||||
max_budget.text()
|
||||
),
|
||||
"end_user",
|
||||
end_user_id,
|
||||
)
|
||||
}
|
||||
Check::Proxy { spend, max_budget } => {
|
||||
if !over(spend, max_budget) {
|
||||
return Decision::Pass;
|
||||
}
|
||||
refuse(
|
||||
spend,
|
||||
max_budget,
|
||||
format!(
|
||||
"Budget has been exceeded! Current cost: {}, Max budget: {}",
|
||||
spend.text(),
|
||||
max_budget.text()
|
||||
),
|
||||
"proxy",
|
||||
None,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case("sk-...", true)]
|
||||
#[case("sk-...abcd", true)]
|
||||
#[case("sk-...abc", false)]
|
||||
#[case("sk-...ab d", false)]
|
||||
#[case("sk-...abcde", false)]
|
||||
#[case("sk-...ab\u{85}d", false)]
|
||||
#[case("sk-...ab\u{3000}d", false)]
|
||||
#[case("sk-...ab\u{202f}d", false)]
|
||||
#[case("sk-...ab\u{2028}d", false)]
|
||||
#[case("sk-...ab\u{200b}d", true)]
|
||||
#[case("sk-...ab\u{180e}d", true)]
|
||||
#[case("sk-...ab\u{feff}d", true)]
|
||||
#[case("my-key", false)]
|
||||
fn masked_key_names_match_the_proxy_pattern(#[case] name: &str, #[case] masked: bool) {
|
||||
assert_eq!(is_masked_key_name(name), masked);
|
||||
}
|
||||
}
|
||||
992
litellm/rust/crates/callbacks-budgets/src/engine.rs
Normal file
992
litellm/rust/crates/callbacks-budgets/src/engine.rs
Normal file
|
|
@ -0,0 +1,992 @@
|
|||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
|
||||
use litellm_callbacks::{BudgetReservation, ReservationEntry};
|
||||
use litellm_python_compat::repr::float_repr;
|
||||
use litellm_redis_store::{
|
||||
Number,
|
||||
spend_counters::{NotASpendCounter, SpendCounter, SpendCounters},
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
plan::{Counter, Plan, entity_type_value},
|
||||
window::{self, Moment, ReservationError},
|
||||
};
|
||||
|
||||
#[derive(Clone, Debug, Default, PartialEq, Deserialize)]
|
||||
#[serde(default, deny_unknown_fields)]
|
||||
pub struct BudgetCounterSpend {
|
||||
pub counter_key: Option<String>,
|
||||
pub spend: Option<f64>,
|
||||
}
|
||||
|
||||
const EPSILON: f64 = 1e-12;
|
||||
|
||||
static ENTRY_IDS: AtomicUsize = AtomicUsize::new(0);
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Entry {
|
||||
pub counter_key: String,
|
||||
pub entity_type: String,
|
||||
pub entity_id: String,
|
||||
pub reserved_cost: f64,
|
||||
pub applied_adjustment: f64,
|
||||
#[serde(skip)]
|
||||
id: usize,
|
||||
#[serde(skip)]
|
||||
counter: SpendCounter,
|
||||
}
|
||||
|
||||
impl Entry {
|
||||
pub fn counter(&self) -> &SpendCounter {
|
||||
&self.counter
|
||||
}
|
||||
|
||||
fn new(counter: &Counter, reserved_cost: f64) -> Self {
|
||||
Self {
|
||||
counter_key: counter.planned.counter_key.clone(),
|
||||
entity_type: counter.planned.entity_type.clone(),
|
||||
entity_id: counter.planned.entity_id.clone(),
|
||||
reserved_cost,
|
||||
applied_adjustment: 0.0,
|
||||
id: ENTRY_IDS.fetch_add(1, Ordering::Relaxed),
|
||||
counter: counter.counter.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Reservation {
|
||||
pub reserved_cost: f64,
|
||||
pub entries: Vec<Entry>,
|
||||
pub input_cost: f64,
|
||||
pub input_tokens: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Exceeded {
|
||||
pub current_cost: f64,
|
||||
pub max_budget: f64,
|
||||
pub message: String,
|
||||
pub entity_type: Option<&'static str>,
|
||||
pub entity_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum EngineError {
|
||||
Exceeded(Exceeded),
|
||||
Unavailable,
|
||||
Inconsistent(String),
|
||||
Store(litellm_redis_store::Error),
|
||||
Plan(NotASpendCounter),
|
||||
Reservation(ReservationError),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for EngineError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
Self::Exceeded(exceeded) => f.write_str(&exceeded.message),
|
||||
Self::Unavailable => f.write_str(UNAVAILABLE),
|
||||
Self::Inconsistent(message) => f.write_str(message),
|
||||
Self::Store(error) => write!(f, "{error}"),
|
||||
Self::Plan(error) => write!(f, "{error}"),
|
||||
Self::Reservation(error) => write!(f, "{error}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<ReservationError> for EngineError {
|
||||
fn from(error: ReservationError) -> Self {
|
||||
Self::Reservation(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<litellm_redis_store::Error> for EngineError {
|
||||
fn from(error: litellm_redis_store::Error) -> Self {
|
||||
Self::Store(error)
|
||||
}
|
||||
}
|
||||
|
||||
pub const UNAVAILABLE: &str = "Budget enforcement unavailable: the budget reservation could not be written to the spend counter backend, and fail_closed_budget_enforcement is enabled, so the request was rejected to avoid exceeding the configured budget. Retry shortly.";
|
||||
|
||||
fn exceeded(counter: &Counter, current_cost: f64, estimated_cost: Option<f64>) -> EngineError {
|
||||
let planned = &counter.planned;
|
||||
let estimate = estimated_cost
|
||||
.map(|cost| format!("Estimated request cost: {}, ", float_repr(cost)))
|
||||
.unwrap_or_default();
|
||||
EngineError::Exceeded(Exceeded {
|
||||
current_cost,
|
||||
max_budget: planned.max_budget,
|
||||
message: format!(
|
||||
"Budget has been exceeded! {}={} Current cost: {}, {}Max budget: {}",
|
||||
planned.entity_type,
|
||||
planned.entity_id,
|
||||
float_repr(current_cost),
|
||||
estimate,
|
||||
float_repr(planned.max_budget),
|
||||
),
|
||||
entity_type: entity_type_value(&planned.entity_type),
|
||||
entity_id: planned
|
||||
.spend_log_entity_id
|
||||
.clone()
|
||||
.unwrap_or_else(|| planned.entity_id.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn observed_spend(observed: &[BudgetCounterSpend], counter: &Counter) -> f64 {
|
||||
observed
|
||||
.iter()
|
||||
.find(|spend| spend.counter_key.as_deref() == Some(counter.key()))
|
||||
.and_then(|spend| spend.spend)
|
||||
.unwrap_or(counter.planned.fallback_spend)
|
||||
}
|
||||
|
||||
fn applies_to(value: Option<f64>, adjustment: f64) -> bool {
|
||||
match value {
|
||||
None => false,
|
||||
Some(value) => !(adjustment < 0.0 && value + adjustment < -EPSILON),
|
||||
}
|
||||
}
|
||||
|
||||
async fn can_apply(
|
||||
store: &SpendCounters,
|
||||
counter: &SpendCounter,
|
||||
adjustment: f64,
|
||||
) -> Result<bool, EngineError> {
|
||||
Ok(applies_to(
|
||||
store.read(counter).await?.value().map(Number::as_f64),
|
||||
adjustment,
|
||||
))
|
||||
}
|
||||
|
||||
async fn increment_or_invalidate(
|
||||
store: &SpendCounters,
|
||||
applicable: Vec<(SpendCounter, Number)>,
|
||||
) -> Result<(), EngineError> {
|
||||
if applicable.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(error) = store.increment_many(&applicable).await {
|
||||
let counters: Vec<SpendCounter> = applicable.into_iter().map(|(c, _)| c).collect();
|
||||
store.invalidate(&counters).await;
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, PartialEq, Eq)]
|
||||
pub(crate) enum Inconsistent {
|
||||
Drop,
|
||||
Refuse,
|
||||
}
|
||||
|
||||
struct Adjustment {
|
||||
index: usize,
|
||||
counter: SpendCounter,
|
||||
counter_key: String,
|
||||
target: f64,
|
||||
adjustment: f64,
|
||||
}
|
||||
|
||||
async fn apply(
|
||||
store: &SpendCounters,
|
||||
adjustments: &[Adjustment],
|
||||
inconsistent: Inconsistent,
|
||||
) -> Result<Vec<bool>, EngineError> {
|
||||
let mut consistent = Vec::with_capacity(adjustments.len());
|
||||
for adjustment in adjustments {
|
||||
consistent.push(can_apply(store, &adjustment.counter, adjustment.adjustment).await?);
|
||||
}
|
||||
if inconsistent == Inconsistent::Refuse
|
||||
&& let Some((adjustment, _)) = adjustments
|
||||
.iter()
|
||||
.zip(&consistent)
|
||||
.find(|(_, consistent)| !**consistent)
|
||||
{
|
||||
return Err(EngineError::Inconsistent(format!(
|
||||
"Cannot resize budget reservation against inconsistent counter {}",
|
||||
adjustment.counter_key
|
||||
)));
|
||||
}
|
||||
let applicable: Vec<(SpendCounter, Number)> = adjustments
|
||||
.iter()
|
||||
.zip(&consistent)
|
||||
.filter(|(_, consistent)| **consistent)
|
||||
.map(|(adjustment, _)| {
|
||||
(
|
||||
adjustment.counter.clone(),
|
||||
Number::Float(adjustment.adjustment),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
increment_or_invalidate(store, applicable).await?;
|
||||
if inconsistent == Inconsistent::Drop {
|
||||
for (adjustment, _) in adjustments
|
||||
.iter()
|
||||
.zip(&consistent)
|
||||
.filter(|(_, consistent)| !**consistent)
|
||||
{
|
||||
store
|
||||
.invalidate(std::slice::from_ref(&adjustment.counter))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
Ok(consistent)
|
||||
}
|
||||
|
||||
pub(crate) async fn apply_actual(
|
||||
store: &SpendCounters,
|
||||
entries: &mut [Entry],
|
||||
actual: f64,
|
||||
inconsistent: Inconsistent,
|
||||
) -> Result<(), EngineError> {
|
||||
let adjustments: Vec<Adjustment> = entries
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, entry)| {
|
||||
let target = actual - entry.reserved_cost;
|
||||
let adjustment = target - entry.applied_adjustment;
|
||||
(adjustment != 0.0).then(|| Adjustment {
|
||||
index,
|
||||
counter: entry.counter.clone(),
|
||||
counter_key: entry.counter_key.clone(),
|
||||
target,
|
||||
adjustment,
|
||||
})
|
||||
})
|
||||
.collect();
|
||||
apply(store, &adjustments, inconsistent).await?;
|
||||
for adjustment in adjustments {
|
||||
entries[adjustment.index].applied_adjustment = adjustment.target;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn release_best_effort(store: &SpendCounters, entries: &mut [Entry]) {
|
||||
for entry in entries {
|
||||
if apply_actual(store, std::slice::from_mut(entry), 0.0, Inconsistent::Drop)
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
tracing::error!(
|
||||
counter = entry.counter_key,
|
||||
"Failed to release partial budget reservation during exception cleanup"
|
||||
);
|
||||
store.invalidate(std::slice::from_ref(&entry.counter)).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct Admission<'a> {
|
||||
store: &'a SpendCounters,
|
||||
observed: &'a [BudgetCounterSpend],
|
||||
plan: &'a Plan,
|
||||
applied: Vec<Entry>,
|
||||
cost: f64,
|
||||
}
|
||||
|
||||
impl Admission<'_> {
|
||||
async fn charge(&mut self, counters: &[Counter]) -> Result<(), EngineError> {
|
||||
let fits = counters.iter().all(|counter| {
|
||||
observed_spend(self.observed, counter) + self.cost <= counter.planned.max_budget
|
||||
});
|
||||
let groups: Vec<&[Counter]> = if fits {
|
||||
vec![counters]
|
||||
} else {
|
||||
counters.chunks(1).collect()
|
||||
};
|
||||
for group in groups {
|
||||
self.charge_group(group).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn charge_group(&mut self, group: &[Counter]) -> Result<(), EngineError> {
|
||||
let charged = self.cost;
|
||||
let entries: Vec<Entry> = group.iter().map(|c| Entry::new(c, charged)).collect();
|
||||
let ids: Vec<usize> = entries.iter().map(|entry| entry.id).collect();
|
||||
self.applied.extend(entries);
|
||||
let increments: Vec<(SpendCounter, Number)> = group
|
||||
.iter()
|
||||
.map(|counter| (counter.counter.clone(), Number::Float(charged)))
|
||||
.collect();
|
||||
let totals = match self.store.increment_many(&increments).await {
|
||||
Ok(totals) => totals,
|
||||
Err(error) => {
|
||||
tracing::warn!(%error, "Skipping budget reservation because spend counter reservation failed");
|
||||
let counters: Vec<SpendCounter> = increments.into_iter().map(|(c, _)| c).collect();
|
||||
self.store.invalidate(&counters).await;
|
||||
self.applied.retain(|entry| !ids.contains(&entry.id));
|
||||
if self.plan.fail_closed {
|
||||
return Err(EngineError::Unavailable);
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
};
|
||||
for ((counter, id), total) in group.iter().zip(ids).zip(totals) {
|
||||
if !self.applied.iter().any(|entry| entry.id == id) {
|
||||
continue;
|
||||
}
|
||||
let current = total.as_f64() - (charged - self.cost);
|
||||
if current > counter.planned.max_budget {
|
||||
self.over_budget(counter, id, current).await?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn over_budget(
|
||||
&mut self,
|
||||
counter: &Counter,
|
||||
id: usize,
|
||||
current: f64,
|
||||
) -> Result<(), EngineError> {
|
||||
if self.plan.throttle_counter.as_deref() == Some(counter.key()) {
|
||||
if let Some(position) = self.applied.iter().position(|entry| entry.id == id) {
|
||||
let mut entry = self.applied.remove(position);
|
||||
release_best_effort(self.store, std::slice::from_mut(&mut entry)).await;
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
let remaining = counter.planned.max_budget - (current - self.cost);
|
||||
if remaining <= EPSILON {
|
||||
return Err(exceeded(counter, current, None));
|
||||
}
|
||||
if self.plan.fail_closed && current - counter.planned.max_budget > EPSILON {
|
||||
return Err(exceeded(counter, current - self.cost, Some(self.cost)));
|
||||
}
|
||||
apply_actual(
|
||||
self.store,
|
||||
&mut self.applied,
|
||||
remaining,
|
||||
Inconsistent::Refuse,
|
||||
)
|
||||
.await?;
|
||||
for entry in &mut self.applied {
|
||||
entry.reserved_cost = remaining;
|
||||
entry.applied_adjustment = 0.0;
|
||||
}
|
||||
self.cost = remaining;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn reserve(
|
||||
store: &SpendCounters,
|
||||
plan: &Plan,
|
||||
observed: &[BudgetCounterSpend],
|
||||
) -> Result<Option<Reservation>, EngineError> {
|
||||
let counters = plan.counters().map_err(EngineError::Plan)?;
|
||||
let mut admission = Admission {
|
||||
store,
|
||||
observed,
|
||||
plan,
|
||||
applied: Vec::new(),
|
||||
cost: plan.reservation_cost,
|
||||
};
|
||||
if let Err(error) = admission.charge(&counters).await {
|
||||
release_best_effort(store, &mut admission.applied).await;
|
||||
return Err(error);
|
||||
}
|
||||
if admission.applied.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some(Reservation {
|
||||
reserved_cost: admission.cost,
|
||||
input_cost: plan.input_cost.min(admission.cost),
|
||||
input_tokens: plan.input_tokens,
|
||||
entries: admission.applied,
|
||||
}))
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Reseed {
|
||||
pub counter_key: String,
|
||||
pub actual_cost: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct PendingIncrement {
|
||||
pub counter_key: String,
|
||||
pub increment: f64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize)]
|
||||
pub struct Settlement {
|
||||
pub adjustments: Vec<Option<f64>>,
|
||||
pub reseed: Vec<Reseed>,
|
||||
pub pending: Vec<PendingIncrement>,
|
||||
}
|
||||
|
||||
pub struct Settle<'a> {
|
||||
pub reservation: &'a BudgetReservation,
|
||||
pub window_resets: &'a [Option<String>],
|
||||
pub observed: &'a [Option<f64>],
|
||||
pub actual: f64,
|
||||
pub apply_consistent: bool,
|
||||
}
|
||||
|
||||
fn rejected(rejected: Option<&[String]>, member: &str) -> bool {
|
||||
rejected.is_some_and(|names| names.iter().any(|name| name == member))
|
||||
}
|
||||
|
||||
fn held_adjustment(
|
||||
index: usize,
|
||||
entry: &ReservationEntry,
|
||||
window_reset: Option<&str>,
|
||||
actual: f64,
|
||||
default_reserved_cost: f64,
|
||||
now: &Moment,
|
||||
) -> Result<Option<(Adjustment, f64)>, EngineError> {
|
||||
let names = entry.rejected.as_deref();
|
||||
if rejected(names, "counter_key") {
|
||||
return Err(ReservationError::type_error(
|
||||
"str() of a reservation entry counter_key raises",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let Some(counter_key) = &entry.counter_key else {
|
||||
return Ok(None);
|
||||
};
|
||||
let settled_cost = match window_reset.map(window::parse_reset).transpose()?.flatten() {
|
||||
Some(reset_at) if reset_at <= *now => 0.0,
|
||||
_ => actual,
|
||||
};
|
||||
let reserved_cost = match entry.reserved_cost {
|
||||
Some(cost) if !rejected(names, "reserved_cost") => cost,
|
||||
_ => default_reserved_cost,
|
||||
};
|
||||
if rejected(names, "applied_adjustment") {
|
||||
return Err(ReservationError::value_error(
|
||||
"could not convert the reservation entry applied_adjustment to float",
|
||||
)
|
||||
.into());
|
||||
}
|
||||
let target = settled_cost - reserved_cost;
|
||||
let adjustment = target - entry.applied_adjustment.unwrap_or(0.0);
|
||||
if adjustment == 0.0 {
|
||||
return Ok(None);
|
||||
}
|
||||
Ok(Some((
|
||||
Adjustment {
|
||||
index,
|
||||
counter: SpendCounter::parse(counter_key).map_err(EngineError::Plan)?,
|
||||
counter_key: counter_key.clone(),
|
||||
target,
|
||||
adjustment,
|
||||
},
|
||||
settled_cost,
|
||||
)))
|
||||
}
|
||||
|
||||
pub async fn settle(
|
||||
store: &SpendCounters,
|
||||
request: &Settle<'_>,
|
||||
now: &Moment,
|
||||
) -> Result<Settlement, EngineError> {
|
||||
let entries = request.reservation.entries.as_deref().unwrap_or_default();
|
||||
if entries.iter().any(Option::is_none) {
|
||||
return Err(
|
||||
ReservationError::type_error("a budget reservation entry is not a mapping").into(),
|
||||
);
|
||||
}
|
||||
let default_reserved_cost = request.reservation.reserved_cost.unwrap_or(0.0);
|
||||
let mut adjustments = Vec::new();
|
||||
let mut settled_costs = Vec::new();
|
||||
for (index, entry) in entries.iter().enumerate() {
|
||||
let Some(entry) = entry else { continue };
|
||||
let window_reset = request.window_resets.get(index).and_then(Option::as_deref);
|
||||
if let Some((adjustment, settled_cost)) = held_adjustment(
|
||||
index,
|
||||
entry,
|
||||
window_reset,
|
||||
request.actual,
|
||||
default_reserved_cost,
|
||||
now,
|
||||
)? {
|
||||
adjustments.push(adjustment);
|
||||
settled_costs.push(settled_cost);
|
||||
}
|
||||
}
|
||||
let consistent: Vec<bool> = adjustments
|
||||
.iter()
|
||||
.map(|adjustment| {
|
||||
applies_to(
|
||||
request.observed.get(adjustment.index).copied().flatten(),
|
||||
adjustment.adjustment,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
if request.apply_consistent {
|
||||
let applicable = adjustments
|
||||
.iter()
|
||||
.zip(&consistent)
|
||||
.filter(|(_, consistent)| **consistent)
|
||||
.map(|(adjustment, _)| {
|
||||
(
|
||||
adjustment.counter.clone(),
|
||||
Number::Float(adjustment.adjustment),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
increment_or_invalidate(store, applicable).await?;
|
||||
}
|
||||
let mut settlement = Settlement {
|
||||
adjustments: vec![None; entries.len()],
|
||||
reseed: Vec::new(),
|
||||
pending: Vec::new(),
|
||||
};
|
||||
for ((adjustment, settled_cost), consistent) in
|
||||
adjustments.iter().zip(settled_costs).zip(consistent)
|
||||
{
|
||||
if !consistent {
|
||||
settlement.reseed.push(Reseed {
|
||||
counter_key: adjustment.counter_key.clone(),
|
||||
actual_cost: settled_cost,
|
||||
});
|
||||
} else if !request.apply_consistent {
|
||||
settlement.pending.push(PendingIncrement {
|
||||
counter_key: adjustment.counter_key.clone(),
|
||||
increment: adjustment.adjustment,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
settlement.adjustments[adjustment.index] = Some(adjustment.target);
|
||||
}
|
||||
Ok(settlement)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use litellm_redis_store::{
|
||||
Number,
|
||||
spend_counters::{SpendCounter, SpendCounters},
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
plan::{Plan, PlannedCounter},
|
||||
window::ErrorKind,
|
||||
};
|
||||
|
||||
fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> PlannedCounter {
|
||||
PlannedCounter {
|
||||
counter_key: key.to_owned(),
|
||||
max_budget,
|
||||
fallback_spend: 0.0,
|
||||
entity_type: entity_type.to_owned(),
|
||||
entity_id: entity_id.to_owned(),
|
||||
spend_log_entity_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn plan(counters: Vec<PlannedCounter>, cost: f64) -> Plan {
|
||||
Plan {
|
||||
counters,
|
||||
reservation_cost: cost,
|
||||
input_cost: cost / 4.0,
|
||||
input_tokens: Some(12),
|
||||
throttle_counter: None,
|
||||
fail_closed: false,
|
||||
status_code: 400,
|
||||
}
|
||||
}
|
||||
|
||||
fn store(seed: &[(&str, f64)]) -> Arc<SpendCounters> {
|
||||
let store = Arc::new(SpendCounters::new(None));
|
||||
for (key, value) in seed {
|
||||
store
|
||||
.set_local(
|
||||
&SpendCounter::parse(key).unwrap(),
|
||||
Number::Float(*value),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
fn value(store: &SpendCounters, key: &str) -> Option<f64> {
|
||||
store
|
||||
.get_local(&SpendCounter::parse(key).unwrap())
|
||||
.unwrap()
|
||||
.map(Number::as_f64)
|
||||
}
|
||||
|
||||
const KEY: &str = "spend:key:hashed";
|
||||
const TEAM: &str = "spend:team:t1";
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn counters_that_fit_are_charged_the_whole_estimate() {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, 2.0)]);
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
3.0,
|
||||
);
|
||||
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
|
||||
assert_eq!(reservation.reserved_cost, 3.0);
|
||||
assert_eq!(reservation.input_cost, 0.75);
|
||||
assert_eq!(value(&store, KEY), Some(4.0));
|
||||
assert_eq!(value(&store, TEAM), Some(5.0));
|
||||
assert_eq!(
|
||||
serde_json::to_value(&reservation).unwrap()["entries"][1],
|
||||
json!({"counter_key": TEAM, "entity_type": "Team", "entity_id": "t1", "reserved_cost": 3.0, "applied_adjustment": 0.0})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_partial_fit_shrinks_every_applied_entry_to_what_is_left() {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, 9.0)]);
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
3.0,
|
||||
);
|
||||
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
|
||||
assert_eq!(reservation.reserved_cost, 1.0);
|
||||
assert_eq!(value(&store, KEY), Some(2.0));
|
||||
assert_eq!(value(&store, TEAM), Some(10.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::spent(
|
||||
10.0,
|
||||
false,
|
||||
"Budget has been exceeded! Team=t1 Current cost: 11.0, Max budget: 10.0",
|
||||
11.0
|
||||
)]
|
||||
#[case::fail_closed(
|
||||
9.5,
|
||||
true,
|
||||
"Budget has been exceeded! Team=t1 Current cost: 9.5, Estimated request cost: 1.0, Max budget: 10.0",
|
||||
9.5
|
||||
)]
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_over_budget_counter_refuses_and_releases_what_was_charged(
|
||||
#[case] team_spend: f64,
|
||||
#[case] fail_closed: bool,
|
||||
#[case] message: &str,
|
||||
#[case] current_cost: f64,
|
||||
) {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, team_spend)]);
|
||||
let plan = Plan {
|
||||
fail_closed,
|
||||
..plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
1.0,
|
||||
)
|
||||
};
|
||||
let Err(EngineError::Exceeded(exceeded)) = reserve(&store, &plan, &[]).await else {
|
||||
panic!("expected a refusal");
|
||||
};
|
||||
assert_eq!(exceeded.message, message);
|
||||
assert_eq!(exceeded.current_cost, current_cost);
|
||||
assert_eq!(exceeded.entity_type, Some("team"));
|
||||
assert_eq!(value(&store, KEY), Some(1.0));
|
||||
assert_eq!(value(&store, TEAM), Some(team_spend));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_throttled_key_releases_its_own_entry_and_keeps_the_rest() {
|
||||
let store = store(&[(KEY, 10.0), (TEAM, 0.0)]);
|
||||
let plan = Plan {
|
||||
throttle_counter: Some(KEY.to_owned()),
|
||||
..plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
1.0,
|
||||
)
|
||||
};
|
||||
let reservation = reserve(&store, &plan, &[]).await.unwrap().unwrap();
|
||||
assert_eq!(reservation.entries.len(), 1);
|
||||
assert_eq!(reservation.entries[0].counter_key, TEAM);
|
||||
assert_eq!(value(&store, KEY), Some(10.0));
|
||||
assert_eq!(value(&store, TEAM), Some(1.0));
|
||||
}
|
||||
|
||||
fn record(reservation: &Reservation) -> BudgetReservation {
|
||||
serde_json::from_value(serde_json::to_value(reservation).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn stamp(record: &mut BudgetReservation, settlement: &Settlement) {
|
||||
for (entry, target) in record
|
||||
.entries
|
||||
.iter_mut()
|
||||
.flatten()
|
||||
.zip(&settlement.adjustments)
|
||||
{
|
||||
if let (Some(entry), Some(target)) = (entry, target) {
|
||||
entry.applied_adjustment = Some(*target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn observed(store: &SpendCounters, record: &BudgetReservation) -> Vec<Option<f64>> {
|
||||
let mut observed = Vec::new();
|
||||
for entry in record.entries.iter().flatten() {
|
||||
let counter = entry
|
||||
.as_ref()
|
||||
.and_then(|entry| entry.counter_key.as_deref())
|
||||
.and_then(|key| SpendCounter::parse(key).ok());
|
||||
observed.push(match counter {
|
||||
Some(counter) => store
|
||||
.read(&counter)
|
||||
.await
|
||||
.unwrap()
|
||||
.value()
|
||||
.map(Number::as_f64),
|
||||
None => None,
|
||||
});
|
||||
}
|
||||
observed
|
||||
}
|
||||
|
||||
async fn settle_as(
|
||||
store: &SpendCounters,
|
||||
record: &BudgetReservation,
|
||||
resets: &[Option<String>],
|
||||
actual: f64,
|
||||
apply_consistent: bool,
|
||||
) -> Result<Settlement, EngineError> {
|
||||
let observed = observed(store, record).await;
|
||||
settle(
|
||||
store,
|
||||
&Settle {
|
||||
reservation: record,
|
||||
window_resets: resets,
|
||||
observed: &observed,
|
||||
actual,
|
||||
apply_consistent,
|
||||
},
|
||||
&Moment::from_unix_micros(NOW),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn settle_now(
|
||||
store: &SpendCounters,
|
||||
record: &BudgetReservation,
|
||||
resets: &[Option<String>],
|
||||
actual: f64,
|
||||
) -> Result<Settlement, EngineError> {
|
||||
settle_as(store, record, resets, actual, true).await
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_settlement_that_leaves_writes_to_the_caller_hands_back_the_consistent_increments() {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
2.0,
|
||||
);
|
||||
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
|
||||
store
|
||||
.invalidate(&[SpendCounter::parse(TEAM).unwrap()])
|
||||
.await;
|
||||
let settlement = settle_as(&store, &record, &[], 0.5, false).await.unwrap();
|
||||
assert_eq!(value(&store, KEY), Some(3.0));
|
||||
assert_eq!(
|
||||
settlement.pending,
|
||||
vec![PendingIncrement {
|
||||
counter_key: KEY.to_owned(),
|
||||
increment: -1.5
|
||||
}]
|
||||
);
|
||||
assert_eq!(
|
||||
settlement.reseed,
|
||||
vec![Reseed {
|
||||
counter_key: TEAM.to_owned(),
|
||||
actual_cost: 0.5
|
||||
}]
|
||||
);
|
||||
assert_eq!(settlement.adjustments, vec![None, Some(-1.5)]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_counter_the_caller_saw_missing_is_reseeded_whatever_the_store_holds_now() {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
|
||||
let settlement = settle(
|
||||
&store,
|
||||
&Settle {
|
||||
reservation: &record,
|
||||
window_resets: &[],
|
||||
observed: &[None],
|
||||
actual: 0.5,
|
||||
apply_consistent: true,
|
||||
},
|
||||
&Moment::from_unix_micros(NOW),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(value(&store, KEY), Some(3.0));
|
||||
assert_eq!(
|
||||
settlement.reseed,
|
||||
vec![Reseed {
|
||||
counter_key: KEY.to_owned(),
|
||||
actual_cost: 0.5
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
const NOW: i128 = 1_763_035_200_000_000;
|
||||
|
||||
#[rstest]
|
||||
#[case::success(0.4, 1.4)]
|
||||
#[case::refund(0.0, 1.0)]
|
||||
#[tokio::test]
|
||||
async fn settling_moves_each_counter_from_the_hold_to_the_actual_cost(
|
||||
#[case] actual: f64,
|
||||
#[case] after: f64,
|
||||
) {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let mut record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
|
||||
let settlement = settle_now(&store, &record, &[], actual).await.unwrap();
|
||||
assert_eq!(settlement.adjustments, vec![Some(actual - 2.0)]);
|
||||
stamp(&mut record, &settlement);
|
||||
let again = settle_now(&store, &record, &[], actual).await.unwrap();
|
||||
assert_eq!(again.adjustments, vec![None]);
|
||||
assert_eq!(value(&store, KEY), Some(after));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::missing(None, 0.4, None)]
|
||||
#[case::would_go_negative(Some(0.5), 0.0, Some(0.5))]
|
||||
#[tokio::test]
|
||||
async fn settling_an_inconsistent_counter_reports_it_and_leaves_it_alone(
|
||||
#[case] before_settle: Option<f64>,
|
||||
#[case] actual: f64,
|
||||
#[case] after: Option<f64>,
|
||||
) {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
2.0,
|
||||
);
|
||||
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
|
||||
let key = SpendCounter::parse(KEY).unwrap();
|
||||
match before_settle {
|
||||
Some(spend) => store.set_local(&key, Number::Float(spend), None).unwrap(),
|
||||
None => store.invalidate(std::slice::from_ref(&key)).await,
|
||||
}
|
||||
let settlement = settle_now(&store, &record, &[], actual).await.unwrap();
|
||||
assert_eq!(
|
||||
settlement.reseed,
|
||||
vec![Reseed {
|
||||
counter_key: KEY.to_owned(),
|
||||
actual_cost: actual
|
||||
}]
|
||||
);
|
||||
assert_eq!(value(&store, KEY), after);
|
||||
assert_eq!(value(&store, TEAM), Some(2.0 + (actual - 2.0)));
|
||||
assert_eq!(settlement.adjustments, vec![Some(actual - 2.0); 2]);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::reset_before_settlement("2025-11-13T11:59:59+00:00", 0.0)]
|
||||
#[case::reset_at_settlement("2025-11-13T12:00:00Z", 0.0)]
|
||||
#[case::still_open("2025-11-13T12:00:00.000001+00:00", 0.4)]
|
||||
#[case::unparsable("next month", 0.4)]
|
||||
#[tokio::test]
|
||||
async fn an_entry_whose_window_reset_before_settlement_settles_at_no_cost(
|
||||
#[case] reset: &str,
|
||||
#[case] settled: f64,
|
||||
) {
|
||||
let store = store(&[(KEY, 1.0), (TEAM, 0.0)]);
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
counter(TEAM, 10.0, "Team", "t1"),
|
||||
],
|
||||
2.0,
|
||||
);
|
||||
let record = record(&reserve(&store, &plan, &[]).await.unwrap().unwrap());
|
||||
store
|
||||
.invalidate(&[SpendCounter::parse(TEAM).unwrap()])
|
||||
.await;
|
||||
let settlement = settle_now(&store, &record, &[Some(reset.to_owned()), None], 0.4)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(value(&store, KEY), Some(1.0 + settled));
|
||||
assert_eq!(
|
||||
settlement.adjustments,
|
||||
vec![Some(settled - 2.0), Some(0.4 - 2.0)]
|
||||
);
|
||||
assert_eq!(
|
||||
settlement.reseed,
|
||||
vec![Reseed {
|
||||
counter_key: TEAM.to_owned(),
|
||||
actual_cost: 0.4
|
||||
}]
|
||||
);
|
||||
}
|
||||
|
||||
fn held(entries: serde_json::Value) -> BudgetReservation {
|
||||
serde_json::from_value(json!({"reserved_cost": 2.0, "entries": entries})).unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_a_mapping(json!([{"counter_key": KEY, "reserved_cost": 2.0}, null]), None, ErrorKind::Type)]
|
||||
#[case::naive_reset(json!([{"counter_key": KEY, "reserved_cost": 2.0}]), Some("2025-11-13T12:00:00"), ErrorKind::Type)]
|
||||
#[case::applied_adjustment_float_refuses(json!([{"counter_key": KEY, "rejected": ["applied_adjustment"]}]), None, ErrorKind::Value)]
|
||||
#[tokio::test]
|
||||
async fn an_entry_python_raises_on_fails_the_settlement_before_any_counter_moves(
|
||||
#[case] entries: serde_json::Value,
|
||||
#[case] reset: Option<&str>,
|
||||
#[case] kind: ErrorKind,
|
||||
) {
|
||||
let store = store(&[(KEY, 3.0)]);
|
||||
let Err(EngineError::Reservation(error)) =
|
||||
settle_now(&store, &held(entries), &[reset.map(str::to_owned)], 0.5).await
|
||||
else {
|
||||
panic!("expected a reservation error");
|
||||
};
|
||||
assert_eq!(error.kind, kind);
|
||||
assert_eq!(value(&store, KEY), Some(3.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_counter_key(json!([{"reserved_cost": 2.0}]), 3.0, vec![None])]
|
||||
#[case::reserved_cost_float_refuses(json!([{"counter_key": KEY, "rejected": ["reserved_cost"]}]), 1.5, vec![Some(-1.5)])]
|
||||
#[case::reserved_cost_missing(json!([{"counter_key": KEY}]), 1.5, vec![Some(-1.5)])]
|
||||
#[case::own_reserved_cost(json!([{"counter_key": KEY, "reserved_cost": 1.0, "applied_adjustment": 0.25}]), 2.25, vec![Some(-0.5)])]
|
||||
#[tokio::test]
|
||||
async fn each_entry_member_is_read_as_budgets_reads_it(
|
||||
#[case] entries: serde_json::Value,
|
||||
#[case] after: f64,
|
||||
#[case] adjustments: Vec<Option<f64>>,
|
||||
) {
|
||||
let store = store(&[(KEY, 3.0)]);
|
||||
let settlement = settle_now(&store, &held(entries), &[], 0.5).await.unwrap();
|
||||
assert_eq!(settlement.adjustments, adjustments);
|
||||
assert_eq!(value(&store, KEY), Some(after));
|
||||
}
|
||||
}
|
||||
16
litellm/rust/crates/callbacks-budgets/src/lib.rs
Normal file
16
litellm/rust/crates/callbacks-budgets/src/lib.rs
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
mod admission;
|
||||
mod checks;
|
||||
mod engine;
|
||||
mod plan;
|
||||
mod window;
|
||||
|
||||
pub use admission::{Admission, Budgets};
|
||||
pub use checks::{Check, Decision, Refusal, decide};
|
||||
pub use engine::{
|
||||
BudgetCounterSpend, EngineError, Entry, Exceeded, PendingIncrement, Reseed, Reservation,
|
||||
Settle, Settlement, UNAVAILABLE, reserve, settle,
|
||||
};
|
||||
pub use litellm_callbacks::{BudgetPlan, BudgetReservation};
|
||||
pub use litellm_python_compat::number::PyNumber;
|
||||
pub use plan::{Plan, PlannedCounter};
|
||||
pub use window::{ErrorKind, Moment, ReservationError};
|
||||
114
litellm/rust/crates/callbacks-budgets/src/plan.rs
Normal file
114
litellm/rust/crates/callbacks-budgets/src/plan.rs
Normal file
|
|
@ -0,0 +1,114 @@
|
|||
use litellm_callbacks::{BudgetPlan, BudgetPlanCounter};
|
||||
use litellm_redis_store::spend_counters::{NotASpendCounter, SpendCounter};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct PlannedCounter {
|
||||
pub counter_key: String,
|
||||
pub max_budget: f64,
|
||||
pub fallback_spend: f64,
|
||||
pub entity_type: String,
|
||||
pub entity_id: String,
|
||||
pub spend_log_entity_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct Plan {
|
||||
pub counters: Vec<PlannedCounter>,
|
||||
pub reservation_cost: f64,
|
||||
pub input_cost: f64,
|
||||
pub input_tokens: Option<i64>,
|
||||
pub throttle_counter: Option<String>,
|
||||
pub fail_closed: bool,
|
||||
pub status_code: u16,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct MissingMember(pub &'static str);
|
||||
|
||||
impl std::fmt::Display for MissingMember {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "the budget plan has no {}", self.0)
|
||||
}
|
||||
}
|
||||
|
||||
fn required<T>(value: Option<T>, member: &'static str) -> Result<T, MissingMember> {
|
||||
value.ok_or(MissingMember(member))
|
||||
}
|
||||
|
||||
impl TryFrom<BudgetPlanCounter> for PlannedCounter {
|
||||
type Error = MissingMember;
|
||||
|
||||
fn try_from(counter: BudgetPlanCounter) -> Result<Self, Self::Error> {
|
||||
Ok(Self {
|
||||
counter_key: required(counter.counter_key, "counters[].counter_key")?,
|
||||
max_budget: required(counter.max_budget, "counters[].max_budget")?,
|
||||
fallback_spend: required(counter.fallback_spend, "counters[].fallback_spend")?,
|
||||
entity_type: required(counter.entity_type, "counters[].entity_type")?,
|
||||
entity_id: required(counter.entity_id, "counters[].entity_id")?,
|
||||
spend_log_entity_id: counter.spend_log_entity_id,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl TryFrom<BudgetPlan> for Plan {
|
||||
type Error = MissingMember;
|
||||
|
||||
fn try_from(plan: BudgetPlan) -> Result<Self, Self::Error> {
|
||||
Ok(Self {
|
||||
counters: required(plan.counters, "counters")?
|
||||
.into_iter()
|
||||
.map(PlannedCounter::try_from)
|
||||
.collect::<Result<_, _>>()?,
|
||||
reservation_cost: required(plan.reservation_cost, "reservation_cost")?,
|
||||
input_cost: required(plan.input_cost, "input_cost")?,
|
||||
input_tokens: plan.input_tokens,
|
||||
throttle_counter: plan.throttle_counter,
|
||||
fail_closed: plan.fail_closed.unwrap_or_default(),
|
||||
status_code: required(plan.status_code, "status_code")?,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct Counter {
|
||||
pub planned: PlannedCounter,
|
||||
pub counter: SpendCounter,
|
||||
}
|
||||
|
||||
impl Counter {
|
||||
pub fn key(&self) -> &str {
|
||||
&self.planned.counter_key
|
||||
}
|
||||
}
|
||||
|
||||
impl Plan {
|
||||
pub(crate) fn counters(&self) -> Result<Vec<Counter>, NotASpendCounter> {
|
||||
self.counters
|
||||
.iter()
|
||||
.map(|planned| {
|
||||
Ok(Counter {
|
||||
counter: SpendCounter::parse(&planned.counter_key)?,
|
||||
planned: planned.clone(),
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn entity_type_value(entity_type: &str) -> Option<&'static str> {
|
||||
match entity_type {
|
||||
"Key" => Some("key"),
|
||||
"Team" => Some("team"),
|
||||
"TeamMember" => Some("team_member"),
|
||||
"User" => Some("user"),
|
||||
"EndUser" => Some("end_user"),
|
||||
"Tag" => Some("tag"),
|
||||
"Model access group" => Some("model_access_group"),
|
||||
"Organization" => Some("organization"),
|
||||
"Project" => Some("project"),
|
||||
"customer_org" => Some("customer_org"),
|
||||
"customer_team" => Some("customer_team"),
|
||||
"customer_member" => Some("customer_member"),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
109
litellm/rust/crates/callbacks-budgets/src/window.rs
Normal file
109
litellm/rust/crates/callbacks-budgets/src/window.rs
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_python_compat::datetime::{IsoDateTime, fromisoformat};
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum ErrorKind {
|
||||
Type,
|
||||
Value,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ReservationError {
|
||||
pub kind: ErrorKind,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
impl ReservationError {
|
||||
pub(crate) fn type_error(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: ErrorKind::Type,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn value_error(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: ErrorKind::Value,
|
||||
message: message.into(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for ReservationError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str(&self.message)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
|
||||
pub struct Moment(i128);
|
||||
|
||||
impl Moment {
|
||||
pub fn now() -> Self {
|
||||
let since = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default();
|
||||
Self(i128::try_from(since.as_micros()).unwrap_or(i128::MAX))
|
||||
}
|
||||
|
||||
pub const fn from_unix_micros(micros: i128) -> Self {
|
||||
Self(micros)
|
||||
}
|
||||
}
|
||||
|
||||
/// A reservation entry's `window_reset_at`, read as the proxy's `_coerce_datetime` reads it:
|
||||
/// `datetime.fromisoformat(text.replace("Z", "+00:00"))`, `None` where that raises.
|
||||
pub(crate) fn parse_reset(text: &str) -> Result<Option<Moment>, ReservationError> {
|
||||
match fromisoformat(&text.replace('Z', "+00:00")) {
|
||||
None => Ok(None),
|
||||
Some(IsoDateTime::Aware(micros)) => Ok(Some(Moment(i128::from(micros)))),
|
||||
Some(IsoDateTime::Naive(_)) => Err(ReservationError::type_error(
|
||||
"can't compare offset-naive and offset-aware datetimes",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
const NOON: i128 = 1_763_035_200_000_000;
|
||||
|
||||
#[rstest]
|
||||
#[case::isoformat("2025-11-13T12:00:00+00:00", Some(NOON))]
|
||||
#[case::zulu("2025-11-13T12:00:00Z", Some(NOON))]
|
||||
#[case::micros("2025-11-13T12:00:00.000001+00:00", Some(NOON + 1))]
|
||||
#[case::short_fraction("2025-11-13T12:00:00.5+00:00", Some(NOON + 500_000))]
|
||||
#[case::long_fraction("2025-11-13T12:00:00.1234567+00:00", Some(NOON + 123_456))]
|
||||
#[case::offset("2025-11-13T13:30:00+01:30", Some(NOON))]
|
||||
#[case::negative_offset("2025-11-13T11:00:00-01:00", Some(NOON))]
|
||||
#[case::space_separator("2025-11-13 12:00+00:00", Some(NOON))]
|
||||
#[case::basic("20251113T120000+0000", Some(NOON))]
|
||||
#[case::not_a_date("soon", None)]
|
||||
#[case::bad_day("2025-02-30T12:00:00+00:00", None)]
|
||||
#[case::bad_hour("2025-11-13T25:00:00+00:00", None)]
|
||||
#[case::midnight_rollover("2025-11-13T24:00:00+00:00", Some(NOON + 43_200_000_000))]
|
||||
#[case::trailing("2025-11-13T12:00:00+00:00 ", None)]
|
||||
fn an_aware_reset_parses_as_python_fromisoformat_does(
|
||||
#[case] text: &str,
|
||||
#[case] expected: Option<i128>,
|
||||
) {
|
||||
assert_eq!(
|
||||
parse_reset(text).unwrap(),
|
||||
expected.map(Moment::from_unix_micros)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::date_only("2025-11-13")]
|
||||
#[case::no_offset("2025-11-13T12:00:00")]
|
||||
fn a_naive_reset_raises_the_comparison_type_error(#[case] text: &str) {
|
||||
assert_eq!(
|
||||
parse_reset(text).unwrap_err(),
|
||||
ReservationError::type_error("can't compare offset-naive and offset-aware datetimes")
|
||||
);
|
||||
}
|
||||
}
|
||||
230
litellm/rust/crates/callbacks-budgets/tests/admission.rs
Normal file
230
litellm/rust/crates/callbacks-budgets/tests/admission.rs
Normal file
|
|
@ -0,0 +1,230 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use litellm_callbacks::{BudgetPlan, BudgetPlanCounter, BudgetReservation};
|
||||
use litellm_callbacks_budgets::{Admission, BudgetCounterSpend, Budgets, Settle, Settlement};
|
||||
use litellm_redis_store::{
|
||||
Number,
|
||||
spend_counters::{SpendCounter, SpendCounters},
|
||||
};
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
fn counter(key: &str, max_budget: f64, entity_type: &str, entity_id: &str) -> BudgetPlanCounter {
|
||||
BudgetPlanCounter {
|
||||
counter_key: Some(key.to_owned()),
|
||||
max_budget: Some(max_budget),
|
||||
fallback_spend: Some(0.0),
|
||||
entity_type: Some(entity_type.to_owned()),
|
||||
entity_id: Some(entity_id.to_owned()),
|
||||
spend_log_entity_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn plan(counters: Vec<BudgetPlanCounter>, cost: f64) -> BudgetPlan {
|
||||
BudgetPlan {
|
||||
counters: Some(counters),
|
||||
reservation_cost: Some(cost),
|
||||
input_cost: Some(cost / 4.0),
|
||||
input_tokens: Some(12),
|
||||
throttle_counter: None,
|
||||
fail_closed: Some(false),
|
||||
status_code: Some(400),
|
||||
reservation_id: Some("reservation-1".to_owned()),
|
||||
..BudgetPlan::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn store(seed: &[(&str, f64)]) -> Arc<SpendCounters> {
|
||||
let store = Arc::new(SpendCounters::new(None));
|
||||
for (key, value) in seed {
|
||||
store
|
||||
.set_local(
|
||||
&SpendCounter::parse(key).unwrap(),
|
||||
Number::Float(*value),
|
||||
None,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
store
|
||||
}
|
||||
|
||||
fn value(store: &SpendCounters, key: &str) -> Option<f64> {
|
||||
store
|
||||
.get_local(&SpendCounter::parse(key).unwrap())
|
||||
.unwrap()
|
||||
.map(Number::as_f64)
|
||||
}
|
||||
|
||||
const KEY: &str = "spend:key:hashed";
|
||||
|
||||
fn observed(spends: &[(&str, f64)]) -> Vec<BudgetCounterSpend> {
|
||||
spends
|
||||
.iter()
|
||||
.map(|(key, spend)| BudgetCounterSpend {
|
||||
counter_key: Some((*key).to_owned()),
|
||||
spend: Some(*spend),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn admit(budgets: &Budgets, plan: &BudgetPlan, spends: &[(&str, f64)]) -> Admission {
|
||||
budgets
|
||||
.admit(plan.clone(), &observed(spends))
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
async fn held(budgets: &Budgets, plan: &BudgetPlan) -> BudgetReservation {
|
||||
let Admission::Held { reservation } = admit(budgets, plan, &[]).await else {
|
||||
panic!("expected a reservation");
|
||||
};
|
||||
reservation
|
||||
}
|
||||
|
||||
fn stamp(record: &mut BudgetReservation, adjustments: &[Option<f64>]) {
|
||||
for (entry, target) in record.entries.iter_mut().flatten().zip(adjustments) {
|
||||
if let (Some(entry), Some(target)) = (entry, target) {
|
||||
entry.applied_adjustment = Some(*target);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn settle(
|
||||
budgets: &Budgets,
|
||||
store: &SpendCounters,
|
||||
record: &BudgetReservation,
|
||||
actual: f64,
|
||||
) -> Settlement {
|
||||
let mut observed = Vec::new();
|
||||
for entry in record.entries.iter().flatten().flatten() {
|
||||
let counter = SpendCounter::parse(entry.counter_key.as_deref().unwrap()).unwrap();
|
||||
observed.push(
|
||||
store
|
||||
.read(&counter)
|
||||
.await
|
||||
.unwrap()
|
||||
.value()
|
||||
.map(Number::as_f64),
|
||||
);
|
||||
}
|
||||
budgets
|
||||
.settle(&Settle {
|
||||
reservation: record,
|
||||
window_resets: &[],
|
||||
observed: &observed,
|
||||
actual,
|
||||
apply_consistent: true,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_admission_holds_and_returns_the_hold_as_the_reservation() {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let mut record = held(&budgets, &plan).await;
|
||||
assert_eq!(value(&store, KEY), Some(3.0));
|
||||
assert_eq!(record.reserved_cost, Some(2.0));
|
||||
assert_eq!(record.input_cost, Some(0.5));
|
||||
assert_eq!(record.input_tokens, Some(12));
|
||||
assert_eq!(record.finalized, None);
|
||||
let settled = settle(&budgets, &store, &record, 0.5).await;
|
||||
assert_eq!(settled.adjustments, vec![Some(-1.5)]);
|
||||
assert!(settled.reseed.is_empty());
|
||||
assert_eq!(value(&store, KEY), Some(1.5));
|
||||
stamp(&mut record, &settled.adjustments);
|
||||
let again = settle(&budgets, &store, &record, 0.5).await;
|
||||
assert_eq!(again.adjustments, vec![None]);
|
||||
assert_eq!(value(&store, KEY), Some(1.5));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::no_counters(BudgetPlan { counters: None, ..plan(vec![], 2.0) })]
|
||||
#[case::no_status_code(BudgetPlan { status_code: None, ..plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0) })]
|
||||
#[tokio::test]
|
||||
async fn a_plan_missing_a_required_member_fails_without_holding(#[case] plan: BudgetPlan) {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
assert!(budgets.admit(plan, &[]).await.is_err());
|
||||
assert_eq!(value(&store, KEY), Some(1.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::once(&[0.25])]
|
||||
#[case::twice_on_the_same_cost(&[0.25, 0.25])]
|
||||
#[case::grown_then_shrunk(&[0.75, 0.25])]
|
||||
#[tokio::test]
|
||||
async fn a_settlement_through_the_reservation_dict_lands_on_the_last_cost(#[case] costs: &[f64]) {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let mut record = held(&budgets, &plan).await;
|
||||
for cost in costs {
|
||||
let settled = settle(&budgets, &store, &record, *cost).await;
|
||||
stamp(&mut record, &settled.adjustments);
|
||||
}
|
||||
assert_eq!(value(&store, KEY), Some(1.0 + costs[costs.len() - 1]));
|
||||
assert_eq!(record.finalized, None);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn a_refusal_returns_the_typed_budget_error() {
|
||||
let store = store(&[(KEY, 10.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let Admission::Refused { kind, fields } = admit(&budgets, &plan, &[]).await else {
|
||||
panic!("expected a refusal");
|
||||
};
|
||||
assert_eq!(kind, "budget_exceeded");
|
||||
assert_eq!(
|
||||
serde_json::Value::Object(fields),
|
||||
json!({
|
||||
"current_cost": 12.0,
|
||||
"max_budget": 10.0,
|
||||
"message": "Budget has been exceeded! Key=hashed Current cost: 12.0, Max budget: 10.0",
|
||||
"entity_type": "key",
|
||||
"entity_id": "hashed",
|
||||
})
|
||||
);
|
||||
assert_eq!(value(&store, KEY), Some(10.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::observed(&[(KEY, 1.0)])]
|
||||
#[case::not_observed(&[])]
|
||||
#[tokio::test]
|
||||
async fn a_counter_python_observed_is_charged_on_the_store(#[case] observed: &[(&str, f64)]) {
|
||||
let store = store(&[(KEY, 1.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
let plan = plan(vec![counter(KEY, 10.0, "Key", "hashed")], 2.0);
|
||||
let Admission::Held { .. } = admit(&budgets, &plan, observed).await else {
|
||||
panic!("expected the reservation");
|
||||
};
|
||||
assert_eq!(value(&store, KEY), Some(3.0));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[tokio::test]
|
||||
async fn an_observed_spend_with_no_room_charges_counters_one_at_a_time_and_refuses_the_full_one() {
|
||||
const TEAM: &str = "spend:team:team-1";
|
||||
let store = store(&[(KEY, 10.0), (TEAM, 0.0)]);
|
||||
let budgets = Budgets::new(Arc::clone(&store));
|
||||
let plan = plan(
|
||||
vec![
|
||||
counter(TEAM, 10.0, "Team", "team-1"),
|
||||
counter(KEY, 10.0, "Key", "hashed"),
|
||||
],
|
||||
2.0,
|
||||
);
|
||||
let Admission::Refused { kind, .. } = admit(&budgets, &plan, &[(KEY, 10.0), (TEAM, 0.0)]).await
|
||||
else {
|
||||
panic!("expected the refusal");
|
||||
};
|
||||
assert_eq!(kind, "budget_exceeded");
|
||||
assert_eq!(value(&store, KEY), Some(10.0));
|
||||
assert_eq!(value(&store, TEAM), Some(0.0));
|
||||
}
|
||||
|
|
@ -159,6 +159,7 @@
|
|||
"endpoint_type",
|
||||
"request_body",
|
||||
"chunks",
|
||||
"error"
|
||||
"error",
|
||||
"head"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ struct PassThroughStream {
|
|||
|
||||
/// What the Messages stream iterator keeps for its end-of-stream billing.
|
||||
struct DeliveredStream {
|
||||
head: Py<PyAny>,
|
||||
chunks: Py<PyList>,
|
||||
first_chunk: Option<Py<PyAny>>,
|
||||
}
|
||||
|
|
@ -65,22 +66,14 @@ pub struct LegacyLogging {
|
|||
/// When the provider request went out, until its window is recorded (Messages only).
|
||||
api_started: Option<f64>,
|
||||
host_answered: bool,
|
||||
/// Whether the call's Messages stream starts only on its first read, as Python's iterator
|
||||
/// does. Only `azure_ai`: by the owner's ruling `anthropic` keeps main's billing of an
|
||||
/// unread stream.
|
||||
/// Whether the call's stream starts only on its first read, as Python's Messages iterator
|
||||
/// does.
|
||||
stream_starts_on_first_read: bool,
|
||||
/// Set once the route sends through the host's own client, whose timed `post` already
|
||||
/// reports the exchange.
|
||||
host_client: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
/// The one provider whose unread Messages stream bills nothing, as Python's does.
|
||||
const STREAM_STARTS_ON_FIRST_READ_PROVIDER: &str = "azure_ai";
|
||||
|
||||
/// The one provider whose exchange through the host's client is timed only by that client, as
|
||||
/// Python's is. By the owner's ruling `anthropic` keeps main's second report.
|
||||
const HOST_CLIENT_TIMED_PROVIDER: &str = "azure_ai";
|
||||
|
||||
fn retry_reason(reason: &RetryReason) -> Value {
|
||||
match reason {
|
||||
RetryReason::ThinkingSignature => serde_json::json!({"kind": "thinking_signature"}),
|
||||
|
|
@ -504,6 +497,7 @@ impl LegacyLogging {
|
|||
),
|
||||
&stream.chunks,
|
||||
error,
|
||||
&stream.head,
|
||||
),
|
||||
);
|
||||
match scheduled {
|
||||
|
|
@ -675,8 +669,7 @@ impl LegacyLogging {
|
|||
)?;
|
||||
if self.operation == LoggingOperation::Messages
|
||||
&& self.asynchronous
|
||||
&& !(context.custom_llm_provider == HOST_CLIENT_TIMED_PROVIDER
|
||||
&& self.host_client.load(Ordering::Acquire))
|
||||
&& !self.host_client.load(Ordering::Acquire)
|
||||
&& self.times_provider_exchange(py)?
|
||||
{
|
||||
self.api_started = Some(epoch_seconds());
|
||||
|
|
@ -729,8 +722,7 @@ impl LegacyLogging {
|
|||
) -> PyResult<HookStep<Self, ()>> {
|
||||
use litellm_host::interceptors::{Cost, ResultSource};
|
||||
|
||||
self.stream_starts_on_first_read =
|
||||
facts.provider.provider == STREAM_STARTS_ON_FIRST_READ_PROVIDER;
|
||||
self.stream_starts_on_first_read = self.operation == LoggingOperation::Messages;
|
||||
self.cache_key = match &facts.source {
|
||||
ResultSource::Provider => None,
|
||||
ResultSource::Cache { key } => Some(key.clone()),
|
||||
|
|
@ -940,6 +932,7 @@ impl LegacyLogging {
|
|||
),
|
||||
)?;
|
||||
self.stream = Some(DeliveredStream {
|
||||
head: head.clone_ref(py),
|
||||
chunks: PyList::empty(py).unbind(),
|
||||
first_chunk: None,
|
||||
});
|
||||
|
|
@ -995,6 +988,7 @@ impl PythonOwned for LegacyLogging {
|
|||
visit.call(&self.response)?;
|
||||
visit.call(&self.error)?;
|
||||
if let Some(stream) = &self.stream {
|
||||
visit.call(&stream.head)?;
|
||||
visit.call(&stream.chunks)?;
|
||||
visit.call(&stream.first_chunk)?;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,7 +107,7 @@ FAKES = {
|
|||
'stream_success': lambda logger, url_route, endpoint_type, request_body, chunks, start, end, first_chunk, stream_teardown: logger.record(
|
||||
'stream_success', list(chunks)
|
||||
),
|
||||
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error: logger.record('stream_failure', error),
|
||||
'stream_failure': lambda logger, endpoint_type, request_body, chunks, error, head: logger.record('stream_failure', error),
|
||||
}
|
||||
assert FAKES.keys() == CONTRACT.keys(), sorted(FAKES.keys() ^ CONTRACT.keys())
|
||||
for name, fake in FAKES.items():
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use litellm_callbacks::Moment;
|
||||
use litellm_python_compat::datetime::UtcDateTime;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::settings::{Granularity, Settings};
|
||||
|
|
@ -58,16 +59,18 @@ impl Stamp {
|
|||
pub fn utc(time: SystemTime) -> Self {
|
||||
let elapsed = time.duration_since(UNIX_EPOCH).unwrap_or_default();
|
||||
let seconds = elapsed.as_secs();
|
||||
let (year, month, day) = civil_from_days((seconds / 86_400) as i64);
|
||||
let of_day = seconds % 86_400;
|
||||
let moment = UtcDateTime::from_timestamp(
|
||||
i64::try_from(seconds).unwrap_or(i64::MAX),
|
||||
elapsed.subsec_micros(),
|
||||
);
|
||||
Self {
|
||||
year,
|
||||
month,
|
||||
day,
|
||||
hour: (of_day / 3600) as u32,
|
||||
minute: (of_day / 60 % 60) as u32,
|
||||
second: (of_day % 60) as u32,
|
||||
micros: elapsed.subsec_micros(),
|
||||
year: moment.year,
|
||||
month: moment.month,
|
||||
day: moment.day,
|
||||
hour: moment.hour,
|
||||
minute: moment.minute,
|
||||
second: moment.second,
|
||||
micros: moment.microsecond,
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -84,24 +87,6 @@ impl Stamp {
|
|||
}
|
||||
}
|
||||
|
||||
fn civil_from_days(days: i64) -> (i64, u32, u32) {
|
||||
let shifted = days + 719_468;
|
||||
let era = shifted.div_euclid(146_097);
|
||||
let day_of_era = shifted.rem_euclid(146_097);
|
||||
let year_of_era =
|
||||
(day_of_era - day_of_era / 1460 + day_of_era / 36_524 - day_of_era / 146_096) / 365;
|
||||
let day_of_year = day_of_era - (365 * year_of_era + year_of_era / 4 - year_of_era / 100);
|
||||
let month_index = (5 * day_of_year + 2) / 153;
|
||||
let day = (day_of_year - (153 * month_index + 2) / 5 + 1) as u32;
|
||||
let month = if month_index < 10 {
|
||||
month_index + 3
|
||||
} else {
|
||||
month_index - 9
|
||||
} as u32;
|
||||
let year = year_of_era + era * 400 + i64::from(month <= 2);
|
||||
(year, month, day)
|
||||
}
|
||||
|
||||
pub fn logging_id(start: &Stamp, id: &str) -> String {
|
||||
format!("time-{}_{id}", start.clock_micros())
|
||||
}
|
||||
|
|
@ -212,15 +197,7 @@ pub fn parent(key: &str) -> &str {
|
|||
}
|
||||
|
||||
pub fn quote(key: &str) -> String {
|
||||
let mut encoded = String::with_capacity(key.len());
|
||||
for byte in key.bytes() {
|
||||
if byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'.' | b'-' | b'~' | b'/') {
|
||||
encoded.push(byte as char);
|
||||
} else {
|
||||
encoded.push_str(&format!("%{byte:02X}"));
|
||||
}
|
||||
}
|
||||
encoded
|
||||
litellm_python_compat::urllib::quote(key, "/")
|
||||
}
|
||||
|
||||
pub fn dns_suffix(region: Option<&str>) -> &'static str {
|
||||
|
|
|
|||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Reference in a new issue